judgelint 0.1.0

data/lib/judgelint/scaffold.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # Files written by `judgelint init`.
+  module Scaffold
+    CONFIG = <<~YAML
+      # judgelint settings. See https://github.com/jpcamara/judgelint#configuration
+      models: [typesafe]        # `judgelint providers` lists the others; list two to average them
+      # threshold: 0.7
+      # ignore: ["vendor/**"]
+      # max_cost_usd: 1.00
+    YAML
+
+    # A spec: each bullet is a convention judgelint checks. A README with
+    # paths: frontmatter works the same way, next to the code it governs.
+    SPEC = <<~MARKDOWN
+      ---
+      paths: ["**/*.rb"]
+      excludes: ["test/**", "spec/**"]
+      ---
+      # Ruby conventions
+
+      - **No silent rescues.** A rescue logs the error, re-raises it, or turns it
+        into a result the caller sees. Swallowed exceptions hide bugs until they
+        cost us data.
+      - **Error messages say what to do next.** A raised or returned error names
+        what was wrong and what would work instead.
+    MARKDOWN
+
+    PASS_EXAMPLE = <<~RUBY
+      def fetch_profile(id)
+        client.profile(id)
+      rescue Client::Timeout => e
+        logger.warn("profile \#{id} timed out: \#{e.message}")
+        raise
+      end
+    RUBY
+
+    FAIL_EXAMPLE = <<~RUBY
+      def fetch_profile(id)
+        client.profile(id)
+      rescue StandardError
+        nil
+      end
+    RUBY
+
+    def self.write(root, out)
+      examples = ".judgelint/examples/ruby/no-silent-rescues"
+      files = {
+        Config::FILE => CONFIG,
+        ".judgelint/rules/ruby.md" => SPEC,
+        File.join(examples, "pass", "logs_and_reraises.rb") => PASS_EXAMPLE,
+        File.join(examples, "fail", "returns_nil.rb") => FAIL_EXAMPLE
+      }
+      files.each do |name, content|
+        path = root == "." ? name : File.join(root, name)
+        if File.exist?(path)
+          out.puts "exists  #{path}"
+        else
+          FileUtils.mkdir_p(File.dirname(path))
+          File.write(path, content)
+          out.puts "create  #{path}"
+        end
+      end
+    end
+  end
+end

data/lib/judgelint/source.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # What the model is shown: one file, or every file in a directory, with each
+  # line numbered so a question can point at a range, followed by any context
+  # files the rule names, which are there to read and never judged.
+  #
+  # Inline suppressions turn a rule off where a person has decided the code is
+  # fine, and say why: a comment (#, //, --, /*, <!--, or ;) starting with
+  # `judgelint:disable <rule ids> -- <reason>` covers the window it falls in,
+  # and `judgelint:disable-file <rule ids> -- <reason>` covers the whole file.
+  # In a directory, either one covers the directory.
+  class Source
+    SUPPRESSION = %r{(?:#|//|--|/\*|<!--|;)\s*judgelint:(disable-file|disable)\b(.*)\z}
+    REASONED = %r{\A\s+([A-Za-z0-9_./, -]+?)\s+--\s+\S}
+
+    attr_reader :path, :shown_as, :line_count
+
+    def self.directory(dir, shown_as: dir, context: [])
+      files = Glob.expand([File.join(dir, "**", "*")])
+      new(dir, shown_as: shown_as, files: files, context: context, directory: true)
+    end
+
+    def initialize(path, shown_as: path, files: [path], context: [], directory: false)
+      @path = path
+      @shown_as = shown_as
+      @directory = directory
+      @files = files.map { |file| [file, File.read(file).lines] }
+      @context = context
+      @line_count = @files.sum { |_file, lines| lines.size }
+      @disabled = {}
+      @disabled_file = []
+      @files.each { |file, lines| find_suppressions(file, lines) }
+    end
+
+    def directory?
+      @directory
+    end
+
+    def state
+      body =
+        if directory?
+          sections = @files.map { |file, lines| "Path: #{shown(file)}\n\n#{numbered(lines)}" }
+          "Directory: #{shown_as}\n\n#{sections.join("\n")}"
+        else
+          "Path: #{shown_as}\n\n#{numbered(@files.first[1])}"
+        end
+      return body if @context.empty?
+
+      references = @context.map { |file| "Path: #{file}\n\n#{File.read(file)}" }
+      "#{body}\nContext, for reference only (not under review):\n\n#{references.join("\n")}"
+    end
+
+    # [first, last] ranges of +size+ lines that cover the file.
+    def windows(size)
+      (1..line_count).step(size).map { |first| [first, [first + size - 1, line_count].min] }
+    end
+
+    def suppressed?(rule_id, first, last)
+      return true if @disabled_file.include?(rule_id)
+
+      lines = @disabled[rule_id] || []
+      directory? ? !lines.empty? : lines.any? { |line| line.between?(first, last) }
+    end
+
+    def suppressed_ids
+      (@disabled.keys + @disabled_file).uniq
+    end
+
+    private
+
+    def numbered(lines)
+      lines.each_with_index.map { |line, index| "#{index + 1}: #{line.chomp}\n" }.join
+    end
+
+    # A file's path inside the directory, as the model sees it.
+    def shown(file)
+      "#{shown_as}/#{file.delete_prefix("#{path}/")}"
+    end
+
+    def find_suppressions(file, lines)
+      lines.each_with_index do |line, index|
+        match = SUPPRESSION.match(line.chomp)
+        next unless match
+
+        reasoned = REASONED.match(match[2])
+        raise ConfigError, "#{file}:#{index + 1}: judgelint:#{match[1]} needs rule ids and a reason after --" unless reasoned
+
+        ids = reasoned[1].split(/[\s,]+/).reject(&:empty?)
+        if match[1] == "disable-file"
+          @disabled_file.concat(ids)
+        else
+          ids.each { |id| (@disabled[id] ||= []) << index + 1 }
+        end
+      end
+    end
+  end
+end

data/lib/judgelint/spec.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # Finds rules where a team already writes its conventions: a README (or any
+  # *.judgelint.md) whose frontmatter says which files it governs, with one
+  # convention per top-level bullet, as praxis reads them.
+  #
+  #   ---
+  #   paths: ["src/services/*.ts"]
+  #   excludes: ["src/services/legacy-import.ts"]
+  #   ---
+  #   # Service conventions
+  #
+  #   - **Domain failures are values, never exceptions.** A service returns
+  #     { ok: false, error } for anything a caller could plausibly cause.
+  #   - **No I/O except through the injected Store.** No timers, no fetch.
+  #
+  # Each bullet becomes a rule, with an id from its lead words, such as
+  # src/services/domain-failures-are-values-never-exceptions. The model sees
+  # the spec's prose alongside the file, and is asked whether the code breaks
+  # the bullet and whether it follows it. Indented `violation:` and
+  # `compliant:` lines under a bullet say what each looks like:
+  #
+  #   - **No I/O except through the injected Store.**
+  #     violation: calls fetch, console, or a timer directly
+  #     compliant: reads and writes only through the Store it's given
+  #
+  # Files in .judgelint/rules/ work the same way, and one there with a
+  # question: in its frontmatter is a single rule asked exactly that way.
+  module Spec
+    PATTERNS = ["**/README.md", "**/*.judgelint.md"].freeze
+    KEYS = %w[paths excludes context scope cohort type severity threshold].freeze
+    COHORTS = { "by_file" => "file", "by_directory" => "directory" }.freeze
+    BULLET = /\A(?:[-*+]|\d+[.)])\s+(\S.*)\z/
+    CRITERION = /\A(violation|compliant):\s*(\S.*)\z/i
+    LEAD = /\A\*\*(.+?)\*\*/
+
+    # +readmes+ false looks in +rules_dir+ alone.
+    def self.discover(rules_dir:, default_threshold:, ignore: [], readmes: true)
+      # path.dup: Spinel can't yet share these Strings into the rules it builds.
+      found = Dir.glob(File.join(rules_dir, "*.md")).sort.flat_map do |path|
+        load(path.dup, default_threshold: default_threshold, required: true)
+      end
+      specs = readmes ? Glob.expand(PATTERNS) : []
+      specs = specs.reject { |path| path.start_with?("#{rules_dir}/") || Glob.match_any?(ignore, path) }
+      found += specs.flat_map { |path| load(path.dup, default_threshold: default_threshold, required: false) }
+      raise ConfigError, "No rules: add a README with paths: frontmatter, or run `judgelint init`" if found.empty?
+
+      found.group_by(&:question_key).each_value do |same|
+        raise ConfigError, "Rule ids collide: #{same.map(&:id).join(', ')}" if same.size > 1
+      end
+      found
+    end
+
+    # A README without frontmatter, or without paths:, isn't a spec.
+    def self.load(path, default_threshold:, required:)
+      match = Rule::FRONTMATTER.match(File.read(path))
+      unless match
+        raise ConfigError, "#{path}: rules start with a --- frontmatter block" if required
+
+        return []
+      end
+
+      data = Judgelint.parse_yaml(match[1], path)
+      body = match[2]
+      return [Rule.new(data, source: path, body: body.strip, default_threshold: default_threshold)] if data.key?("question")
+      return [] unless required || data.key?("paths")
+
+      unknown = data.keys - KEYS
+      raise ConfigError, "#{path}: unknown keys #{unknown.join(', ')}" unless unknown.empty?
+
+      shared = shared_settings(path, data)
+      found = bullets(body)
+      raise ConfigError, "#{path}: a rule needs a question, or bullets to turn into rules" if found.empty? && required
+
+      prose = body.strip
+      ids = []
+      found.map do |text, criteria|
+        id = unique("#{name(path)}/#{slug(text)}", ids)
+        rule = shared.merge("id" => id, "convention" => plain(text), "message" => lead(text))
+        rule["yes_means"] = plain(criteria["violation"]) if criteria["violation"]
+        rule["no_means"] = plain(criteria["compliant"]) if criteria["compliant"]
+        rule["severity"] = severity(text) unless data.key?("severity")
+        Rule.new(rule, source: path, default_threshold: default_threshold, prose: prose,
+                       examples_dir: File.join(".judgelint", "examples", id))
+      end
+    end
+
+    def self.shared_settings(path, data)
+      settings = {}
+      %w[paths excludes context severity threshold].each { |key| settings[key] = data[key] if data.key?(key) }
+      if data.key?("cohort")
+        cohort = data["cohort"].to_s
+        raise ConfigError, "#{path}: cohort is by_file or by_directory, not #{cohort}" unless COHORTS.key?(cohort)
+
+        settings["scope"] = COHORTS.fetch(cohort)
+      end
+      settings["scope"] = data["scope"] if data.key?("scope")
+      settings
+    end
+
+    # The top-level bullets of the body, outside code fences, as [text,
+    # criteria]: the text with its continuation lines and nested bullets, and
+    # any indented violation: or compliant: lines.
+    def self.bullets(body)
+      found = []
+      current = nil
+      criteria = {}
+      criterion = nil
+      fenced = false
+      body.each_line do |raw|
+        line = raw.chomp
+        if line.lstrip.start_with?("```")
+          fenced = !fenced
+          next
+        end
+        next if fenced
+
+        bullet = BULLET.match(line)
+        if bullet
+          found << [current, criteria] if current
+          current = bullet[1]
+          criteria = {}
+          criterion = nil
+        elsif current && line.start_with?(" ", "\t") && !line.strip.empty?
+          nested = BULLET.match(line.strip)
+          text = nested ? nested[1] : line.strip
+          named = CRITERION.match(text)
+          if named
+            criterion = named[1].downcase
+            criteria[criterion] = named[2]
+          elsif criterion
+            criteria[criterion] = "#{criteria[criterion]} #{text}"
+          else
+            current = "#{current} #{text}"
+          end
+        elsif current && !line.strip.empty?
+          found << [current, criteria]
+          current = nil
+        end
+      end
+      found << [current, criteria] if current
+      found
+    end
+
+    # The bullet with its markdown emphasis removed: what the model is asked about.
+    def self.plain(text)
+      text.gsub("**", "").gsub(/\s+/, " ").strip
+    end
+
+    # The bold lead, or the first sentence: what judgelint prints for a finding.
+    def self.lead(text)
+      bold = LEAD.match(text)
+      sentence = bold ? bold[1] : plain(text)
+      sentence = sentence[0, sentence.index(/[.!?:](\s|\z)/) || sentence.size]
+      "#{sentence.strip.sub(/[.!?:]\z/, '')}."
+    end
+
+    def self.slug(text)
+      words = lead(text).downcase.scan(/[a-z0-9]+/).first(6)
+      words.empty? ? "rule" : words.join("-")
+    end
+
+    def self.unique(id, taken)
+      candidate = id
+      number = 1
+      while taken.include?(candidate)
+        number += 1
+        candidate = "#{id}-#{number}"
+      end
+      taken << candidate
+      candidate
+    end
+
+    # Spec names: a README's directory, a *.judgelint.md's stem, or a rules file's name.
+    def self.name(path)
+      dir = File.dirname(path)
+      return(dir == "." ? "root" : dir) if File.basename(path) == "README.md"
+
+      File.basename(path).delete_suffix(".md").delete_suffix(".judgelint")
+    end
+
+    # Praxis's reading: "should" and "prefer" are advice, anything else is a rule.
+    def self.severity(text)
+      advisory = /\b(should|prefer|ideally|consider)\b/i.match?(text)
+      advisory && !/\b(must|never|always)\b/i.match?(text) ? "warning" : "error"
+    end
+  end
+end

data/lib/judgelint/usage.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # Requests and input tokens across a run, and what they cost. Decision
+  # models bill input only, so input tokens are the cost. Shared by every
+  # model's client in a run.
+  class Usage
+    attr_reader :requests, :input_tokens
+
+    def initialize
+      @requests = 0
+      @input_tokens = 0
+      @cost = 0.0
+      @unpriced = false
+      @lock = Mutex.new
+    end
+
+    # OpenRouter reports each request's cost; for the rest it's tokens x the
+    # model's price. A request with neither makes the run's cost unknown.
+    def record(data, price_per_million)
+      usage = data["usage"].is_a?(Hash) ? data["usage"] : {}
+      tokens = usage["input_tokens"].to_i
+      @lock.synchronize do
+        @requests += 1
+        @input_tokens += tokens
+        if usage["cost"].is_a?(Numeric)
+          @cost += usage["cost"].to_f
+        elsif price_per_million
+          @cost += tokens * price_per_million / 1_000_000.0
+        else
+          @unpriced = true
+        end
+      end
+    end
+
+    def cost
+      @unpriced ? nil : @cost
+    end
+
+    def to_h
+      { "requests" => requests, "input_tokens" => input_tokens, "cost_usd" => cost&.round(6) }
+    end
+
+    def line
+      cost_text = cost ? format("$%.4f", cost) : "cost unknown (no price for a model)"
+      "judgelint: #{requests} #{requests == 1 ? 'request' : 'requests'}, #{Usage.delimit(input_tokens)} input tokens, #{cost_text}"
+    end
+
+    def self.delimit(number)
+      number.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
+    end
+  end
+end

data/lib/judgelint/verdict.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # One rule's outcome for one file.
+  #
+  # +status+ is :violation (at or above the rule's threshold), :uncertain
+  # (between warn_at and the threshold), :pass, :skipped (the file is too
+  # large to judge), :refused (the model declined the question), or
+  # :suppressed (a judgelint:disable comment covers it). Skipped and refused
+  # files are reported, never silently passed. +lines+ is the [first, last]
+  # range the verdict came from.
+  #
+  # An example's verdict also knows the answer it should have had: +expected+
+  # is :pass or :fail for files under rules/<id>/pass and rules/<id>/fail.
+  # Not Data.define: Spinel can't compile a Data or Struct in a program that also uses net/https and json.
+  class Verdict
+    attr_reader :rule, :path, :probability, :status, :lines, :expected
+
+    def initialize(rule:, path:, probability:, status:, lines: nil, expected: nil)
+      @rule = rule
+      @path = path
+      @probability = probability
+      @status = status
+      @lines = lines
+      @expected = expected
+    end
+
+    def as_example(expected)
+      Verdict.new(rule: rule, path: path, probability: probability, status: status, lines: lines, expected: expected)
+    end
+
+    # +probability+ is the probability the code breaks the rule.
+    def self.classify(rule, path, probability, warn_at:, lines: nil)
+      status =
+        if probability >= rule.threshold then :violation
+        elsif probability >= warn_at then :uncertain
+        else :pass
+        end
+      new(rule: rule, path: path, probability: probability, status: status, lines: lines)
+    end
+
+    def failing?
+      status == :violation && rule.error?
+    end
+
+    # A refused, skipped, or suppressed example is a miss either way: the rule
+    # didn't work on it.
+    def correct?
+      return false if %i[refused skipped suppressed].include?(status)
+
+      (status == :violation) == (expected == :fail)
+    end
+
+    def to_h
+      {
+        "path" => path, "lines" => lines, "rule" => rule.id, "severity" => rule.severity, "status" => status.to_s,
+        "probability" => probability&.round(3), "message" => rule.message
+      }.merge(expected ? { "expected" => expected.to_s, "correct" => correct? } : {})
+    end
+  end
+end

data/lib/judgelint/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Judgelint
+  VERSION = "0.1.0"
+end

data/lib/judgelint.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "json"
+require "yaml"
+require "digest"
+require "fileutils"
+require "net/https"
+require "open3"
+
+require_relative "judgelint/version"
+require_relative "judgelint/errors"
+require_relative "judgelint/glob"
+require_relative "judgelint/rule"
+require_relative "judgelint/spec"
+require_relative "judgelint/source"
+require_relative "judgelint/question"
+require_relative "judgelint/changes"
+require_relative "judgelint/providers"
+require_relative "judgelint/config"
+require_relative "judgelint/cache"
+require_relative "judgelint/usage"
+require_relative "judgelint/pool"
+require_relative "judgelint/verdict"
+require_relative "judgelint/protocol"
+require_relative "judgelint/client"
+require_relative "judgelint/runner"
+require_relative "judgelint/reporter"
+require_relative "judgelint/scaffold"
+require_relative "judgelint/cli"
+
+# Lints files against conventions that only a reader can check. Each rule is a
+# yes/no question that a System One decision model answers with a probability.
+module Judgelint
+  NUMBER = /\A-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?\z/
+
+  # Float(), without the rescue: answers nil for anything that isn't a number.
+  def self.number(value)
+    string = value.to_s.strip
+    NUMBER.match?(string) ? string.to_f : nil
+  end
+
+  def self.read_yaml(path, required:)
+    return parse_yaml(File.read(path), path) if File.exist?(path)
+    raise ConfigError, "No config file at #{path}" if required
+
+    {}
+  end
+
+  def self.parse_yaml(yaml, path)
+    data = YAML.safe_load(yaml) || {}
+    raise ConfigError, "#{path} must be a mapping" unless data.is_a?(Hash)
+
+    data
+  rescue Psych::SyntaxError => e
+    raise ConfigError, "#{path}: #{e.message}"
+  end
+end

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification
+name: judgelint
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- JP Camara
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: |
+  judgelint checks files against conventions that only a reader can judge. Each
+  rule is a yes/no question answered by a fast decision model such as TypeSafe's
+  Jev over the System One protocol. It also compiles to a native binary with
+  Spinel.
+email:
+- johnpcamara@gmail.com
+executables:
+- judgelint
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- exe/judgelint
+- lib/judgelint.rb
+- lib/judgelint/cache.rb
+- lib/judgelint/changes.rb
+- lib/judgelint/cli.rb
+- lib/judgelint/client.rb
+- lib/judgelint/config.rb
+- lib/judgelint/errors.rb
+- lib/judgelint/glob.rb
+- lib/judgelint/pool.rb
+- lib/judgelint/protocol.rb
+- lib/judgelint/providers.rb
+- lib/judgelint/question.rb
+- lib/judgelint/reporter.rb
+- lib/judgelint/rule.rb
+- lib/judgelint/runner.rb
+- lib/judgelint/scaffold.rb
+- lib/judgelint/source.rb
+- lib/judgelint/spec.rb
+- lib/judgelint/usage.rb
+- lib/judgelint/verdict.rb
+- lib/judgelint/version.rb
+homepage: https://github.com/jpcamara/judgelint
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/jpcamara/judgelint
+  changelog_uri: https://github.com/jpcamara/judgelint/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Lint for conventions a linter can't check, judged by System One decision
+  models.
+test_files: []