remlint 0.1.0

data/lib/remlint/expr_lexer.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+module RemLint
+  # One lexeme of a command's body, with where it sat.
+  #
+  # `offset` is a character offset into the logical line's text, which
+  # {LogicalLine#position_at} turns into a file line and column. Tokens carry
+  # the offset rather than a line number because a continued command's tokens
+  # are spread over several lines and only the offset survives the join.
+  Token = Struct.new(
+    :type,
+    :value,
+    :offset,
+    keyword_init: true,
+  ) do
+    def length
+      value.length
+    end
+
+    def end_offset
+      offset + length
+    end
+  end
+
+  # Tokenises the inside of a Remind command.
+  #
+  # Not a parser. Remind's expression grammar is real, but a linter needs far
+  # less than a grammar and gets it much more cheaply: what it needs to know is
+  # where the string literals are (so it does not count a `[` inside one), where
+  # the bracketed expressions are, and where the function calls are. Everything
+  # else can stay an undifferentiated run of characters.
+  #
+  # Strings matter most. `MSG see [ansicolor("")]` has balanced brackets;
+  # `MSG a "]" b` does not contain a closing bracket at all. Scanning
+  # characters without tracking quotes gets both wrong.
+  class ExprLexer
+    # Remind takes both quote characters, with backslash escaping inside.
+    DOUBLE_QUOTED = /"(?:[^"\\]|\\.)*"?/
+    SINGLE_QUOTED = /'(?:[^'\\]|\\.)*'?/
+
+    # `%a`, `%_`, `%"` -- the substitution sequences, one character after the
+    # percent. Matched before anything else can claim the character, which is
+    # what remind.vim's `remindSubst` pattern does.
+    SUBSTITUTION = /%[^\s]/
+
+    # `$SysInclude`, `$Latitude`.
+    SYSVAR = /\$[A-Za-z_]\w*/
+
+    # A name immediately followed by `(` is a call; a name not followed by one
+    # is a variable read, and the two need different rules applied to them.
+    FUNCTION = /[A-Za-z_]\w*(?=\()/
+    NAME     = /[A-Za-z_]\w*/
+
+    NUMBER = /\d+(?:\.\d+)?/
+
+    WHITESPACE = /\s+/
+
+    def initialize(text)
+      @scanner = StringScanner.new(text)
+    end
+
+    def self.tokenise(text)
+      new(text).tokenise
+    end
+
+    def tokenise
+      [].tap do |tokens|
+        until @scanner.eos?
+          tokens << next_token
+        end
+      end
+    end
+
+    # Just the tokens that carry meaning -- everything but whitespace.
+    def self.significant(text)
+      tokenise(text).reject { |token| token.type == :whitespace }
+    end
+
+    private
+
+      # Ordered by specificity, not by frequency: a `%` that opens a
+      # substitution must not be read as an operator, and a name before `(`
+      # must not be read as a bare name.
+      def next_token
+        emit(:string,       DOUBLE_QUOTED) ||
+          emit(:string,     SINGLE_QUOTED) ||
+          emit(:whitespace, WHITESPACE) ||
+          emit(:lbracket,   /\[/) ||
+          emit(:rbracket,   /\]/) ||
+          emit(:lparen,     /\(/) ||
+          emit(:rparen,     /\)/) ||
+          emit(:comma,      /,/) ||
+          emit(:substitution, SUBSTITUTION) ||
+          emit(:sysvar,     SYSVAR) ||
+          emit(:function,   FUNCTION) ||
+          emit(:name,       NAME) ||
+          emit(:number,     NUMBER) ||
+          single_character
+      end
+
+      def emit(type, pattern)
+        offset = @scanner.pos
+        value = @scanner.scan(pattern)
+
+        if value
+          Token.new(type: type, value: value, offset: offset)
+        end
+      end
+
+      # Operators, punctuation and anything else: one character, unclassified.
+      # A rule that cares can look at the value.
+      def single_character
+        offset = @scanner.pos
+
+        Token.new(type: :other, value: @scanner.getch, offset: offset)
+      end
+  end
+end
+
+__END__
+
+describe "RemLint::ExprLexer" do
+  lex = proc { |text| RemLint::ExprLexer.significant(text) }
+  types = proc { |text| lex.(text).map(&:type) }
+  values = proc { |text| lex.(text).map(&:value) }
+
+  it "keeps whitespace in the full stream and drops it from the significant one" do
+    RemLint::ExprLexer.tokenise("a b").map(&:type).should == %i[name whitespace name]
+    types.("a b").should == %i[name name]
+  end
+
+  it "records the offset of every token" do
+    tokens = lex.("MSG [x]")
+
+    tokens.map(&:offset).should == [0, 4, 5, 6]
+    tokens.last.end_offset.should == 7
+  end
+
+  describe "strings" do
+    it "takes a double-quoted string whole" do
+      values.(%(a "hello world" b)).should == ["a", %("hello world"), "b"]
+    end
+
+    it "takes a single-quoted string whole, as Remind's date literals need" do
+      values.("x '2026-01-01' y").should == ["x", "'2026-01-01'", "y"]
+    end
+
+    it "honours backslash escapes inside a string" do
+      values.(%("a \\" b" c)).should == [%("a \\" b"), "c"]
+    end
+
+    it "does not let a bracket inside a string count as a bracket" do
+      types.(%(MSG "]" done)).should == %i[name string name]
+    end
+
+    it "runs an unterminated string to end of line rather than looping" do
+      values.(%(MSG "oops)).should == ["MSG", %("oops)]
+    end
+  end
+
+  describe "Remind's sigils" do
+    it "recognises system variables" do
+      types.("INCLUDE [$SysInclude]/ansitext.rem").should ==
+        %i[name lbracket sysvar rbracket other name other name]
+    end
+
+    it "recognises a substitution as the percent plus one character" do
+      values.("MSG at %2 on %_").should == ["MSG", "at", "%2", "on", "%_"]
+    end
+
+    it "does not read a percent before a space as a substitution" do
+      types.("BANNER % ").should == %i[name other]
+    end
+  end
+
+  describe "calls" do
+    it "distinguishes a call from a bare name" do
+      types.("trigger(x)").should == %i[function lparen name rparen]
+      types.("trigger").should == %i[name]
+    end
+
+    it "lexes a nested call" do
+      values.("[ansicolor(0,255,0) + center(x)]").should ==
+        ["[", "ansicolor", "(", "0", ",", "255", ",", "0", ")", "+", "center", "(", "x", ")", "]"]
+    end
+  end
+
+  it "gives every other character its own token rather than dropping it" do
+    types.("a >= b").should == %i[name other other name]
+  end
+
+  it "lexes across the newlines a joined command carries" do
+    types.("IF a && \n   b").should == %i[name name other other name]
+  end
+end

data/lib/remlint/extractors.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+require_relative "source"
+
+module RemLint
+  # Turns a file on disk into zero or more {Source} runs of Remind code.
+  #
+  # A linter that only globs `*.rem` misses most real Remind code: the shipped
+  # `examples/ansitext` and `examples/astro` are shell scripts that pipe Remind
+  # in through heredocs, and `astro` alone holds four of them. Linting such a
+  # file as if it were Remind produces nothing but noise about the shell.
+  #
+  # So extraction is its own layer, mirroring how RuboCop pulls Ruby out of ERB
+  # and Haml: each extractor either recognises the file and returns its Remind
+  # runs, or returns nil and lets the next one look.
+  module Extractors
+    # `remind ... <<'EOF'` / `<<-EOF` / `<< "EOF"`, with the flags and
+    # redirections in between ignored. The quoting around the delimiter is what
+    # stops the shell interpolating `$Latitude` before Remind sees it; both
+    # quoted and bare forms open a heredoc, so both are matched.
+    HEREDOC_OPENER = /\bremind\b[^\n<]*<<(?<dash>-?)\s*(?<quote>['"]?)(?<delimiter>\w+)\k<quote>/
+
+    # A file Remind is meant to read directly, either by extension or because
+    # it runs itself through Remind.
+    REM_SHEBANG = /\A#![^\n]*\bremind\b/
+
+    module_function
+
+    # A plain Remind file: the whole text, no offset.
+    def rem_file(path, text)
+      if File.extname(path).casecmp?(".rem") || text.match?(REM_SHEBANG)
+        [Source.new(path: path, text: text)]
+      end
+    end
+
+    # Remind embedded in shell heredocs. Returns nil rather than an empty array
+    # when the file holds none, so the extractor chain keeps looking.
+    def shell_heredoc(path, text)
+      sources = scan_heredocs(path, text)
+
+      unless sources.empty?
+        sources
+      end
+    end
+
+    # Ordered most-specific-first: a `.rem` file is taken whole, and only a file
+    # that is not itself Remind gets searched for embedded heredocs.
+    ALL = [method(:rem_file), method(:shell_heredoc)].freeze
+
+    # The first extractor that recognises the file wins.
+    def extract(path, text)
+      ALL.lazy.filter_map { |extractor| extractor.call(path, text) }.first || []
+    end
+
+    def scan_heredocs(path, text)
+      lines = text.lines
+      sources = []
+      index = 0
+
+      while index < lines.length
+        match = lines[index].match(HEREDOC_OPENER)
+
+        if match
+          body_start = index + 1
+          body_end = find_terminator(
+            lines,
+            body_start,
+            match[:delimiter],
+            match[:dash] == "-",
+          )
+
+          sources << Source.new(
+            path:        path,
+            text:        lines[body_start...body_end].join,
+            line_offset: body_start,
+            description: "heredoc at line #{index + 1}",
+          )
+
+          index = body_end
+        end
+
+        index += 1
+      end
+
+      sources
+    end
+
+    # `<<-` lets the closing delimiter be indented; plain `<<` demands it sit
+    # at column zero. An unterminated heredoc runs to end of file, which is
+    # what the shell would do too.
+    def find_terminator(lines, from, delimiter, allow_indent)
+      cursor = from
+
+      while cursor < lines.length && !terminator?(lines[cursor], delimiter, allow_indent)
+        cursor += 1
+      end
+
+      cursor
+    end
+
+    def terminator?(line, delimiter, allow_indent)
+      body = line.chomp
+
+      if allow_indent
+        body.sub(/\A\t+/, "") == delimiter
+      else
+        body == delimiter
+      end
+    end
+  end
+end
+
+__END__
+
+describe "RemLint::Extractors" do
+  extract = proc { |path, text| RemLint::Extractors.extract(path, text) }
+
+  describe "plain Remind files" do
+    it "takes a .rem file whole, with no offset" do
+      sources = extract.("holidays.rem", "REM 1 Jan MSG New Year\n")
+
+      sources.length.should == 1
+      sources.first.text.should == "REM 1 Jan MSG New Year\n"
+      sources.first.line_offset.should == 0
+    end
+
+    it "recognises the extension regardless of case" do
+      extract.("Holidays.REM", "MSG hi\n").length.should == 1
+    end
+
+    it "recognises a Remind shebang on a file with no extension" do
+      sources = extract.("alignment", "#!/usr/bin/env -S remind -@2\nMSG hi\n")
+
+      sources.length.should == 1
+      sources.first.line_offset.should == 0
+    end
+  end
+
+  describe "shell heredocs" do
+    script = <<~SHELL
+      #!/bin/sh
+      # a comment
+      remind -@2 - <<'EOF'
+      BANNER %
+      MSG hello
+      EOF
+      exit 0
+    SHELL
+
+    it "extracts the heredoc body and nothing else" do
+      sources = extract.("ansitext", script)
+
+      sources.length.should == 1
+      sources.first.text.should == "BANNER %\nMSG hello\n"
+    end
+
+    it "offsets by the number of lines before the body" do
+      # The body starts on physical line 4, so the offset is the 3 lines above.
+      extract.("ansitext", script).first.line_offset.should == 3
+    end
+
+    it "finds every heredoc in a file that has several" do
+      four = <<~SHELL
+        #!/bin/sh
+        remind -q -@2 - "$@" <<'EOF'
+        MSG one
+        EOF
+        echo between
+        remind -q -@2 - "$@" <<'EOF'
+        MSG two
+        EOF
+      SHELL
+
+      sources = extract.("astro", four)
+      sources.length.should == 2
+      sources.map(&:text).should == ["MSG one\n", "MSG two\n"]
+      sources.map(&:line_offset).should == [2, 6]
+    end
+
+    it "handles an unquoted delimiter" do
+      extract.("s", "#!/bin/sh\nremind - <<EOF\nMSG hi\nEOF\n").first.text.should == "MSG hi\n"
+    end
+
+    it "allows an indented terminator only for the <<- form" do
+      dash = "#!/bin/sh\nremind - <<-EOF\n\tMSG hi\n\tEOF\nafter\n"
+      extract.("s", dash).first.text.should == "\tMSG hi\n"
+
+      plain = "#!/bin/sh\nremind - <<EOF\nMSG hi\n\tEOF\n"
+      extract.("s", plain).first.text.should == "MSG hi\n\tEOF\n"
+    end
+
+    it "runs an unterminated heredoc to end of file" do
+      extract.("s", "#!/bin/sh\nremind - <<'EOF'\nMSG hi\n").first.text.should == "MSG hi\n"
+    end
+
+    it "describes which heredoc a source came from" do
+      extract.("astro", script).first.label.should == "astro (heredoc at line 3)"
+    end
+  end
+
+  describe "files that are not Remind at all" do
+    it "yields nothing" do
+      extract.("build.sh", "#!/bin/sh\necho hello\n").should.be.empty
+    end
+
+    it "does not mistake a mention of remind in a comment for a heredoc" do
+      extract.("notes.txt", "# see remind for details\n").should.be.empty
+    end
+  end
+end

data/lib/remlint/formatter.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require_relative "offense"
+
+module RemLint
+  # Renders offences for a terminal, an editor, or a CI annotation.
+  #
+  # The default line is `path:line:column: severity: [Rule] message`, which
+  # vim's default `errorformat` already parses and which most CI annotators
+  # recognise. `--log-format` overrides it with the same `%{...}` placeholders
+  # puppet-lint uses, so a project that needs GitHub's `::error file=...`
+  # shape can have it without a new formatter class.
+  class Formatter
+    DEFAULT_TEMPLATE = "%{path}:%{line}:%{column}: %{severity}: [%{rule}] %{message}"
+
+    attr_reader :template
+
+    def initialize(template: DEFAULT_TEMPLATE)
+      @template = template
+    end
+
+    def lines(offenses)
+      offenses.map { |offense| offense.format_with(template) }
+    end
+
+    # One line per offence, then a count. Empty output for a clean run: a
+    # linter that prints "0 offences" on every commit trains people to stop
+    # reading its output.
+    def render(offenses)
+      if offenses.empty?
+        ""
+      else
+        "#{(lines(offenses) + [summary(offenses)]).join("\n")}\n"
+      end
+    end
+
+    def summary(offenses)
+      counts = SEVERITIES.filter_map do |severity|
+        count = offenses.count { |offense| offense.severity == severity }
+
+        if count.positive?
+          "#{count} #{severity}#{count == 1 ? '' : 's'}"
+        end
+      end
+
+      "#{offenses.length} offence#{offenses.length == 1 ? '' : 's'} (#{counts.join(', ')})"
+    end
+  end
+end
+
+__END__
+
+describe "RemLint::Formatter" do
+  offense = proc do |overrides = {}|
+    defaults = {
+      path:    "holidays.rem",
+      line:    12,
+      column:  4,
+      rule:    "TrailingWhitespace",
+      message: "Trailing whitespace",
+    }
+
+    RemLint::Offense.new(**defaults.merge(overrides))
+  end
+
+  it "renders the default line format" do
+    RemLint::Formatter.new.lines([offense.()]).should ==
+      ["holidays.rem:12:4: warning: [TrailingWhitespace] Trailing whitespace"]
+  end
+
+  it "renders through a caller-supplied template" do
+    formatter = RemLint::Formatter.new(template: "::error file=%{path},line=%{line}::%{message}")
+
+    formatter.lines([offense.()]).should == ["::error file=holidays.rem,line=12::Trailing whitespace"]
+  end
+
+  it "prints nothing at all for a clean run" do
+    RemLint::Formatter.new.render([]).should == ""
+  end
+
+  it "ends the report with a count by severity" do
+    report = RemLint::Formatter.new.render([offense.(), offense.(severity: "error")])
+
+    report.should.match(/2 offences \(1 error, 1 warning\)\n\z/)
+  end
+
+  it "uses the singular for one of anything" do
+    RemLint::Formatter.new.summary([offense.()]).should == "1 offence (1 warning)"
+  end
+
+  it "lists only the severities that occurred, worst first" do
+    offenses = [offense.(severity: "info"), offense.(severity: "error")]
+
+    RemLint::Formatter.new.summary(offenses).should == "2 offences (1 error, 1 info)"
+  end
+end

data/lib/remlint/invocation.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module RemLint
+  # How a file is meant to be run, declared in the file.
+  #
+  # Three of the checks a Remind file wants need something that is not in the
+  # file: the command line. `-g` decides whether a sort happens, `-p` versus
+  # `-pp` decides whether `INFO` headers reach the back-end at all, and
+  # calendar mode versus agenda mode decides whether `TODO` means anything.
+  #
+  # Rather than guess three times, the file says so once:
+  #
+  #     # remlint:invocation remind -pp -g /path/to/file
+  #
+  # No declaration means no opinion, and the rules that depend on it stay
+  # silent. That is the honest default: a file with no declaration is one whose
+  # invocation the linter genuinely does not know.
+  class Invocation
+    DIRECTIVE = /[#;]\s*remlint:invocation\s+(?<command>.+?)\s*\z/
+
+    # `-p`, `-pp`, `-s`, `-c` and the `a`/`+`/digit suffixes each may carry.
+    CALENDAR = /\A-(?<kind>[psc])(?<rest>[a-z+0-9]*)\z/
+
+    SORT = /\A-g(?<spec>[a-z]*)\z/
+
+    attr_reader :arguments
+
+    def initialize(arguments)
+      @arguments = arguments
+    end
+
+    # Reads the first declaration in a document, or an empty one.
+    def self.of(document)
+      declaration = document.raw_lines.filter_map { |raw| raw.match(DIRECTIVE) }.first
+
+      new(declaration ? declaration[:command].split : [])
+    end
+
+    def declared?
+      !arguments.empty?
+    end
+
+    # Any of the calendar-producing modes. Everything else is agenda mode,
+    # which is where TODO semantics live.
+    def calendar?
+      arguments.any? { |argument| argument.match?(CALENDAR) }
+    end
+
+    def agenda?
+      declared? && !calendar?
+    end
+
+    # `-pp` and above carry INFO headers to the back-end; plain `-p` does not.
+    def carries_info?
+      arguments.any? { |argument| argument.match?(/\A-p{2,}/) }
+    end
+
+    def simple_calendar?
+      arguments.any? { |argument| argument.match?(/\A-p/) }
+    end
+
+    # The `-g` spec, or nil. Remind takes up to four characters, each `a` or
+    # `d`, for date, time, priority and timedness.
+    def sort_spec
+      match = arguments.filter_map { |argument| argument.match(SORT) }.first
+
+      match && match[:spec]
+    end
+  end
+end
+
+__END__
+
+require_relative "document"
+
+describe "RemLint::Invocation" do
+  of = proc do |text|
+    source = RemLint::Source.new(path: "t.rem", text: text)
+
+    RemLint::Invocation.of(RemLint::Document.new(source))
+  end
+
+  it "is undeclared for a file that says nothing" do
+    of.("MSG hi\n").should.not.be.declared
+  end
+
+  it "reads a declaration from a comment" do
+    of.("# remlint:invocation remind -pp -g\nMSG hi\n").should.be.declared
+  end
+
+  it "reads one written with a semicolon" do
+    of.("; remlint:invocation remind -pp\nMSG hi\n").should.be.declared
+  end
+
+  it "takes the first declaration when there are two" do
+    invocation = of.("# remlint:invocation remind -p\n# remlint:invocation remind -c\nMSG hi\n")
+
+    invocation.simple_calendar?.should.be.true
+  end
+
+  describe "calendar and agenda mode" do
+    it "reads -c as a calendar" do
+      of.("# remlint:invocation remind -c3 file\n").should.be.calendar
+      of.("# remlint:invocation remind -c3 file\n").agenda?.should.be.false
+    end
+
+    it "reads -p and -s as calendars" do
+      of.("# remlint:invocation remind -p file\n").should.be.calendar
+      of.("# remlint:invocation remind -sa file\n").should.be.calendar
+    end
+
+    it "reads anything else as agenda mode" do
+      of.("# remlint:invocation remind -q file\n").should.be.agenda
+    end
+
+    it "has no opinion when nothing is declared" do
+      of.("MSG hi\n").agenda?.should.be.false
+      of.("MSG hi\n").calendar?.should.be.false
+    end
+  end
+
+  describe "INFO headers" do
+    it "knows -pp carries them" do
+      of.("# remlint:invocation remind -pp file\n").should.be.carries_info
+    end
+
+    it "knows plain -p does not" do
+      of.("# remlint:invocation remind -p file\n").carries_info?.should.be.false
+    end
+  end
+
+  describe "the sort spec" do
+    it "reads the characters after -g" do
+      of.("# remlint:invocation remind -gaad file\n").sort_spec.should == "aad"
+    end
+
+    it "reads a bare -g" do
+      of.("# remlint:invocation remind -g file\n").sort_spec.should == ""
+    end
+
+    it "is nil when there is no -g" do
+      of.("# remlint:invocation remind -q file\n").sort_spec.should.be.nil
+    end
+  end
+end