remlint 0.1.0

data/lib/remlint/trigger.rb

@@ -0,0 +1,359 @@
+# frozen_string_literal: true
+
+require_relative "expr_lexer"
+require_relative "vocabulary"
+
+module RemLint
+  # The clauses of one command's trigger, and where its body starts.
+  #
+  # Everything before the reminder type -- `MSG`, `MSF`, `RUN`, `CAL`,
+  # `SATISFY`, `SPECIAL`, `PS`, `PSFILE` -- is the trigger. Everything after is
+  # the body, and the body is *text*.
+  #
+  # That boundary is the whole point of this class. `REM Tue AT 15:00 MSG Meet
+  # Bob at the pub` contains the word `at` twice: once as the clause that sets
+  # the time, once as an ordinary English preposition. A rule that grepped the
+  # line for `AT` would find both. Scanning stops at the body keyword, so only
+  # the first one is a clause.
+  #
+  # Hyphenated clause keywords (`MAX-OVERDUE`, `COMPLETE-THROUGH`) lex as three
+  # tokens, so a name followed by a hyphen and another name is retried as one
+  # word before being taken as two.
+  class Trigger
+    Clause = Struct.new(
+      :keyword,
+      :offset,
+      :end_offset,
+      :token_index,
+      :token_width,
+      keyword_init: true,
+    ) do
+      def name
+        keyword.name
+      end
+
+      # Where the clause's argument starts, in tokens. A hyphenated keyword
+      # such as `MAX-OVERDUE` occupies three of them, so this is not always
+      # `token_index + 1`.
+      def argument_index
+        token_index + token_width
+      end
+    end
+
+    # The commands that open a trigger. Everything else -- `ERRMSG`, `SET`,
+    # `BANNER`, `INCLUDE` -- takes its own argument, and scanning that argument
+    # for a reminder type finds words rather than keywords. Remind's own
+    # `tests/tstlang.rem` has
+    #
+    #   errmsg Please run [filename()] with the -q and -r options
+    #
+    # where `run` is English. Treating it as the start of a RUN body makes the
+    # rest of the sentence look like a shell command.
+    TRIGGER_COMMANDS = %w[REM OMIT IFTRIG].freeze
+
+    attr_reader :clauses, :body
+
+    def initialize(tokens, triggered: true)
+      @tokens = tokens
+      @clauses = []
+      @body = nil
+      @triggered = triggered
+
+      if triggered
+        scan
+      end
+    end
+
+    # Whether this command carries a trigger at all. Rules that look for
+    # something positional -- a repeat, a delta -- need it, because `SET a 3*14`
+    # has a `*` in it and no trigger for that `*` to be part of.
+    def triggered?
+      @triggered
+    end
+
+    def self.of(tokens, triggered: true)
+      new(tokens, triggered: triggered)
+    end
+
+    # Whether this command carries a trigger at all: the three commands that
+    # open one, a bare reminder type (`MSG hello`), or a trigger written with
+    # the `REM` left off.
+    def self.triggered?(command)
+      command.implicit? ||
+        command.keyword?(*TRIGGER_COMMANDS) ||
+        !!command.keyword&.body?
+    end
+
+    # The clause of that name, or nil. Names are canonical, so `find("UNTIL")`
+    # matches an abbreviated `UNTIL` written as `UNTI`.
+    def find(name)
+      clauses.find { |clause| clause.name == name.upcase }
+    end
+
+    def include?(*names)
+      wanted = names.map(&:upcase)
+
+      clauses.any? { |clause| wanted.include?(clause.name) }
+    end
+
+    def body?
+      !body.nil?
+    end
+
+    # A body Remind runs the substitution filter over: everything but SATISFY.
+    #
+    # SATISFY's body is an *expression*, and in an expression `%` is the modulo
+    # operator (`expr.c:1473`). `SATISFY [($Ty % 4) == 0]` in Remind's own
+    # `examples/defs.rem` is arithmetic, not a malformed substitution, and a
+    # rule that scanned it for `%` sequences would say so on every leap-year
+    # test ever written.
+    EXPRESSION_BODIES = %w[SATISFY].freeze
+
+    def text_body?
+      body? && !EXPRESSION_BODIES.include?(body.name)
+    end
+
+    # The tokens after the reminder type, which are the body's text. Empty for
+    # a command with no body keyword at all.
+    def body_tokens
+      if body
+        @tokens[(body.token_index + 1)..] || []
+      else
+        []
+      end
+    end
+
+    # Where the body's text begins in the logical line, or nil.
+    def body_offset
+      body&.end_offset
+    end
+
+    private
+
+      def scan
+        index = 0
+
+        while index < @tokens.length && body.nil?
+          index += consume(index)
+        end
+      end
+
+      # Returns how many tokens were taken, so a hyphenated keyword advances
+      # past all three of its tokens rather than rediscovering its own tail.
+      def consume(index)
+        token = @tokens[index]
+
+        if token.type != :name
+          1
+        else
+          classify(index, token)
+        end
+      end
+
+      def classify(index, token)
+        joined = hyphenated(index)
+        keyword = Vocabulary.keyword(joined || token.value)
+        if joined
+          width = 3
+        else
+          width = 1
+        end
+
+        if keyword.nil?
+          1
+        else
+          record(
+            keyword,
+            token,
+            index,
+            width,
+          )
+          width
+        end
+      end
+
+      # `MAX-OVERDUE` arrives as name, `-`, name. Only accepted when the joined
+      # form actually resolves, so an ordinary `a-b` stays two names.
+      def hyphenated(index)
+        hyphen = @tokens[index + 1]
+        tail = @tokens[index + 2]
+
+        if hyphen&.type == :other && hyphen.value == "-" && tail&.type == :name
+          candidate = "#{@tokens[index].value}-#{tail.value}"
+
+          if Vocabulary.keyword(candidate)&.name&.include?("-")
+            candidate
+          end
+        end
+      end
+
+      def record(keyword, token, index, width)
+        last = @tokens[index + width - 1]
+
+        clause = Clause.new(
+          keyword:     keyword,
+          offset:      token.offset,
+          end_offset:  last.end_offset,
+          token_index: index,
+          token_width: width,
+        )
+
+        if keyword.body?
+          @body = clause
+        else
+          @clauses << clause
+        end
+      end
+  end
+end
+
+__END__
+
+require_relative "document"
+
+describe "RemLint::Trigger" do
+  trigger = proc do |text|
+    source = RemLint::Source.new(path: "t.rem", text: text)
+    document = RemLint::Document.new(source)
+    line = document.logical_lines.first
+
+    RemLint::Trigger.of(document.tokens_for(line))
+  end
+
+  names = proc { |text| trigger.(text).clauses.map(&:name) }
+
+  describe "finding clauses" do
+    it "lists the clauses of a trigger, and not the reminder type" do
+      # MSG ends the trigger, so it is the body rather than a clause.
+      names.("REM Tuesday AT 15:00 MSG Staff Meeting").should == %w[REM TUESDAY AT]
+    end
+
+    it "answers include? by canonical name" do
+      trigger.("REM 1 Jan UNTIL 2027-01-01 MSG hi").include?("UNTIL").should.be.true
+      trigger.("REM 1 Jan MSG hi").include?("UNTIL").should.be.false
+    end
+
+    it "matches an abbreviated keyword" do
+      trigger.("REM 1 Jan SCAN -7 MSG hi").include?("SCANFROM").should.be.true
+    end
+
+    it "is case-insensitive" do
+      trigger.("rem tuesday at 15:00 msg hi").include?("AT").should.be.true
+    end
+  end
+
+  describe "the trigger/body boundary" do
+    it "stops scanning at the reminder type" do
+      # The second `at` is English, not a clause.
+      trigger.("REM Tue AT 15:00 MSG Meet Bob at the pub").clauses.count { |c|
+        c.name == "AT"
+      }.should == 1
+    end
+
+    it "does not find a clause keyword that only appears in the body" do
+      trigger.("REM 1 Jan MSG Ask about the duration until Friday").include?("DURATION").should.be.false
+      trigger.("REM 1 Jan MSG Ask about the duration until Friday").include?("UNTIL").should.be.false
+    end
+
+    it "records which reminder type ended the trigger" do
+      trigger.("REM 1 Jan MSG hi").body.name.should == "MSG"
+      trigger.("REM 1 Jan RUN echo hi").body.name.should == "RUN"
+      trigger.("REM 1 Jan SATISFY [1]").body.name.should == "SATISFY"
+    end
+
+    it "reports no body for a command that has none" do
+      trigger.("OMIT 1 January").should.not.be.body
+      trigger.("OMIT 1 January").body_tokens.should.be.empty
+    end
+
+    it "gives the offset where the body's text starts" do
+      command = "REM 1 Jan MSG hello"
+
+      trigger.(command).body_offset.should == command.index("MSG") + 3
+    end
+  end
+
+  describe "hyphenated clause keywords" do
+    it "resolves them as one keyword" do
+      trigger.("REM 1 Jan MAX-OVERDUE 5 MSG hi").include?("MAX-OVERDUE").should.be.true
+    end
+
+    it "reports the argument index past all three tokens" do
+      found = trigger.("REM 1 Jan MAX-OVERDUE 5 MSG hi").find("MAX-OVERDUE")
+
+      found.token_width.should == 3
+      found.argument_index.should == found.token_index + 3
+    end
+
+    it "does not split one into its halves" do
+      names.("REM 1 Jan MAX-OVERDUE 5 MSG hi").should.not.include "MAY"
+    end
+
+    it "leaves a hyphen alone when the joined form is not a keyword" do
+      # MAY is a month; MAY-BE is nothing, so the hyphen must not swallow it.
+      names.("REM MAY-BE 5 MSG hi").should == %w[REM MAY]
+    end
+  end
+
+  describe "commands that carry no trigger" do
+    triggered = proc do |text|
+      source = RemLint::Source.new(path: "t.rem", text: text)
+      document = RemLint::Document.new(source)
+
+      RemLint::Trigger.triggered?(document.code_commands.first)
+    end
+
+    it "counts REM, OMIT and IFTRIG" do
+      triggered.("REM 1 Jan MSG hi").should.be.true
+      triggered.("OMIT 1 Jan MSG hi").should.be.true
+      triggered.("IFTRIG 1 Jan").should.be.true
+    end
+
+    it "counts a bare reminder type and an implicit trigger" do
+      triggered.("MSG hello").should.be.true
+      triggered.("1 Nov ++12 MSG hi").should.be.true
+    end
+
+    it "does not count commands that take their own argument" do
+      triggered.("ERRMSG Please run [filename()] with -q").should.be.false
+      triggered.("SET a 1").should.be.false
+      triggered.("BANNER %").should.be.false
+      triggered.("INCLUDE defs.rem").should.be.false
+    end
+
+    it "finds nothing at all when told the command has no trigger" do
+      source = RemLint::Source.new(path: "t.rem", text: "ERRMSG Please run [x] now")
+      document = RemLint::Document.new(source)
+      line = document.logical_lines.first
+      bare = RemLint::Trigger.of(document.tokens_for(line), triggered: false)
+
+      bare.body.should.be.nil
+      bare.clauses.should.be.empty
+    end
+  end
+
+  describe "text bodies versus expression bodies" do
+    it "counts MSG, RUN and the rest as text" do
+      trigger.("REM 1 Jan MSG hi").should.be.text_body
+      trigger.("REM 1 Jan RUN cmd").should.be.text_body
+      trigger.("REM 1 Jan CAL hi").should.be.text_body
+    end
+
+    it "does not count SATISFY, whose body is an expression" do
+      # `%` there is modulo, not a substitution.
+      trigger.("REM Tue Nov 2 SATISFY [($Ty % 4) == 0]").text_body?.should.be.false
+    end
+
+    it "does not count a command with no body at all" do
+      trigger.("OMIT 1 January").text_body?.should.be.false
+    end
+  end
+
+  it "handles a trigger written with no REM at all" do
+    trigger.("1 Nov ++12 MSG Get ready").body.name.should == "MSG"
+  end
+
+  it "handles a comment without finding clauses in it" do
+    trigger.("# until the AT clause").body.should.be.nil
+  end
+end

data/lib/remlint/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module RemLint
+  VERSION = "0.1.0"
+end

data/lib/remlint/vocabulary.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+require_relative "tables"
+
+module RemLint
+  # Lookups over the generated tables, reproducing Remind's own resolution
+  # rules rather than approximating them.
+  #
+  # The three rules differ from each other, and each difference is a bug class
+  # if you get it wrong:
+  #
+  #   keywords   case-insensitive, abbreviable to a per-keyword minimum length
+  #              (src/token.c FindToken)
+  #   functions  case-insensitive, exact -- no abbreviation
+  #              (src/funcs.c FindBuiltinFunc)
+  #   $SysVars   case-insensitive, exact -- no abbreviation
+  #              (src/var.c FindSysVar)
+  module Vocabulary
+    # The clause keywords -- UNTIL, AT, WARN and so on -- appear inside a
+    # trigger rather than at the head of a line, so a rule looking at the first
+    # word of a command needs to tell the two apart.
+    CLAUSE_TYPES = %w[
+      T_Month T_WkDay T_Ordinal T_Skip T_At T_Duration T_Until T_Warn
+      T_Sched T_Scanfrom T_Through T_Once T_Priority T_Tag T_In T_Tz
+      T_BackAdj T_Info T_NoQueue T_MaxOverdue T_CompleteThrough
+      T_MaybeUncomputable T_OmitFunc
+    ].freeze
+
+    Keyword = Struct.new(:name, :minlen, :type) do
+      def clause?
+        CLAUSE_TYPES.include?(type)
+      end
+
+      # `MSG`, `MSF`, `RUN`, `CAL`, `SATISFY`, `SPECIAL`, `PS`, `PSFILE` --
+      # the token that ends a trigger and starts the reminder body.
+      def body?
+        type == "T_RemType"
+      end
+    end
+
+    Function = Struct.new(:name, :min, :max) do
+      def variadic?
+        max == Tables::NO_MAX_ARGS
+      end
+
+      def accepts?(count)
+        count >= min && (variadic? || count <= max)
+      end
+
+      # "1", "2 to 5", "at least 2" -- the phrasing an arity message needs.
+      def arity_description
+        if variadic?
+          "at least #{min}"
+        elsif min == max
+          min.to_s
+        else
+          "#{min} to #{max}"
+        end
+      end
+    end
+
+    SysVar = Struct.new(
+      :name,
+      :modifiable,
+      :type,
+      :min,
+      :max,
+    ) do
+      def bounded?
+        !min.nil? && !max.nil?
+      end
+
+      def in_range?(value)
+        value >= min && value <= max
+      end
+    end
+
+    KEYWORDS = Tables::KEYWORD_ROWS.map { |row| Keyword.new(*row).freeze }.freeze
+
+    FUNCTIONS = Tables::FUNCTION_ROWS.to_h do |row|
+      function = Function.new(*row).freeze
+
+      [function.name, function]
+    end.freeze
+
+    SYSVARS = Tables::SYSVAR_ROWS.to_h do |row|
+      sysvar = SysVar.new(*row).freeze
+
+      [sysvar.name.downcase, sysvar]
+    end.freeze
+
+    # Resolved lookups, because the linter asks the same questions about the
+    # same words over and over: every `:name` token in every trigger goes
+    # through `keyword`, and a linear walk of 91 entries per token is the
+    # difference between 3.3 and 1.9 seconds per megabyte. Bounded by the
+    # distinct words in the files being linted.
+    #
+    # Guarded like Rule::REGISTRY: `scampi` re-executes each file's top level
+    # to reach its specs, and an unguarded literal would drop the cache
+    # mid-run.
+    unless defined?(KEYWORD_CACHE)
+      KEYWORD_CACHE = {}
+    end
+
+    module_function
+
+    # Resolve one whitespace-delimited word the way FindToken does: the first
+    # entry in table order whose name the word prefixes, provided the word is
+    # at least that entry's minimum abbreviation length. A trailing comma is
+    # ignored, as Remind's TokStrCmp ignores it.
+    def keyword(word)
+      needle = word.to_s.downcase.delete_suffix(",")
+
+      if needle.empty?
+        nil
+      else
+        KEYWORD_CACHE.fetch(needle) { KEYWORD_CACHE[needle] = resolve(needle) }
+      end
+    end
+
+    def resolve(needle)
+      KEYWORDS.find do |candidate|
+        needle.length >= candidate.minlen && candidate.name.downcase.start_with?(needle)
+      end
+    end
+
+    def function(name)
+      FUNCTIONS[name.to_s.downcase]
+    end
+
+    def sysvar(name)
+      SYSVARS[name.to_s.delete_prefix("$").downcase]
+    end
+  end
+end
+
+__END__
+
+describe "RemLint::Vocabulary.keyword" do
+  it "resolves a full keyword regardless of case" do
+    RemLint::Vocabulary.keyword("MSG").name.should == "MSG"
+    RemLint::Vocabulary.keyword("msg").name.should == "MSG"
+    RemLint::Vocabulary.keyword("MsG").name.should == "MSG"
+  end
+
+  it "accepts an abbreviation at or past the minimum length" do
+    # `include` has MinLen 3, which is what makes INC an INCLUDE.
+    RemLint::Vocabulary.keyword("inc").name.should == "INCLUDE"
+    RemLint::Vocabulary.keyword("inclu").name.should == "INCLUDE"
+  end
+
+  it "rejects an abbreviation shorter than the minimum length" do
+    # `omit` has MinLen 4, so `omi` is not a keyword at all.
+    RemLint::Vocabulary.keyword("omi").should.be.nil
+  end
+
+  it "resolves the hyphenated context keywords" do
+    RemLint::Vocabulary.keyword("PUSH-OMIT-CONTEXT").name.should == "PUSH-OMIT-CONTEXT"
+    RemLint::Vocabulary.keyword("PUSH").name.should == "PUSH-OMIT-CONTEXT"
+    RemLint::Vocabulary.keyword("POP").name.should == "POP-OMIT-CONTEXT"
+  end
+
+  it "ignores a trailing comma, as TokStrCmp does" do
+    RemLint::Vocabulary.keyword("mon,").name.should == "MONDAY"
+  end
+
+  it "caches a resolved word without changing the answer" do
+    RemLint::Vocabulary.keyword("msg").name.should == "MSG"
+    RemLint::Vocabulary.keyword("msg").name.should == "MSG"
+    RemLint::Vocabulary.keyword("frobnicate").should.be.nil
+    RemLint::Vocabulary.keyword("frobnicate").should.be.nil
+  end
+
+  it "returns nil for a word that is not a keyword" do
+    RemLint::Vocabulary.keyword("frobnicate").should.be.nil
+    RemLint::Vocabulary.keyword("").should.be.nil
+  end
+
+  it "distinguishes clause keywords from body keywords" do
+    RemLint::Vocabulary.keyword("UNTIL").should.be.clause
+    RemLint::Vocabulary.keyword("MSG").should.be.body
+    RemLint::Vocabulary.keyword("MSG").clause?.should.be.false
+  end
+end
+
+describe "RemLint::Vocabulary.function" do
+  it "looks up builtins case-insensitively and exactly" do
+    RemLint::Vocabulary.function("trigger").name.should == "trigger"
+    RemLint::Vocabulary.function("TRIGGER").name.should == "trigger"
+    RemLint::Vocabulary.function("trigg").should.be.nil
+  end
+
+  it "reports arity from Remind's own table" do
+    RemLint::Vocabulary.function("abs").accepts?(1).should.be.true
+    RemLint::Vocabulary.function("abs").accepts?(2).should.be.false
+    RemLint::Vocabulary.function("abs").arity_description.should == "1"
+  end
+
+  it "reports a range for optional arguments" do
+    ampm = RemLint::Vocabulary.function("ampm")
+    ampm.accepts?(1).should.be.true
+    ampm.accepts?(4).should.be.true
+    ampm.accepts?(5).should.be.false
+    ampm.arity_description.should == "1 to 4"
+  end
+
+  it "treats NO_MAX_ARGS as variadic" do
+    char = RemLint::Vocabulary.function("char")
+    char.should.be.variadic
+    char.accepts?(99).should.be.true
+    char.accepts?(0).should.be.false
+    char.arity_description.should == "at least 1"
+  end
+end
+
+describe "RemLint::Vocabulary.sysvar" do
+  it "looks up system variables case-insensitively, with or without the sigil" do
+    RemLint::Vocabulary.sysvar("$FormWidth").name.should == "FormWidth"
+    RemLint::Vocabulary.sysvar("formwidth").name.should == "FormWidth"
+    RemLint::Vocabulary.sysvar("$NoSuchVariable").should.be.nil
+  end
+
+  it "knows which variables reject SET" do
+    RemLint::Vocabulary.sysvar("$CalMode").modifiable.should.be.false
+    RemLint::Vocabulary.sysvar("$FormWidth").modifiable.should.be.true
+  end
+
+  it "carries the range only for the integer variables Remind bounds" do
+    form_width = RemLint::Vocabulary.sysvar("$FormWidth")
+    form_width.should.be.bounded
+    form_width.in_range?(20).should.be.true
+    form_width.in_range?(19).should.be.false
+
+    RemLint::Vocabulary.sysvar("$Ago").bounded?.should.be.false
+  end
+end

data/lib/remlint.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative "remlint/version"
+require_relative "remlint/cli"
+require_relative "remlint/config"
+require_relative "remlint/document"
+require_relative "remlint/extractors"
+require_relative "remlint/formatter"
+require_relative "remlint/rules"
+require_relative "remlint/runner"
+
+# A style and consistency linter for Remind reminder files.
+#
+# Remind is not Ruby, so none of RuboCop's machinery applies: there is no
+# `parser` AST to walk, and its one escape hatch for non-Ruby input exists to
+# pull Ruby back out of templates. The precedent that does apply is puppet-lint
+# -- a linter written in Ruby for a language that is not Ruby -- including
+# where it draws its line: it checks style and structure, and leaves "is this
+# valid" to the real interpreter.
+#
+# Remind's grammar decides the architecture. The manual describes a file as a
+# list of commands, one per line, with backslash continuation; comments open
+# with `#` or `;`; keywords are case-insensitive and abbreviable; and the `REM`
+# that opens a trigger may be left off entirely. That is line-oriented, not
+# tree-oriented, so there is no grammar here -- there is a pipeline:
+#
+#   bytes -> sources -> logical lines -> commands -> tokens -> rules -> offences
+#
+# Each stage is in its own file and each is useful alone. What the vocabulary
+# those stages match against is -- every keyword, its minimum abbreviation, every
+# function's arity, every system variable's range -- is generated from Remind's
+# own dispatch tables by `tasks/generate_tables.rb`, so it is transcription
+# rather than recollection.
+module RemLint
+end