simple_english 0.1.0

data/lib/simple_english/install.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# The resolved LanguageTool installation: where the jars live and which
+# java runs them. Build it once from the environment at the process
+# edge with Install.from_env. Everything downstream takes the value,
+# never the environment. A nil java means "not found": ask with java?
+# or fail with java! and get the fix in the message.
+
+require_relative "languagetool"
+
+module SimpleEnglish
+  class Install
+    class SetupError < StandardError
+    end
+
+    attr_reader :cache_dir, :java
+
+    def initialize(cache_dir:, java: nil)
+      @cache_dir = cache_dir
+      @java = java
+    end
+
+    # The one place the environment is read. SE_JAVA wins outright,
+    # because the user asserted it and nobody re-probes an override.
+    # Otherwise the code probes the PATH candidate by running it,
+    # because a file can exist and still be the macOS stub that
+    # reports no runtime. The last resort is the Homebrew location,
+    # which sits outside PATH.
+    def self.from_env(env = ENV)
+      cache = File.expand_path(env.fetch(LanguageTool::CACHE_DIR_ENV) do
+        File.join(Dir.home, ".cache", "se")
+      end)
+      java = env[LanguageTool::JAVA_ENV] ||
+        ("java" if probe?("java", env)) ||
+        (LanguageTool::HOMEBREW_JAVA if File.executable?(LanguageTool::HOMEBREW_JAVA))
+      new(cache_dir: cache, java: java)
+    end
+
+    # Runs the candidate under the given PATH, isolated from this
+    # process's own PATH so tests can probe a fake environment.
+    def self.probe?(candidate, env)
+      system({"PATH" => env.fetch("PATH", "")}, candidate, "-version",
+        out: File::NULL, err: File::NULL)
+    end
+    private_class_method :probe?
+
+    def java?
+      !java.nil?
+    end
+
+    def java!
+      return java if java?
+      raise SetupError,
+        "java not found. Install a JRE (on macOS: brew install openjdk), " \
+        "or set SE_JAVA to your java binary."
+    end
+
+    def lt_dir
+      File.join(cache_dir, "LanguageTool-#{LanguageTool::LT_VERSION}")
+    end
+
+    def commandline_jar
+      File.join(lt_dir, "languagetool-commandline.jar")
+    end
+
+    def server_jar
+      File.join(lt_dir, "languagetool-server.jar")
+    end
+
+    def setup_error
+      "LanguageTool #{LanguageTool::LT_VERSION} not found at #{server_jar}. " \
+        "This gem pins that version. Run `se setup`."
+    end
+  end
+end

data/lib/simple_english/languagetool.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+# The pinned LanguageTool distro: its version, its rules file, and the
+# downloader. Everything that resolves "where it is on this machine and
+# which java runs it" lives in Install. The engine runs as the daemon's
+# inner HTTP server (lib/simple_english/server.rb).
+
+module SimpleEnglish
+  module LanguageTool
+    RULES_FILE = File.expand_path("../../rules/simple-english.xml", __dir__)
+    # The only place a LanguageTool version number appears. `se
+    # setup` downloads this version, and every jar path derives from it. No
+    # env var can point at another one.
+    LT_VERSION = "6.6"
+    CACHE_DIR_ENV = "SE_CACHE_DIR"
+    JAVA_ENV = "SE_JAVA"
+    # macOS ships a /usr/bin/java stub that reports no runtime. Homebrew
+    # installs the real one here, outside PATH.
+    HOMEBREW_JAVA = "/opt/homebrew/opt/openjdk/bin/java"
+    TIMEOUT_SECONDS = 300
+
+    module_function
+
+    # Download and unpack the pinned LanguageTool into DIR. Idempotent:
+    # returns the install directory, or nil after warning why. Pure Ruby
+    # throughout: the download uses Net::HTTP and the unpack uses rubyzip,
+    # so setup needs no curl or unzip on the machine.
+    def install(dir)
+      require "fileutils"
+      dest = File.join(dir, "LanguageTool-#{LT_VERSION}")
+      if File.exist?(File.join(dest, "languagetool-commandline.jar"))
+        remove_stale_versions(dir, dest)
+        return dest
+      end
+      zip = File.join(dir, "LanguageTool-#{LT_VERSION}.zip")
+      FileUtils.mkdir_p(dir)
+      return nil unless download(download_url, zip)
+      return nil unless extract(zip, dir)
+      File.delete(zip)
+      remove_stale_versions(dir, dest)
+      dest
+    end
+
+    def download_url
+      "https://languagetool.org/download/LanguageTool-#{LT_VERSION}.zip"
+    end
+
+    # Streams URL to TO and follows redirects. Warns and returns false
+    # on any transport or HTTP failure, so callers never see a stack
+    # trace.
+    def download(url, to, redirects: 3)
+      require "net/http"
+      uri = URI(url)
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+        open_timeout: 10, read_timeout: 60) do |http|
+        http.request(Net::HTTP::Get.new(uri)) do |response|
+          case response
+          when Net::HTTPRedirection
+            return false if redirects.zero?
+            return download(response["location"], to, redirects: redirects - 1)
+          when Net::HTTPSuccess
+            File.binwrite(to, "")
+            File.open(to, "wb") do |file|
+              response.read_body { |chunk| file.write(chunk) }
+            end
+            return true
+          else
+            warn "error: download failed: HTTP #{response.code} from #{url}"
+            return false
+          end
+        end
+      end
+    rescue SystemCallError, SocketError, Timeout::Error,
+      OpenSSL::SSL::SSLError, Net::ProtocolError => e
+      warn "error: download failed: #{e.class}: #{e.message}"
+      false
+    end
+
+    # Unpacks ZIP into DIR and refuses entries that try to escape it.
+    # Warns and returns false on any failure.
+    def extract(zip, dir)
+      require "zip"
+      Zip::File.open(zip) do |archive|
+        archive.each do |entry|
+          unless safe_entry_target(dir, entry.name)
+            warn "error: zip entry escapes the install dir: #{entry.name}"
+            return false
+          end
+          entry.extract(entry.name, destination_directory: dir,
+            create_parent_directories: true)
+        end
+      end
+      true
+    rescue SystemCallError, Zip::Error => e
+      warn "error: unpack failed: #{e.class}: #{e.message}"
+      false
+    end
+
+    # The path to write zip entry NAME into, or nil when NAME escapes
+    # DIR. A download is untrusted input: this is the zip-slip
+    # guard.
+    def safe_entry_target(dir, name)
+      target = File.expand_path(File.join(dir, name))
+      base = File.expand_path(dir) + File::SEPARATOR
+      target.start_with?(base) ? target : nil
+    end
+
+    # One gem version pins one LanguageTool version. Older dirs are
+    # hundreds of MB each, so delete them on every setup run and keep
+    # only the current version (and any zips: they are re-deleted or
+    # re-extracted by their own flow).
+    def remove_stale_versions(dir, keep)
+      require "fileutils"
+      Dir.glob(File.join(dir, "LanguageTool-*")).each do |entry|
+        FileUtils.rm_rf(entry) if File.directory?(entry) && entry != keep
+      end
+    end
+
+    # Proves the installation works end to end: java runs the
+    # commandline jar and the custom rules load. The command exits
+    # nonzero on a broken java, a corrupt jar, or invalid rules XML,
+    # so setup can report a real verdict instead of "files exist".
+    # Returns a boolean, never raises or warns: the caller owns the
+    # message.
+    def smoke(install)
+      rule = rule_ids.first
+      return false if rule.nil?
+      require "tmpdir"
+      Dir.mktmpdir do |dir|
+        text = File.join(dir, "smoke.txt")
+        File.write(text, "Check this sentence.\n")
+        system(install.java!, "-jar", install.commandline_jar,
+          "--language", "en", "--rulefile", RULES_FILE,
+          "--enable", rule, "--enabledonly", text,
+          out: File::NULL, err: File::NULL) ? true : false
+      end
+    end
+
+    def rule_ids
+      File.read(RULES_FILE).scan(/<rule(?:group)? id="(\w+)"/).flatten
+    end
+  end
+end

data/lib/simple_english/markdown.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require_relative "paragraph"
+
+# Markdown stripping and paragraph splitting. Block structure comes
+# from the tree-sitter Markdown grammar shipped with the pack the
+# extractor already uses. Inline fixes stay regular expressions.
+# Stripping keeps the line count identical to the source.
+
+module SimpleEnglish
+  module Markdown
+    SENTENCE_END = /[.!?]/
+    # Non-prose blocks: their lines never reach the counting rules or
+    # LanguageTool.
+    BLANKED_BLOCKS = %w[fenced_code_block indented_code_block thematic_break].freeze
+    UNDERLINE = /\Asetext_h\d_underline\z/
+
+    module_function
+
+    # Blank out code and other non-prose blocks and neutralize inline
+    # code and heading markers. The line count stays identical to the
+    # source.
+    def strip(text)
+      lines = text.lines.map(&:chomp)
+      blank_rows(text).each { |row| lines[row] = "" }
+      lines.map { |line| strip_line(line) }.join("\n")
+    end
+
+    def strip_line(line)
+      line.gsub(/`[^`]*`/, "X").sub(/\A\#{1,6} /, "")
+    end
+
+    # A vertical list is not one paragraph: each list item is its own.
+    # Paragraph boundaries come from the tree-sitter Markdown grammar,
+    # so lazy continuations and interrupted lists match the spec
+    # instead of line-shape heuristics.
+    def paragraphs(text)
+      root = parse(text)
+      nodes = []
+      each_node(root) { |node| nodes << node if node.kind == "paragraph" }
+      nodes.sort_by!(&:start_byte)
+      nodes.filter_map do |node|
+        body = text.byteslice(node.start_byte, node.end_byte - node.start_byte)
+        lines = body.lines.map(&:strip).reject(&:empty?)
+        next if lines.empty?
+        Paragraph.new(lines: lines, start_line: node.start_position.row + 1,
+          procedural: inside_list_item?(node))
+      end
+    end
+
+    def sentences_of(paragraph_body)
+      paragraph_body.split(SENTENCE_END).map(&:strip).reject(&:empty?)
+    end
+
+    # The rows (0-based) a node fully occupies. Tree-sitter end
+    # positions are exclusive. A node ending at column 0 does not
+    # cover that row.
+    def node_rows(node)
+      last = node.end_position.column.zero? ? node.end_position.row - 1 : node.end_position.row
+      (node.start_position.row..last)
+    end
+
+    def blank_rows(text)
+      rows = []
+      each_node(parse(text)) do |node|
+        next if node.kind == "paragraph" # prose, not a block to blank
+        rows.concat(node_rows(node).to_a) if
+          BLANKED_BLOCKS.include?(node.kind) || UNDERLINE.match?(node.kind)
+      end
+      rows
+    end
+
+    def each_node(root)
+      stack = [root]
+      until stack.empty?
+        node = stack.pop
+        yield node
+        # No paragraphs live inside code blocks. Skip their subtrees.
+        next if BLANKED_BLOCKS.include?(node.kind)
+        node.child_count.times { |i| stack.push(node.child(i)) }
+      end
+    end
+
+    def inside_list_item?(node)
+      return false unless node.parent
+      node.parent.kind == "list_item" || inside_list_item?(node.parent)
+    end
+
+    def parse(text)
+      require "tree_sitter_language_pack"
+      TreeSitterLanguagePack.get_parser("markdown").parse(text).root_node
+    end
+
+    private_class_method :strip_line, :node_rows, :blank_rows, :each_node,
+      :inside_list_item?, :parse
+  end
+end

data/lib/simple_english/paragraph.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module SimpleEnglish
+  module Markdown
+    # A paragraph of prose: its lines, the first line's number in the
+    # source, and whether it starts with a list item (procedural text
+    # gets a tighter sentence limit).
+    Paragraph = Struct.new(:lines, :start_line, :procedural)
+  end
+end

data/lib/simple_english/plain_text.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SimpleEnglish
+  module Client
+    # The plain-text payload for /v2/check, duck-compatible with
+    # AnnotatedText::Result: #lt_params and #locate.
+    PlainText = Struct.new(:text) do
+      def lt_params
+        {"text" => text}
+      end
+
+      def locate(utf16_offset)
+        [Client.offset_to_line(text, utf16_offset), nil]
+      end
+    end
+  end
+end

data/lib/simple_english/result.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module SimpleEnglish
+  module AnnotatedText
+    # The payload for code comments. Duck interface shared with
+    # Client::PlainText: #lt_params (the form data for LanguageTool)
+    # and #locate (a match offset back to file line and column).
+    Result = Struct.new(:source, :segments, :stream) do
+      def lt_params
+        {"data" => SimpleEnglish::AnnotatedText.data_json(self)}
+      end
+
+      def locate(utf16_offset)
+        SimpleEnglish::AnnotatedText.locate(self, utf16_offset)
+      end
+    end
+  end
+end

data/lib/simple_english/segment.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SimpleEnglish
+  module AnnotatedText
+    # One piece of the annotation stream: checkable comment text or
+    # uncheckable markup (markers, closers, gaps between comments).
+    # stream_start is the segment's offset into Result#stream.
+    Segment = Struct.new(:text, :content, :file_char, :stream_start,
+      :interpret_as)
+  end
+end

data/lib/simple_english/server.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+# The se daemon: process lifecycle only. The wire protocol lives in
+# lib/simple_english/http.rb, the lint engine in lib/simple_english/engine.rb.
+# Failures raise typed errors (PortInUse, SetupError, InnerDied,
+# InnerTimeout). bin/se owns turning them into warnings and exit codes.
+
+require "fileutils"
+require "net/http"
+require "socket"
+require "tempfile"
+require "tmpdir"
+
+require_relative "install"
+
+module SimpleEnglish
+  module Server
+    class ServerError < StandardError; end
+
+    class PortInUse < ServerError
+      def initialize(port)
+        super("port #{port} is already in use.")
+      end
+    end
+
+    class InnerDied < ServerError; end
+    class InnerTimeout < ServerError; end
+
+    module_function
+
+    # LanguageTool's HTTP server has no --rulefile flag. Custom rules
+    # load from the classpath at this exact path.
+    def stage_rules(dir)
+      target = File.join(dir, "org/languagetool/rules/en/grammar_custom.xml")
+      FileUtils.mkdir_p(File.dirname(target))
+      FileUtils.cp(SimpleEnglish::LanguageTool::RULES_FILE, target)
+      target
+    end
+
+    # Probes both loopback families: on IPv6-first resolvers a "localhost"
+    # probe falls back to IPv4 and misses an IPv6 listener.
+    def assert_port_free(port)
+      ["127.0.0.1", "::1"].each do |address|
+        TCPServer.new(address, port).close
+      rescue Errno::EADDRNOTAVAIL
+        next
+      end
+    rescue Errno::EADDRINUSE
+      raise PortInUse, port
+    rescue Errno::EACCES
+      raise ServerError, "port #{port} cannot be bound. Use a port above 1024."
+    end
+
+    def start(port: Client::DEFAULT_PORT, install: Install.from_env, log: $stderr)
+      assert_port_free(port)
+      # The inner JVM lives on port + 1. Guard it too so an occupied
+      # inner port raises in milliseconds instead of timing out later.
+      assert_port_free(port + 1)
+      rules_dir = Dir.mktmpdir("se-rules")
+      stage_rules(rules_dir)
+      # The inner JVM's stderr goes to a file so failure messages can quote
+      # its first line. The dev log has a path. $stderr does not, so fall
+      # back to a temp file that Ruby unlinks when the process exits.
+      inner_log_path =
+        if log.respond_to?(:path)
+          log.path
+        else
+          inner_log = Tempfile.new("se-inner")
+          inner_log.close
+          inner_log.path
+        end
+      inner = spawn_inner(install: install, port: port + 1, rules_dir: rules_dir,
+        log_path: inner_log_path)
+      # TCPServer.new binds synchronously: construction means ready, so
+      # callers need no readiness polling. Closing the socket makes the
+      # serving loop's select raise and the loop exit.
+      server = TCPServer.new(port)
+      inner_died = false
+      # Closing the listening socket does not interrupt a syscall already
+      # blocked on the old fd, so the serving loop selects with a short
+      # timeout: the next IO.select on a closed socket raises IOError, so
+      # every shutdown path (trap, inner-death monitor) exits the loop
+      # within the timeout, on every platform.
+      stop_server = proc do
+        server.close
+      rescue IOError
+        # Already closed by another shutdown path.
+      end
+      # The monitor reaps the inner process, so nil it in the callback to
+      # keep the cleanup idempotent against the already-reaped pid.
+      monitor = monitor_inner(inner, on_death: proc do
+        inner = nil
+        inner_died = true
+        stop_server.call
+      end)
+      trap("INT") { stop_server.call }
+      trap("TERM") { stop_server.call }
+      (log.respond_to?(:puts) ? log : $stderr).puts "se listening on http://localhost:#{port}"
+      loop do
+        ready = IO.select([server], nil, nil, 0.5)
+        # Select timeout on an idle socket: loop back and select again.
+        next if ready.nil?
+        client = server.accept
+        Thread.new(client) { |c| HTTP.handle_client(c, port: port) }
+      rescue IOError, Errno::EBADF
+        # A trap or the inner-death monitor closed the listener. This
+        # happens during the select, or between select and accept.
+        break
+      end
+      monitor.kill if monitor.alive?
+      raise InnerDied, "inner LanguageTool server died. Rerun se serve." if inner_died
+    ensure
+      if inner
+        begin
+          Process.kill("TERM", inner)
+          Process.wait(inner)
+        rescue Errno::ESRCH, Errno::ECHILD
+          # The monitor thread already reaped the inner process.
+        end
+      end
+      monitor&.kill if monitor&.alive?
+      FileUtils.remove_entry(rules_dir) if rules_dir
+    end
+
+    # Reaps the inner process and runs on_death (which stops the outer
+    # server) whenever it exits, so the daemon dies instead of serving
+    # without LanguageTool behind it.
+    def monitor_inner(pid, on_death:)
+      Thread.new do
+        Process.wait(pid)
+        on_death.call
+      end
+    end
+
+    # Boots the inner LanguageTool server and blocks until it answers
+    # /v2/check. Raises InnerDied when it dies during startup, and
+    # InnerTimeout when it never becomes ready (the child is killed and
+    # reaped first, so a failed boot leaks no JVM). Both messages quote
+    # the inner log's first line.
+    def spawn_inner(install:, port:, rules_dir:, log_path:)
+      raise Install::SetupError, install.setup_error unless File.exist?(install.server_jar)
+      java = install.java!
+      begin
+        pid = Process.spawn(java,
+          "-cp", [install.server_jar, rules_dir].join(File::PATH_SEPARATOR),
+          "org.languagetool.server.HTTPServer", "--port", port.to_s,
+          out: File::NULL, err: log_path)
+      rescue Errno::ENOENT
+        # The java lookup can resolve to a path that no longer exists.
+        # Quote what was resolved so the crash names its cause.
+        raise ServerError, "cannot exec #{java.inspect}."
+      end
+      ready = false
+      died = false
+      deadline = Time.now + SimpleEnglish::LanguageTool::TIMEOUT_SECONDS
+      until ready || died || Time.now > deadline
+        # A JVM that dies instantly must not be polled for the full timeout.
+        # WNOHANG reaps it here, so nothing else may wait on this pid after.
+        died = !!Process.wait(pid, Process::WNOHANG)
+        break if died
+        begin
+          Net::HTTP.post_form(URI("http://localhost:#{port}/v2/check"),
+            {"language" => "en", "text" => "a"})
+          ready = true
+        rescue SystemCallError
+          sleep 0.5
+        end
+      end
+      if died
+        raise InnerDied,
+          "inner LanguageTool server exited during startup. " \
+          "First log line: #{first_log_line(log_path)}"
+      end
+      unless ready
+        kill_and_reap(pid)
+        raise InnerTimeout,
+          "inner LanguageTool server did not start " \
+          "within #{SimpleEnglish::LanguageTool::TIMEOUT_SECONDS} seconds. " \
+          "First log line: #{first_log_line(log_path)}"
+      end
+      pid
+    end
+
+    def kill_and_reap(pid)
+      Process.kill("TERM", pid)
+      Process.wait(pid)
+    rescue SystemCallError
+    end
+
+    def first_log_line(path)
+      File.read(path).split("\n").first || "unknown"
+    rescue SystemCallError
+      "unknown"
+    end
+  end
+end

data/lib/simple_english/span.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module SimpleEnglish
+  module Extractor
+    # One comment: its text and byte range in the source file.
+    Span = Struct.new(:text, :start_byte, :end_byte)
+  end
+end

data/lib/simple_english/suppressions.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# Inline suppressions. A line containing `se: ignore` (or
+# `se: ignore=RULE1,RULE2`) suppresses findings reported on that
+# same line. Pattern findings cite the line of the match, not the
+# paragraph or comment start, so put the directive on the line the
+# finding reports.
+
+module SimpleEnglish
+  module Suppressions
+    DIRECTIVE = /se:\s*ignore\s*(?:=\s*(?<rules>[\w,]+))?/
+
+    module_function
+
+    def filter(source, findings)
+      return findings if findings.empty?
+      blocked = directives(source)
+      return findings if blocked.empty?
+      findings.reject do |finding|
+        blocked.any? do |directive|
+          directive[:line] == finding.line &&
+            (directive[:rules].empty? || directive[:rules].include?(finding.rule))
+        end
+      end
+    end
+
+    def directives(source)
+      source.lines.each_with_index.filter_map do |line, index|
+        next unless (match = line.match(DIRECTIVE))
+        rules = match[:rules] ? match[:rules].split(",").map(&:strip) : []
+        {line: index + 1, rules: rules}
+      end
+    end
+
+    private_class_method :directives
+  end
+end