simple_english 0.1.0

data/lib/simple_english/annotated_text.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+# LanguageTool AnnotatedText for code comments. Builds the annotation
+# JSON and maps LT match offsets back to file line and UTF-16 column.
+# Offsets are UTF-16 code units that count into the concatenation of
+# all text and markup strings. interpretAs does not count. Semantics
+# verified against a live LT 6.6 server on 2026-09-25.
+
+require_relative "extractor"
+require_relative "segment"
+require_relative "result"
+
+module SimpleEnglish
+  module AnnotatedText
+    # ASCII comment markers only (MARKER holds the set).
+    MARKER = /\A[\/#*;%-]+/
+    CLOSER = /\A.*?(\s*\*\/)\z/m
+
+    module_function
+
+    def build(source, spans)
+      segments = []
+      cursor = 0 # byte offset into source
+      emit = lambda do |text:, content:, file_char:, interpret_as: nil|
+        next if content.empty?
+        segments << Segment.new(text: text, content: content, file_char: file_char,
+          stream_start: segments.sum { |s| s.content.length }, interpret_as: interpret_as)
+      end
+      spans = spans.sort_by(&:start_byte)
+      spans.each do |span|
+        if cursor < span.start_byte
+          # Interior gaps break sentences between comments. Gaps with no
+          # checkable text before them (leading) change nothing.
+          emit.call(text: false, content: source.byteslice(cursor...span.start_byte),
+            file_char: char_index(source, cursor),
+            interpret_as: (segments.any? { |s| s.text }) ? "\n\n" : nil)
+        end
+        marker, body, closer = split(span.text)
+        file_char = char_index(source, span.start_byte)
+        emit.call(text: false, content: marker, file_char: file_char, interpret_as: " ")
+        emit.call(text: true, content: body, file_char: file_char + marker.length)
+        unless closer.empty?
+          emit.call(text: false, content: closer,
+            file_char: file_char + marker.length + body.length)
+        end
+        cursor = span.end_byte
+      end
+      emit.call(text: false, content: source.byteslice(cursor..),
+        file_char: char_index(source, cursor), interpret_as: nil)
+      Result.new(source: source, segments: segments,
+        stream: segments.map(&:content).join)
+    end
+
+    def data_json(result)
+      require "json"
+      JSON.generate("annotation" => result.segments.map do |segment|
+        if segment.text
+          {"text" => segment.content}
+        else
+          entry = {"markup" => segment.content}
+          entry["interpretAs"] = segment.interpret_as if segment.interpret_as
+          entry
+        end
+      end)
+    end
+
+    # -> [line, column], both 1-based. Column in UTF-16 code units.
+    def locate(result, utf16_offset)
+      char = utf16_to_char(result.stream, utf16_offset)
+      segment = result.segments.find do |s|
+        char >= s.stream_start && char < s.stream_start + s.content.length
+      end || result.segments.last
+      file_char = segment.file_char + (char - segment.stream_start)
+      starts = line_starts(result.source)
+      line = starts.rindex { |start| start <= file_char } + 1
+      [line, utf16_column(result.source, starts[line - 1], file_char)]
+    end
+
+    def split(comment)
+      marker = comment[MARKER] || ""
+      body = comment[marker.length..] || comment
+      closer = body[CLOSER] || ""
+      body = body[0, body.length - closer.length] unless closer.empty?
+      [marker, body, closer]
+    end
+
+    def char_index(source, byte)
+      # ponytail: O(filesize) per call. If large repos make this
+      # measurable, index byte->char once.
+      source.byteslice(0...byte).length
+    end
+
+    def line_starts(source)
+      starts = [0]
+      source.chars.each_with_index { |c, i| starts << i + 1 if c == "\n" }
+      starts
+    end
+
+    def utf16_to_char(stream, offset)
+      units = 0
+      stream.each_char.with_index do |char, index|
+        return index if units >= offset
+        units += (char.ord > 0xFFFF) ? 2 : 1
+      end
+      stream.length
+    end
+
+    def utf16_column(source, from, to)
+      units = 1
+      source[from...to].each_char { |c| units += (c.ord > 0xFFFF) ? 2 : 1 }
+      units
+    end
+
+    private_class_method :split, :char_index, :line_starts,
+      :utf16_to_char, :utf16_column
+  end
+end

data/lib/simple_english/cli.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+# The command-line interface: Thor commands for lint, serve, and
+# setup. bin/se is a thin runner over this, so the installed
+# gem's RubyGems shim can load it.
+
+require "thor"
+require_relative "../simple_english"
+
+module SimpleEnglish
+  class CLI < Thor
+    def self.exit_on_failure? = true
+
+    # Bare file arguments and "-" (stdin) are lint targets, not command
+    # names. Thor only falls back to the default task when the first
+    # argument is an option, so route them to lint explicitly.
+    def self.start(given_args = ARGV, config = {})
+      first = given_args.first
+      if first && !all_commands.key?(first) &&
+          (first == "-" || !first.start_with?("-"))
+        given_args = ["lint", *given_args]
+      end
+      super
+    end
+
+    # The test entry point. It keeps the old module interface.
+    def self.run(argv)
+      start(argv)
+    end
+
+    default_task :lint
+    desc "lint FILE_OR_DIR...", "Lint Markdown prose and code comments (- reads stdin)"
+    method_option :format, type: :string, default: "text", enum: %w[text json sarif],
+      banner: "text|json|sarif"
+    def lint(*paths)
+      if paths.empty?
+        help("lint")
+        return 2
+      end
+      config =
+        begin
+          SimpleEnglish::Config.load
+        rescue SimpleEnglish::Config::ConfigError => e
+          warn "error: #{e.message}"
+          return 2
+        end
+      results = []
+      self.class.expand_paths(paths).each do |path|
+        next if path != "-" && SimpleEnglish::Config.ignore?(config, path)
+        unless path == "-" || File.readable?(path)
+          warn "error: file not readable: #{path}"
+          return 2
+        end
+        text = (path == "-") ? $stdin.read : File.read(path)
+        findings = SimpleEnglish.lint_file(path, text)
+        return 2 if findings.nil? # already warned why
+        SimpleEnglish::Config.filter(config, path, findings).each do |finding|
+          results << [path, finding]
+        end
+      end
+      self.class.report(results, options[:format])
+      results.empty? ? 0 : 1
+    end
+
+    desc "serve", "Run the lint daemon in the foreground"
+    method_option :port, type: :numeric, default: SimpleEnglish::Client::DEFAULT_PORT,
+      desc: "Port to listen on"
+    method_option :"dev-log", type: :string, banner: "PATH",
+      desc: "Write daemon stderr to PATH"
+    def serve
+      log = options[:"dev-log"] ? File.open(options[:"dev-log"], "w") : $stderr
+      SimpleEnglish::Server.start(port: options[:port],
+        install: SimpleEnglish::Install.from_env, log: log)
+      0
+    rescue SimpleEnglish::Install::SetupError, SimpleEnglish::Server::ServerError => e
+      warn "error: #{e.message}"
+      2
+    end
+
+    desc "setup", "Download LanguageTool and locate Java. Idempotent."
+    method_option :dir, type: :string, banner: "PATH",
+      desc: "Install into PATH (default: the shared cache)"
+    def setup
+      install = SimpleEnglish::Install.from_env
+      lt_dir = SimpleEnglish::LanguageTool.install(options[:dir] || install.cache_dir)
+      return 2 unless lt_dir
+      unless install.java?
+        warn "error: java not found. Install a JRE (on macOS: brew install openjdk), " \
+          "or set SE_JAVA to your java binary."
+        return 2
+      end
+      unless SimpleEnglish::LanguageTool.smoke(install)
+        warn "error: LanguageTool smoke test failed. The download may be corrupt. " \
+          "Delete #{lt_dir} and rerun `se setup`."
+        return 2
+      end
+      puts <<~SETUP
+        LanguageTool #{SimpleEnglish::LanguageTool::LT_VERSION} is at #{lt_dir}
+
+        se finds it there automatically. Nothing to export.
+      SETUP
+      0
+    end
+
+    # "-" stays as-is for stdin. Directories expand to all lintable files.
+    # Glob output keeps a "./" prefix when the argument is ".". Strip it so
+    # paths and config ignore globs always see the same form.
+    def self.expand_paths(argv)
+      extensions = (SimpleEnglish::Extractor::EXTENSION_LANGUAGES.keys.map { |e| e.delete_prefix(".") } + ["md"]).uniq.join(",")
+      argv.flat_map do |path|
+        if File.directory?(path)
+          Dir.glob(File.join(path, "**/*.{#{extensions}}")).sort
+        else
+          path
+        end
+      end.map { |path| path.delete_prefix("./") }
+    end
+
+    def self.report(results, format)
+      require "json"
+      case format
+      when "json"
+        puts JSON.pretty_generate(results.map do |path, finding|
+          {"path" => path, "line" => finding.line, "column" => finding.column,
+           "rule" => finding.rule, "message" => finding.message}
+        end)
+      when "sarif"
+        puts JSON.pretty_generate(
+          "version" => "2.1.0",
+          "$schema" => "https://json.schemastore.org/sarif-2.1.0.json",
+          "runs" => [{
+            "tool" => {"driver" => {"name" => "se"}},
+            "results" => results.map do |path, finding|
+              region = {"startLine" => finding.line}
+              region["startColumn"] = finding.column if finding.column
+              {"ruleId" => finding.rule, "level" => "error",
+               "message" => {"text" => finding.message},
+               "locations" => [{"physicalLocation" => {
+                 "artifactLocation" => {"uri" => path},
+                 "region" => region
+               }}]}
+            end
+          }]
+        )
+      else
+        results.each do |path, finding|
+          puts "#{path}:#{finding.line}: [#{finding.rule}] #{finding.message}"
+        end
+      end
+    end
+  end
+end

data/lib/simple_english/client.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+# HTTP client for the LanguageTool server and the se daemon.
+
+require "json"
+require "net/http"
+require "rbconfig"
+require "uri"
+
+require_relative "plain_text"
+require_relative "languagetool"
+
+module SimpleEnglish
+  module Client
+    module_function
+
+    # LanguageTool reports offsets in Java UTF-16 code units.
+    # Astral characters (emoji) count as two. Count them so a
+    # newline comparison cannot drift past a boundary. A match
+    # starting at the newline itself belongs to the next line.
+    def offset_to_line(text, offset)
+      line = 1
+      units = 0
+      text.each_char do |char|
+        line += 1 if char == "\n" && units <= offset
+        return line if units >= offset
+        units += (char.ord > 0xFFFF) ? 2 : 1
+      end
+      line
+    end
+
+    DEFAULT_PORT = 8181
+    REQUEST_TIMEOUT = 30
+
+    def url
+      ENV.fetch("SE_SERVER_URL") { "http://localhost:#{DEFAULT_PORT}" }
+    end
+
+    def post(uri, params, read_timeout: REQUEST_TIMEOUT)
+      Net::HTTP.start(uri.host, uri.port, open_timeout: 5,
+        read_timeout: read_timeout) do |http|
+        request = Net::HTTP::Post.new(uri.request_uri)
+        request.set_form_data(params)
+        http.request(request)
+      end
+    end
+
+    private_class_method :post
+
+    # Pattern rules. `payload` is Markdown-stripped text (String) or an
+    # AnnotatedText::Result (code comments): one payload interface,
+    # #lt_params and #locate, either side of the daemon's LT request.
+    def check(payload, base_url: url)
+      payload = to_payload(payload)
+      params = {"language" => "en",
+                "enabledRules" => SimpleEnglish::LanguageTool.rule_ids.join(","),
+                "enabledOnly" => "true"}.merge(payload.lt_params)
+      response = post(URI("#{base_url}/v2/check"), params)
+      parse_matches(JSON.parse(response.body).fetch("matches"), payload)
+    end
+
+    def parse_matches(matches, payload)
+      payload = to_payload(payload)
+      matches.map do |match|
+        line, column = payload.locate(match.fetch("offset"))
+        Finding.new(line: line, column: column,
+          rule: match.fetch("rule").fetch("id"), message: match.fetch("message"))
+      end
+    end
+
+    def to_payload(payload)
+      payload.is_a?(String) ? PlainText.new(payload) : payload
+    end
+
+    private_class_method :to_payload
+
+    # Full lint via the se daemon. Raw Markdown in, or code
+    # source with a language for the comment pipeline. nil when
+    # unreachable or the response is unusable, so callers can fall
+    # back without seeing a stack trace.
+    def lint(text, base_url: url, language: nil)
+      params = {"text" => text}
+      params["language"] = language if language
+      response = post(URI("#{base_url}/lint"), params)
+      return nil unless response.is_a?(Net::HTTPSuccess)
+      body = JSON.parse(response.body)
+      return nil unless body.is_a?(Array)
+      body.map do |hash|
+        Finding.new(line: hash.fetch("line"), column: hash["column"],
+          rule: hash.fetch("rule"), message: hash.fetch("message"))
+      end
+    rescue SystemCallError, SocketError, Timeout::Error,
+      JSON::ParserError, TypeError
+      nil
+    end
+
+    def up?(base_url: url)
+      uri = URI(base_url)
+      Net::HTTP.start(uri.host, uri.port, open_timeout: 1,
+        read_timeout: 2) { |http| http.head("/") }
+      true
+    rescue Errno::ECONNREFUSED, SocketError, Timeout::Error
+      false
+    end
+
+    # True when the daemon answers. Starts it when we own the default
+    # URL. A custom SE_SERVER_URL belongs to someone else, so
+    # never spawn against it. It returns false when unusable. The
+    # caller owns the exit status. Diagnostics go to stderr here, where the cause
+    # is known.
+    def ensure_up(install: SimpleEnglish::Install.from_env)
+      unless File.exist?(install.server_jar)
+        warn install.setup_error
+        return false
+      end
+      return true if up?
+      if ENV["SE_SERVER_URL"]
+        warn "error: SE_SERVER_URL is set but #{url} does not answer."
+        return false
+      end
+      warn "se: daemon not running; starting it (first lint takes ~15s)..."
+      bin = File.expand_path("../../bin/se", __dir__)
+      Process.spawn(RbConfig.ruby, bin, "serve", out: File::NULL, err: File::NULL)
+      deadline = Time.now + 90
+      until Time.now > deadline
+        return true if up?
+        sleep 0.5
+      end
+      warn "error: se daemon did not come up. Run `se serve` and read its output."
+      false
+    end
+  end
+end

data/lib/simple_english/config.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# .simple-english.yml: ignore (path globs, matched against the paths given
+# on the command line) and disabled-rules. Loaded from the CWD.
+
+module SimpleEnglish
+  module Config
+    ConfigError = Class.new(StandardError)
+    DEFAULT = {ignore: [], disabled_rules: []}.freeze
+
+    module_function
+
+    def load(dir = Dir.pwd)
+      file = File.join(dir, ".simple-english.yml")
+      return DEFAULT unless File.exist?(file)
+      require "yaml"
+      data = YAML.safe_load_file(file) || {}
+      {ignore: Array(data["ignore"]),
+       disabled_rules: Array(data["disabled-rules"])}
+    rescue Psych::SyntaxError => e
+      raise ConfigError, ".simple-english.yml: #{e.message}"
+    end
+
+    def ignore?(config, path)
+      config[:ignore].any? { |pattern| matches?(pattern, path) }
+    end
+
+    # File.fnmatch has no globstar: `**` never crosses directories.
+    # Translate instead: `**` as a whole segment is any depth, `*` and
+    # `?` stay within one segment.
+    def matches?(pattern, path)
+      glob_to_regex(pattern).match?(path)
+    end
+
+    def glob_to_regex(pattern)
+      Regexp.new("\\A" + pattern.split("/").map do |segment|
+        if segment == "**"
+          "(?:[^/]+/)*[^/]*"
+        else
+          Regexp.escape(segment).gsub("\\*", "[^/]*").gsub("\\?", "[^/]")
+        end
+      end.join("/") + "\\z")
+    end
+
+    def filter(config, path, findings)
+      return [] if ignore?(config, path)
+      disabled = config[:disabled_rules]
+      findings.reject { |finding| disabled.include?(finding.rule) }
+    end
+  end
+end

data/lib/simple_english/counts.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# Counting rules: sentence and paragraph length. Pure text analysis.
+
+module SimpleEnglish
+  module Counts
+    PROCEDURAL_WORD_LIMIT = 20
+    DESCRIPTIVE_WORD_LIMIT = 25
+    PARAGRAPH_SENTENCE_LIMIT = 6
+
+    module_function
+
+    # Takes Markdown-stripped text, returns findings for long
+    # sentences and long paragraphs.
+    def check(text)
+      Markdown.paragraphs(text).flat_map { |paragraph| findings_for(paragraph) }
+    end
+
+    def findings_for(paragraph)
+      sentences = Markdown.sentences_of(paragraph.lines.join(" "))
+      limit = paragraph.procedural ? PROCEDURAL_WORD_LIMIT : DESCRIPTIVE_WORD_LIMIT
+
+      findings = sentences
+        .select { |sentence| sentence.split.size > limit }
+        .map do |_sentence|
+        Finding.new(line: paragraph.start_line, column: nil, rule: "SE_SENTENCE_TOO_LONG",
+          message: "Sentence has more than #{limit} words. Split it.")
+      end
+
+      if !paragraph.procedural && sentences.size > PARAGRAPH_SENTENCE_LIMIT
+        findings << Finding.new(line: paragraph.start_line, column: nil, rule: "SE_PARAGRAPH_TOO_LONG",
+          message: "Paragraph has more than #{PARAGRAPH_SENTENCE_LIMIT} " \
+                   "sentences. Give one topic six sentences at most.")
+      end
+      findings
+    end
+
+    private_class_method :findings_for
+  end
+end

data/lib/simple_english/engine.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# The complete lint engine, daemon-side. Never call SimpleEnglish.lint_text
+# here: that call probes the daemon and recurses into the server.
+
+require "json"
+
+module SimpleEnglish
+  module Engine
+    module_function
+
+    def lint(text, base_url:, language: nil)
+      if language
+        spans = SimpleEnglish::Extractor.comment_spans(text, language)
+        spans.empty? ? [] :
+          SimpleEnglish::Client.check(
+            SimpleEnglish::AnnotatedText.build(text, spans), base_url: base_url
+          )
+      else
+        stripped = SimpleEnglish::Markdown.strip(text)
+        (SimpleEnglish::Counts.check(stripped) +
+          SimpleEnglish::Client.check(stripped, base_url: base_url))
+          .sort_by { |finding| [finding.line, finding.rule] }
+      end
+    end
+
+    def lint_json(text, base_url:, language: nil)
+      JSON.generate(lint(text, base_url: base_url, language: language).map(&:to_h))
+    end
+  end
+end

data/lib/simple_english/extractor.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# Tree-sitter comment extraction. One walk, no per-language code:
+# every grammar marks comments with a kind ending in "comment".
+
+require_relative "span"
+
+module SimpleEnglish
+  module Extractor
+    EXTENSION_LANGUAGES = {
+      ".py" => "python",
+      ".rb" => "ruby",
+      ".js" => "javascript",
+      ".mjs" => "javascript",
+      ".ts" => "typescript",
+      ".yaml" => "yaml",
+      ".yml" => "yaml",
+      ".go" => "go",
+      ".rs" => "rust",
+      ".java" => "java",
+      ".sh" => "bash",
+      ".kt" => "kotlin",
+      ".cs" => "csharp"
+    }.freeze
+
+    module_function
+
+    # nil for files we do not lint.
+    def language_for(path)
+      EXTENSION_LANGUAGES[File.extname(path)]
+    end
+
+    def comment_spans(source, language)
+      require "tree_sitter_language_pack"
+      root = TreeSitterLanguagePack.get_parser(language).parse(source).root_node
+      spans = []
+      stack = [root]
+      until stack.empty?
+        node = stack.pop
+        if comment?(node)
+          spans << Span.new(text: source.byteslice(node.start_byte...node.end_byte),
+            start_byte: node.start_byte, end_byte: node.end_byte)
+        end
+        (node.child_count - 1).downto(0) { |i| stack.push(node.child(i)) }
+      end
+      spans.sort_by!(&:start_byte)
+    end
+
+    def comment?(node)
+      node.kind.end_with?("comment") && !node.kind.start_with?("non")
+    end
+
+    private_class_method :comment?
+  end
+end

data/lib/simple_english/finding.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# One lint result. The line and column are 1-based. The column is
+# nil when the rule has no position within the line.
+
+module SimpleEnglish
+  Finding = Struct.new(:line, :column, :rule, :message)
+end

data/lib/simple_english/http.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# Minimal HTTP/1.1 wire framing for the daemon's outer server. Serving
+# one request per connection (Connection: close) keeps this simple: no
+# chunked bodies, no pipelining.
+
+require "json"
+
+module SimpleEnglish
+  module HTTP
+    module_function
+
+    # Reads one HTTP request from io: request line, headers, then
+    # Content-Length bytes of body. nil when malformed or the peer
+    # hung up.
+    def read_request(io)
+      request_line = io.gets
+      parts = request_line ? request_line.split(" ") : []
+      return nil if parts.length < 2
+      content_length = 0
+      while (line = io.gets)
+        break if line == "\r\n" || line == "\n"
+        name, value = line.split(":", 2)
+        next unless value
+        content_length = value.strip.to_i if name.casecmp?("content-length")
+      end
+      return nil if content_length.negative?
+      body = content_length.zero? ? +"" : io.read(content_length)
+      {method: parts[0], path: parts[1], body: body}
+    end
+
+    # Writes a minimal HTTP/1.1 response and closes the logical
+    # connection (Connection: close): status line, always-JSON headers,
+    # Content-Length, then the body.
+    def write_response(io, status:, body:, content_type: "application/json")
+      reason = {200 => "OK", 404 => "Not Found",
+                500 => "Internal Server Error"}.fetch(status, "Unknown")
+      io.write("HTTP/1.1 #{status} #{reason}\r\n" \
+        "Content-Type: #{content_type}\r\n" \
+        "Content-Length: #{body.bytesize}\r\n" \
+        "Connection: close\r\n\r\n")
+      io.write(body)
+    end
+
+    # One thread per request: lint can take seconds, so a slow request
+    # must not block readiness probes or other clients.
+    def handle_client(client, port:)
+      request = read_request(client)
+      if request.nil?
+        # Malformed request or immediate hangup: nothing to answer.
+      elsif request[:method] == "HEAD"
+        # Client.up? does http.head("/") and treats any response as up.
+        write_response(client, status: 200, body: "")
+      elsif request[:method] == "POST" && request[:path] == "/lint"
+        begin
+          # Ruling 2026-09-24: Client.lint posts form-encoded data, so the
+          # handler decodes it. A raw body lints "text=Don%27t...".
+          # The optional language switches to the code-comment pipeline.
+          params = URI.decode_www_form(request[:body]).to_h
+          write_response(client, status: 200,
+            body: SimpleEnglish::Engine.lint_json(params["text"],
+              base_url: "http://localhost:#{port + 1}",
+              language: params["language"]))
+        rescue => e
+          write_response(client, status: 500,
+            body: JSON.generate({"error" => e.message}))
+        end
+      else
+        write_response(client, status: 404,
+          body: JSON.generate({"error" => "not found"}))
+      end
+    ensure
+      client.close
+    end
+  end
+end