simple_english 0.6.0-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +212 -0
  4. data/THIRD_PARTY_NOTICES.md +72 -0
  5. data/bin/se +6 -0
  6. data/docs/RULES.md +496 -0
  7. data/lib/simple_english/cli.rb +183 -0
  8. data/lib/simple_english/client/daemon.rb +233 -0
  9. data/lib/simple_english/client/language_tool.rb +95 -0
  10. data/lib/simple_english/daemon/engine.rb +40 -0
  11. data/lib/simple_english/daemon/http.rb +82 -0
  12. data/lib/simple_english/daemon/server.rb +285 -0
  13. data/lib/simple_english/lint/annotated_text.rb +117 -0
  14. data/lib/simple_english/lint/counts.rb +40 -0
  15. data/lib/simple_english/lint/extractor.rb +60 -0
  16. data/lib/simple_english/lint/finding.rb +24 -0
  17. data/lib/simple_english/lint/lint_plan.rb +22 -0
  18. data/lib/simple_english/lint/markdown.rb +121 -0
  19. data/lib/simple_english/lint/paragraph.rb +10 -0
  20. data/lib/simple_english/lint/plain_text.rb +30 -0
  21. data/lib/simple_english/lint/result.rb +18 -0
  22. data/lib/simple_english/lint/segment.rb +11 -0
  23. data/lib/simple_english/lint/span.rb +8 -0
  24. data/lib/simple_english/lint/suppressions.rb +37 -0
  25. data/lib/simple_english/model_context_protocol.rb +134 -0
  26. data/lib/simple_english/setup/config.rb +63 -0
  27. data/lib/simple_english/setup/fingerprint.rb +38 -0
  28. data/lib/simple_english/setup/install.rb +40 -0
  29. data/lib/simple_english/setup/languagetool.rb +23 -0
  30. data/lib/simple_english/version.rb +5 -0
  31. data/lib/simple_english.rb +60 -0
  32. data/libexec/simple_english/languagetool-server +4 -0
  33. data/libexec/simple_english/languagetool-server.sbom.json +9044 -0
  34. data/licenses/LGPL-2.1.txt +501 -0
  35. data/licenses/OpenJDK-LICENSE.txt +347 -0
  36. data/licenses/languagetool-resource-en/12dicts-readme.html +738 -0
  37. data/licenses/languagetool-resource-en/agid-readme.txt +340 -0
  38. data/licenses/languagetool-resource-en/pos-readme.txt +115 -0
  39. data/rules/simple-english.xml +621 -0
  40. metadata +185 -0
@@ -0,0 +1,285 @@
1
+ # frozen_string_literal: true
2
+
3
+ # The se daemon: process lifecycle only. The wire protocol lives in
4
+ # lib/simple_english/daemon/http.rb, the lint engine in
5
+ # lib/simple_english/daemon/engine.rb.
6
+ # Failures raise typed errors (PortInUse, SetupError, InnerDied,
7
+ # InnerTimeout). bin/se owns turning them into warnings and exit codes.
8
+
9
+ require "net/http"
10
+ require "socket"
11
+ require "tempfile"
12
+
13
+ require_relative "../setup/install"
14
+ require_relative "../client/daemon"
15
+ require_relative "../setup/fingerprint"
16
+ require_relative "../version"
17
+
18
+ module SimpleEnglish
19
+ module Server
20
+ class ServerError < StandardError; end
21
+
22
+ class PortInUse < ServerError
23
+ def initialize(port)
24
+ super("port #{port} is already in use.")
25
+ end
26
+ end
27
+
28
+ class InnerDied < ServerError; end
29
+
30
+ class InnerTimeout < ServerError; end
31
+
32
+ module_function
33
+
34
+ # Probes both loopback families: on IPv6-first resolvers a "localhost"
35
+ # probe falls back to IPv4 and misses an IPv6 listener.
36
+ def assert_port_free(port)
37
+ raise PortInUse, port unless port_free?(port)
38
+ end
39
+
40
+ def port_free?(port)
41
+ ["127.0.0.1", "::1"].each do |address|
42
+ TCPServer.new(address, port).close
43
+ rescue Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT
44
+ # One family may be missing on this host (EADDRNOTAVAIL, or
45
+ # EAFNOSUPPORT when the kernel has no IPv6 at all): skip it
46
+ # and probe the other. A rescue at method scope returns here
47
+ # and never probes the second family.
48
+ next
49
+ end
50
+ true
51
+ rescue Errno::EADDRINUSE
52
+ false
53
+ rescue Errno::EACCES
54
+ raise ServerError, "port #{port} cannot be bound. Use a port above 1024."
55
+ end
56
+
57
+ # `se serve` over a live daemon (gem upgrade): when
58
+ # one of our daemons holds the port (it answers the
59
+ # handshake), stop it and take over the port. A service that
60
+ # does not answer the handshake, pre-handshake daemon included,
61
+ # keeps the PortInUse error: those need a manual look.
62
+ def takeover(port)
63
+ daemon = Client.info(base_url: "http://localhost:#{port}")
64
+ return unless daemon.is_a?(Hash)
65
+ stop(daemon["pid"])
66
+ # The outer listener closes first. The inner server takes
67
+ # longer. Both ports must free, or the next boot hits the inner
68
+ # port and dies.
69
+ deadline = Time.now + 10
70
+ sleep 0.2 until Time.now > deadline ||
71
+ !Client.info(base_url: "http://localhost:#{port}").is_a?(Hash) &&
72
+ port_free?(port + 1)
73
+ end
74
+
75
+ # Graceful only. SIGKILL skips the TERM trap and the inner server
76
+ # teardown. The orphan still holds port + 1 and breaks the next
77
+ # boot. The pid comes off the wire, so only a positive integer is
78
+ # TERMed: 0 signals the caller's own process group, a negative
79
+ # one every process the user may signal.
80
+ def stop(pid)
81
+ return nil unless pid.is_a?(Integer) && pid > 1
82
+ Process.kill("TERM", pid)
83
+ rescue Errno::ESRCH, Errno::EPERM
84
+ nil
85
+ end
86
+
87
+ # Everything a boot needs on the machine, checked before the
88
+ # takeover stops the old daemon: a reload that cannot boot must
89
+ # leave the running daemon alive.
90
+ def preflight_install(install)
91
+ install.executable!
92
+ end
93
+ private_class_method :preflight_install
94
+
95
+ # `se serve --detached`: replace any se daemon on the port, then
96
+ # boot a plain foreground serve as a detached child - the same
97
+ # shape the auto-boot spawns. The parent returns once the
98
+ # handshake answers. nil means the child never came up, and the
99
+ # visible serve (same command, no flag) shows why. dev_log is
100
+ # forwarded so a detached daemon can still get a log file.
101
+ def start_detached(port: Client::DEFAULT_PORT, install: Install.from_env,
102
+ dev_log: nil)
103
+ preflight_install(install)
104
+ takeover(port)
105
+ # A foreign port holder survives the takeover untouched: fail
106
+ # in milliseconds like the foreground boot, not after the
107
+ # 90 s spawn timeout.
108
+ assert_port_free(port)
109
+ assert_port_free(port + 1)
110
+ args = ["--port", port.to_s]
111
+ args.concat(["--dev-log", dev_log]) if dev_log
112
+ pid = Client.spawn_daemon(*args)
113
+ # The child gives its inner server TIMEOUT_SECONDS to boot, and
114
+ # the handshake answers only after that: wait the child's own
115
+ # budget, not the default 90 s, or a slow boot reads as
116
+ # failure while the child is still coming up.
117
+ daemon = Client.wait_for(
118
+ seconds: SimpleEnglish::LanguageTool::TIMEOUT_SECONDS + 10
119
+ ) { Client.info(base_url: "http://localhost:#{port}") }
120
+ return daemon if daemon.is_a?(Hash)
121
+ # Readiness failed. The child is either dead (InnerTimeout and
122
+ # InnerDied make it exit itself) or a straggler that later
123
+ # takes over the port from whatever daemon comes next.
124
+ # Stop it either way, and reap so nothing is left behind.
125
+ stop(pid)
126
+ begin
127
+ Process.wait(pid)
128
+ rescue SystemCallError
129
+ # The child already exited on its own (InnerTimeout,
130
+ # InnerDied): only the reap is left.
131
+ end
132
+ nil
133
+ end
134
+
135
+ def start(port: Client::DEFAULT_PORT, install: Install.from_env, log: $stderr)
136
+ enabled_rules = SimpleEnglish::LanguageTool.rule_ids
137
+ daemon_info = {"version" => VERSION, "pid" => Process.pid,
138
+ "gem_digest" => Fingerprint.gem,
139
+ "rules_digest" => Fingerprint.sha(File.read(LanguageTool::RULES_FILE))}
140
+ # Preflight everything the replacement needs before the
141
+ # takeover stops the old daemon.
142
+ preflight_install(install)
143
+ takeover(port)
144
+ assert_port_free(port)
145
+ # The inner native server lives on port + 1. Guard it too so an occupied
146
+ # inner port raises in milliseconds instead of timing out later.
147
+ assert_port_free(port + 1)
148
+ # The inner native server's stderr goes to a file so failure messages can quote
149
+ # its first line. Only an explicitly opened dev log (a File) is reused
150
+ # for that. $stderr reports path "<STDERR>", so it creates a file
151
+ # by that name in the CWD. Anything else falls back to a temp file
152
+ # that Ruby unlinks when the process exits.
153
+ inner_log_path =
154
+ if log.is_a?(File)
155
+ log.path
156
+ else
157
+ inner_log = Tempfile.new("se-inner")
158
+ inner_log.close
159
+ inner_log.path
160
+ end
161
+ inner = spawn_inner(install: install, port: port + 1,
162
+ log_path: inner_log_path)
163
+ # TCPServer.new binds synchronously: construction means ready, so
164
+ # callers need no readiness polling. Closing the socket makes the
165
+ # serving loop's select raise and the loop exit.
166
+ server = TCPServer.new(port)
167
+ inner_died = false
168
+ # Closing the listening socket does not interrupt a syscall already
169
+ # blocked on the old fd, so the serving loop selects with a short
170
+ # timeout: the next IO.select on a closed socket raises IOError, so
171
+ # every shutdown path (trap, inner-death monitor) exits the loop
172
+ # within the timeout, on every platform.
173
+ stop_server = proc do
174
+ server.close
175
+ rescue IOError
176
+ # Already closed by another shutdown path.
177
+ end
178
+ # The monitor reaps the inner process, so nil it in the callback to
179
+ # keep the cleanup idempotent against the already-reaped pid.
180
+ monitor = monitor_inner(inner, on_death: proc do
181
+ inner = nil
182
+ inner_died = true
183
+ stop_server.call
184
+ end)
185
+ trap("INT") { stop_server.call }
186
+ trap("TERM") { stop_server.call }
187
+ (log.respond_to?(:puts) ? log : $stderr).puts "se listening on http://localhost:#{port}"
188
+ loop do
189
+ ready = IO.select([server], nil, nil, 0.5)
190
+ # Select timeout on an idle socket: loop back and select again.
191
+ next if ready.nil?
192
+ client = server.accept
193
+ Thread.new(client) { |c|
194
+ HTTP.handle_client(c, port: port,
195
+ enabled_rules: enabled_rules, info: daemon_info)
196
+ }
197
+ rescue IOError, Errno::EBADF
198
+ # A trap or the inner-death monitor closed the listener. This
199
+ # happens during the select, or between select and accept.
200
+ break
201
+ end
202
+ monitor.kill if monitor.alive?
203
+ raise InnerDied, "lint engine died. Rerun se serve." if inner_died
204
+ ensure
205
+ if inner
206
+ begin
207
+ Process.kill("TERM", inner)
208
+ Process.wait(inner)
209
+ rescue Errno::ESRCH, Errno::ECHILD
210
+ # The monitor thread already reaped the inner process.
211
+ end
212
+ end
213
+ monitor&.kill if monitor&.alive?
214
+ end
215
+
216
+ # Reaps the inner process and runs on_death (which stops the outer
217
+ # server) whenever it exits, so the daemon dies instead of serving
218
+ # without the lint engine behind it.
219
+ def monitor_inner(pid, on_death:)
220
+ Thread.new do
221
+ Process.wait(pid)
222
+ on_death.call
223
+ end
224
+ end
225
+
226
+ # Boots the inner native server and blocks until it answers
227
+ # /v2/check. Raises InnerDied when it dies during startup, and
228
+ # InnerTimeout when it never becomes ready (the child is killed and
229
+ # reaped first, so a failed boot leaks no process). Both messages quote
230
+ # the inner log's first line.
231
+ def spawn_inner(install:, port:, log_path:)
232
+ executable = install.executable!
233
+ begin
234
+ pid = Process.spawn(executable, "--port", port.to_s,
235
+ out: File::NULL, err: log_path)
236
+ rescue Errno::ENOENT
237
+ raise ServerError, "cannot exec #{executable.inspect}."
238
+ end
239
+ ready = false
240
+ died = false
241
+ deadline = Time.now + SimpleEnglish::LanguageTool::TIMEOUT_SECONDS
242
+ until ready || died || Time.now > deadline
243
+ # A process that dies instantly must not be polled for the full timeout.
244
+ # WNOHANG reaps it here, so nothing else may wait on this pid after.
245
+ died = !!Process.wait(pid, Process::WNOHANG)
246
+ break if died
247
+ begin
248
+ Net::HTTP.post_form(URI("http://localhost:#{port}/v2/check"),
249
+ {"language" => "en",
250
+ "enabledRules" => SimpleEnglish::LanguageTool.rule_ids.first,
251
+ "enabledOnly" => "true",
252
+ "text" => "a"})
253
+ ready = true
254
+ rescue SystemCallError
255
+ sleep 0.5
256
+ end
257
+ end
258
+ if died
259
+ raise InnerDied,
260
+ "lint engine exited during startup. " \
261
+ "First log line: #{first_log_line(log_path)}"
262
+ end
263
+ unless ready
264
+ kill_and_reap(pid)
265
+ raise InnerTimeout,
266
+ "lint engine did not start " \
267
+ "within #{SimpleEnglish::LanguageTool::TIMEOUT_SECONDS} seconds. " \
268
+ "First log line: #{first_log_line(log_path)}"
269
+ end
270
+ pid
271
+ end
272
+
273
+ def kill_and_reap(pid)
274
+ Process.kill("TERM", pid)
275
+ Process.wait(pid)
276
+ rescue SystemCallError
277
+ end
278
+
279
+ def first_log_line(path)
280
+ File.read(path).split("\n").first || "unknown"
281
+ rescue SystemCallError
282
+ "unknown"
283
+ end
284
+ end
285
+ end
@@ -0,0 +1,117 @@
1
+ # frozen_string_literal: true
2
+
3
+ # LanguageTool AnnotatedText for code comments. Builds the annotation
4
+ # JSON and maps LT match offsets back to file line and UTF-16 column.
5
+ # Offsets are UTF-16 code units that count into the concatenation of
6
+ # all text and markup strings. interpretAs does not count. Semantics
7
+ # verified against a live LT 6.6 server on 2026-09-25.
8
+
9
+ require_relative "extractor"
10
+ require_relative "segment"
11
+ require_relative "result"
12
+
13
+ module SimpleEnglish
14
+ module AnnotatedText
15
+ # ASCII comment markers only (MARKER holds the set).
16
+ MARKER = /\A[\/#*;%-]+/
17
+ CLOSER = /\A.*?(\s*\*\/)\z/m
18
+
19
+ module_function
20
+
21
+ def build(source, spans)
22
+ segments = []
23
+ cursor = 0 # byte offset into source
24
+ emit = lambda do |text:, content:, file_char:, interpret_as: nil|
25
+ next if content.empty?
26
+ segments << Segment.new(text: text, content: content, file_char: file_char,
27
+ stream_start: segments.sum { |s| s.content.length }, interpret_as: interpret_as)
28
+ end
29
+ spans = spans.sort_by(&:start_byte)
30
+ spans.each do |span|
31
+ if cursor < span.start_byte
32
+ # Interior gaps break sentences between comments. Gaps with no
33
+ # checkable text before them (leading) change nothing.
34
+ emit.call(text: false, content: source.byteslice(cursor...span.start_byte),
35
+ file_char: char_index(source, cursor),
36
+ interpret_as: (segments.any? { |s| s.text }) ? "\n\n" : nil)
37
+ end
38
+ marker, body, closer = split(span.text)
39
+ file_char = char_index(source, span.start_byte)
40
+ emit.call(text: false, content: marker, file_char: file_char, interpret_as: " ")
41
+ emit.call(text: true, content: body, file_char: file_char + marker.length)
42
+ unless closer.empty?
43
+ emit.call(text: false, content: closer,
44
+ file_char: file_char + marker.length + body.length)
45
+ end
46
+ cursor = span.end_byte
47
+ end
48
+ emit.call(text: false, content: source.byteslice(cursor..),
49
+ file_char: char_index(source, cursor), interpret_as: nil)
50
+ Result.new(source: source, segments: segments,
51
+ stream: segments.map(&:content).join)
52
+ end
53
+
54
+ def data_json(result)
55
+ require "json"
56
+ JSON.generate("annotation" => result.segments.map do |segment|
57
+ if segment.text
58
+ {"text" => segment.content}
59
+ else
60
+ entry = {"markup" => segment.content}
61
+ entry["interpretAs"] = segment.interpret_as if segment.interpret_as
62
+ entry
63
+ end
64
+ end)
65
+ end
66
+
67
+ # -> [line, column], both 1-based. Column in UTF-16 code units.
68
+ def locate(result, utf16_offset)
69
+ char = utf16_to_char(result.stream, utf16_offset)
70
+ segment = result.segments.find do |s|
71
+ char >= s.stream_start && char < s.stream_start + s.content.length
72
+ end || result.segments.last
73
+ file_char = segment.file_char + (char - segment.stream_start)
74
+ starts = line_starts(result.source)
75
+ line = starts.rindex { |start| start <= file_char } + 1
76
+ [line, utf16_column(result.source, starts[line - 1], file_char)]
77
+ end
78
+
79
+ def split(comment)
80
+ marker = comment[MARKER] || ""
81
+ body = comment[marker.length..] || comment
82
+ closer = body[CLOSER] || ""
83
+ body = body[0, body.length - closer.length] unless closer.empty?
84
+ [marker, body, closer]
85
+ end
86
+
87
+ def char_index(source, byte)
88
+ # ponytail: O(filesize) per call. If large repos make this
89
+ # measurable, index byte->char once.
90
+ source.byteslice(0...byte).length
91
+ end
92
+
93
+ def line_starts(source)
94
+ starts = [0]
95
+ source.chars.each_with_index { |c, i| starts << i + 1 if c == "\n" }
96
+ starts
97
+ end
98
+
99
+ def utf16_to_char(stream, offset)
100
+ units = 0
101
+ stream.each_char.with_index do |char, index|
102
+ return index if units >= offset
103
+ units += (char.ord > 0xFFFF) ? 2 : 1
104
+ end
105
+ stream.length
106
+ end
107
+
108
+ def utf16_column(source, from, to)
109
+ units = 1
110
+ source[from...to].each_char { |c| units += (c.ord > 0xFFFF) ? 2 : 1 }
111
+ units
112
+ end
113
+
114
+ private_class_method :split, :char_index, :line_starts,
115
+ :utf16_to_char, :utf16_column
116
+ end
117
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Counting rules: sentence and paragraph length. Pure text analysis.
4
+
5
+ module SimpleEnglish
6
+ module Counts
7
+ PROCEDURAL_WORD_LIMIT = 20
8
+ DESCRIPTIVE_WORD_LIMIT = 25
9
+ PARAGRAPH_SENTENCE_LIMIT = 6
10
+
11
+ module_function
12
+
13
+ # Takes Markdown-stripped text, returns findings for long
14
+ # sentences and long paragraphs.
15
+ def check(text)
16
+ Markdown.paragraphs(text).flat_map { |paragraph| findings_for(paragraph) }
17
+ end
18
+
19
+ def findings_for(paragraph)
20
+ sentences = Markdown.sentences_of(paragraph.lines.join(" "))
21
+ limit = paragraph.procedural ? PROCEDURAL_WORD_LIMIT : DESCRIPTIVE_WORD_LIMIT
22
+
23
+ findings = sentences
24
+ .select { |sentence| sentence.split.size > limit }
25
+ .map do |_sentence|
26
+ Finding.new(line: paragraph.start_line, column: nil, rule: "SE_SENTENCE_TOO_LONG",
27
+ message: "Sentence has more than #{limit} words. Split it.")
28
+ end
29
+
30
+ if !paragraph.procedural && sentences.size > PARAGRAPH_SENTENCE_LIMIT
31
+ findings << Finding.new(line: paragraph.start_line, column: nil, rule: "SE_PARAGRAPH_TOO_LONG",
32
+ message: "Paragraph has more than #{PARAGRAPH_SENTENCE_LIMIT} " \
33
+ "sentences. Give one topic six sentences at most.")
34
+ end
35
+ findings
36
+ end
37
+
38
+ private_class_method :findings_for
39
+ end
40
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Tree-sitter comment extraction. One walk, no per-language code:
4
+ # every grammar marks comments with a kind ending in "comment".
5
+
6
+ require_relative "span"
7
+
8
+ module SimpleEnglish
9
+ module Extractor
10
+ EXTENSION_LANGUAGES = {
11
+ ".py" => "python",
12
+ ".rb" => "ruby",
13
+ ".js" => "javascript",
14
+ ".mjs" => "javascript",
15
+ ".ts" => "typescript",
16
+ ".yaml" => "yaml",
17
+ ".yml" => "yaml",
18
+ ".go" => "go",
19
+ ".rs" => "rust",
20
+ ".java" => "java",
21
+ ".sh" => "bash",
22
+ ".kt" => "kotlin",
23
+ ".cs" => "csharp",
24
+ ".cpp" => "cpp",
25
+ ".cc" => "cpp",
26
+ ".cxx" => "cpp",
27
+ ".hpp" => "cpp",
28
+ ".h" => "cpp"
29
+ }.freeze
30
+
31
+ module_function
32
+
33
+ # nil for files we do not lint.
34
+ def language_for(path)
35
+ EXTENSION_LANGUAGES[File.extname(path)]
36
+ end
37
+
38
+ def comment_spans(source, language)
39
+ require "tree_sitter_language_pack"
40
+ root = TreeSitterLanguagePack.get_parser(language).parse(source).root_node
41
+ spans = []
42
+ stack = [root]
43
+ until stack.empty?
44
+ node = stack.pop
45
+ if comment?(node)
46
+ spans << Span.new(text: source.byteslice(node.start_byte...node.end_byte),
47
+ start_byte: node.start_byte, end_byte: node.end_byte)
48
+ end
49
+ (node.child_count - 1).downto(0) { |i| stack.push(node.child(i)) }
50
+ end
51
+ spans.sort_by!(&:start_byte)
52
+ end
53
+
54
+ def comment?(node)
55
+ node.kind.end_with?("comment") && !node.kind.start_with?("non")
56
+ end
57
+
58
+ private_class_method :comment?
59
+ end
60
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ # One lint result. Positions are 1-based. The end position is exclusive.
4
+ # Counting findings have no columns or end positions. Findings from an older
5
+ # daemon can have a start column without an end position.
6
+
7
+ module SimpleEnglish
8
+ Finding = Struct.new(:line, :column, :end_line, :end_column, :rule, :message) do
9
+ # One finding as a text line: `path:line:col-end: [rule] message`.
10
+ # The CLI text output, the MCP tool, and the hook feedback all share
11
+ # this format. Keep them in step by rendering through this method.
12
+ def to_line(path)
13
+ location = "#{path}:#{line}"
14
+ if column
15
+ location += ":#{column}"
16
+ if end_line && end_column
17
+ finish = (end_line == line) ? end_column : "#{end_line}:#{end_column}"
18
+ location += "-#{finish}"
19
+ end
20
+ end
21
+ "#{location}: [#{rule}] #{message}"
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ # One decision, two callers: the CLI (SimpleEnglish.lint_file) and
4
+ # the daemon (Engine.lint) must agree on what kind of lint an input
5
+ # is. The CLI decides to skip comment-free files without a boot.
6
+ # The daemon decides again on its side of the wire. Both call this
7
+ # module, so the two tiers cannot drift.
8
+
9
+ module SimpleEnglish
10
+ module LintPlan
11
+ module_function
12
+
13
+ # Returns :prose when there is no language (the Markdown
14
+ # pipeline), nil for a comment-free code file (nothing to lint:
15
+ # no boot, no POST), or the comment spans for a code file.
16
+ def call(text, language)
17
+ return :prose unless language
18
+ spans = SimpleEnglish::Extractor.comment_spans(text, language)
19
+ spans.empty? ? nil : spans
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,121 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "paragraph"
4
+
5
+ # Markdown stripping and paragraph splitting. Block structure comes
6
+ # from the tree-sitter Markdown grammar shipped with the pack the
7
+ # extractor already uses. Inline fixes stay regular expressions.
8
+ # Stripping keeps the line count identical to the source.
9
+
10
+ module SimpleEnglish
11
+ module Markdown
12
+ # A terminator ends a sentence only before whitespace or at the
13
+ # end of the text, so periods inside URLs and file names do not
14
+ # split sentences.
15
+ SENTENCE_END = /(?<=[.!?])(?:\s+|$)/
16
+ # Non-prose blocks: their lines never reach the counting rules or
17
+ # LanguageTool.
18
+ BLANKED_BLOCKS = %w[fenced_code_block indented_code_block thematic_break].freeze
19
+ UNDERLINE = /\Asetext_h\d_underline\z/
20
+
21
+ module_function
22
+
23
+ # Blank out code and other non-prose blocks and neutralize inline
24
+ # code and heading markers. Line count and UTF-16 widths stay identical
25
+ # to the source so LanguageTool offsets remain source positions.
26
+ def strip(text)
27
+ blank = blank_rows(text).to_h { |row| [row, true] }
28
+ text.lines.each_with_index.map do |line, row|
29
+ ending = if line.end_with?("\r\n")
30
+ "\r\n"
31
+ elsif line.end_with?("\n")
32
+ "\n"
33
+ else
34
+ ""
35
+ end
36
+ body = line.delete_suffix(ending)
37
+ (blank[row] ? spaces_for(body) : strip_line(body)) + ending
38
+ end.join
39
+ end
40
+
41
+ def strip_line(line)
42
+ # Width-preserving replacements keep LanguageTool positions aligned
43
+ # with the source.
44
+ line
45
+ .gsub(/`[^`]*`/) { |code| "X" * utf16_length(code) }
46
+ .sub(/\A\#{1,6} /) { |marker| " " * marker.length }
47
+ end
48
+
49
+ def spaces_for(text)
50
+ " " * utf16_length(text)
51
+ end
52
+
53
+ def utf16_length(text)
54
+ text.each_char.sum { |char| (char.ord > 0xFFFF) ? 2 : 1 }
55
+ end
56
+
57
+ # A vertical list is not one paragraph: each list item is its own.
58
+ # Paragraph boundaries come from the tree-sitter Markdown grammar,
59
+ # so lazy continuations and interrupted lists match the spec
60
+ # instead of line-shape heuristics.
61
+ def paragraphs(text)
62
+ root = parse(text)
63
+ nodes = []
64
+ each_node(root) { |node| nodes << node if node.kind == "paragraph" }
65
+ nodes.sort_by!(&:start_byte)
66
+ nodes.filter_map do |node|
67
+ body = text.byteslice(node.start_byte, node.end_byte - node.start_byte)
68
+ lines = body.lines.map(&:strip).reject(&:empty?)
69
+ next if lines.empty?
70
+ Paragraph.new(lines: lines, start_line: node.start_position.row + 1,
71
+ procedural: inside_list_item?(node))
72
+ end
73
+ end
74
+
75
+ def sentences_of(paragraph_body)
76
+ paragraph_body.split(SENTENCE_END).map(&:strip).reject(&:empty?)
77
+ end
78
+
79
+ # The rows (0-based) a node fully occupies. Tree-sitter end
80
+ # positions are exclusive. A node ending at column 0 does not
81
+ # cover that row.
82
+ def node_rows(node)
83
+ last = node.end_position.column.zero? ? node.end_position.row - 1 : node.end_position.row
84
+ (node.start_position.row..last)
85
+ end
86
+
87
+ def blank_rows(text)
88
+ rows = []
89
+ each_node(parse(text)) do |node|
90
+ next if node.kind == "paragraph" # prose, not a block to blank
91
+ rows.concat(node_rows(node).to_a) if
92
+ BLANKED_BLOCKS.include?(node.kind) || UNDERLINE.match?(node.kind)
93
+ end
94
+ rows
95
+ end
96
+
97
+ def each_node(root)
98
+ stack = [root]
99
+ until stack.empty?
100
+ node = stack.pop
101
+ yield node
102
+ # No paragraphs live inside code blocks. Skip their subtrees.
103
+ next if BLANKED_BLOCKS.include?(node.kind)
104
+ node.child_count.times { |i| stack.push(node.child(i)) }
105
+ end
106
+ end
107
+
108
+ def inside_list_item?(node)
109
+ return false unless node.parent
110
+ node.parent.kind == "list_item" || inside_list_item?(node.parent)
111
+ end
112
+
113
+ def parse(text)
114
+ require "tree_sitter_language_pack"
115
+ TreeSitterLanguagePack.get_parser("markdown").parse(text).root_node
116
+ end
117
+
118
+ private_class_method :strip_line, :spaces_for, :utf16_length, :node_rows,
119
+ :blank_rows, :each_node, :inside_list_item?, :parse
120
+ end
121
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SimpleEnglish
4
+ module Markdown
5
+ # A paragraph of prose: its lines, the first line's number in the
6
+ # source, and whether it starts with a list item (procedural text
7
+ # gets a tighter sentence limit).
8
+ Paragraph = Struct.new(:lines, :start_line, :procedural)
9
+ end
10
+ end