simple_english 0.6.0-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +212 -0
- data/THIRD_PARTY_NOTICES.md +72 -0
- data/bin/se +6 -0
- data/docs/RULES.md +496 -0
- data/lib/simple_english/cli.rb +183 -0
- data/lib/simple_english/client/daemon.rb +233 -0
- data/lib/simple_english/client/language_tool.rb +95 -0
- data/lib/simple_english/daemon/engine.rb +40 -0
- data/lib/simple_english/daemon/http.rb +82 -0
- data/lib/simple_english/daemon/server.rb +285 -0
- data/lib/simple_english/lint/annotated_text.rb +117 -0
- data/lib/simple_english/lint/counts.rb +40 -0
- data/lib/simple_english/lint/extractor.rb +60 -0
- data/lib/simple_english/lint/finding.rb +24 -0
- data/lib/simple_english/lint/lint_plan.rb +22 -0
- data/lib/simple_english/lint/markdown.rb +121 -0
- data/lib/simple_english/lint/paragraph.rb +10 -0
- data/lib/simple_english/lint/plain_text.rb +30 -0
- data/lib/simple_english/lint/result.rb +18 -0
- data/lib/simple_english/lint/segment.rb +11 -0
- data/lib/simple_english/lint/span.rb +8 -0
- data/lib/simple_english/lint/suppressions.rb +37 -0
- data/lib/simple_english/model_context_protocol.rb +134 -0
- data/lib/simple_english/setup/config.rb +63 -0
- data/lib/simple_english/setup/fingerprint.rb +38 -0
- data/lib/simple_english/setup/install.rb +40 -0
- data/lib/simple_english/setup/languagetool.rb +23 -0
- data/lib/simple_english/version.rb +5 -0
- data/lib/simple_english.rb +60 -0
- data/libexec/simple_english/languagetool-server +4 -0
- data/libexec/simple_english/languagetool-server.sbom.json +9044 -0
- data/licenses/LGPL-2.1.txt +501 -0
- data/licenses/OpenJDK-LICENSE.txt +347 -0
- data/licenses/languagetool-resource-en/12dicts-readme.html +738 -0
- data/licenses/languagetool-resource-en/agid-readme.txt +340 -0
- data/licenses/languagetool-resource-en/pos-readme.txt +115 -0
- data/rules/simple-english.xml +621 -0
- metadata +185 -0
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# The se daemon: process lifecycle only. The wire protocol lives in
|
|
4
|
+
# lib/simple_english/daemon/http.rb, the lint engine in
|
|
5
|
+
# lib/simple_english/daemon/engine.rb.
|
|
6
|
+
# Failures raise typed errors (PortInUse, SetupError, InnerDied,
|
|
7
|
+
# InnerTimeout). bin/se owns turning them into warnings and exit codes.
|
|
8
|
+
|
|
9
|
+
require "net/http"
|
|
10
|
+
require "socket"
|
|
11
|
+
require "tempfile"
|
|
12
|
+
|
|
13
|
+
require_relative "../setup/install"
|
|
14
|
+
require_relative "../client/daemon"
|
|
15
|
+
require_relative "../setup/fingerprint"
|
|
16
|
+
require_relative "../version"
|
|
17
|
+
|
|
18
|
+
module SimpleEnglish
|
|
19
|
+
module Server
|
|
20
|
+
class ServerError < StandardError; end
|
|
21
|
+
|
|
22
|
+
class PortInUse < ServerError
|
|
23
|
+
def initialize(port)
|
|
24
|
+
super("port #{port} is already in use.")
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
class InnerDied < ServerError; end
|
|
29
|
+
|
|
30
|
+
class InnerTimeout < ServerError; end
|
|
31
|
+
|
|
32
|
+
module_function
|
|
33
|
+
|
|
34
|
+
# Probes both loopback families: on IPv6-first resolvers a "localhost"
|
|
35
|
+
# probe falls back to IPv4 and misses an IPv6 listener.
|
|
36
|
+
def assert_port_free(port)
|
|
37
|
+
raise PortInUse, port unless port_free?(port)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def port_free?(port)
|
|
41
|
+
["127.0.0.1", "::1"].each do |address|
|
|
42
|
+
TCPServer.new(address, port).close
|
|
43
|
+
rescue Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT
|
|
44
|
+
# One family may be missing on this host (EADDRNOTAVAIL, or
|
|
45
|
+
# EAFNOSUPPORT when the kernel has no IPv6 at all): skip it
|
|
46
|
+
# and probe the other. A rescue at method scope returns here
|
|
47
|
+
# and never probes the second family.
|
|
48
|
+
next
|
|
49
|
+
end
|
|
50
|
+
true
|
|
51
|
+
rescue Errno::EADDRINUSE
|
|
52
|
+
false
|
|
53
|
+
rescue Errno::EACCES
|
|
54
|
+
raise ServerError, "port #{port} cannot be bound. Use a port above 1024."
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# `se serve` over a live daemon (gem upgrade): when
|
|
58
|
+
# one of our daemons holds the port (it answers the
|
|
59
|
+
# handshake), stop it and take over the port. A service that
|
|
60
|
+
# does not answer the handshake, pre-handshake daemon included,
|
|
61
|
+
# keeps the PortInUse error: those need a manual look.
|
|
62
|
+
def takeover(port)
|
|
63
|
+
daemon = Client.info(base_url: "http://localhost:#{port}")
|
|
64
|
+
return unless daemon.is_a?(Hash)
|
|
65
|
+
stop(daemon["pid"])
|
|
66
|
+
# The outer listener closes first. The inner server takes
|
|
67
|
+
# longer. Both ports must free, or the next boot hits the inner
|
|
68
|
+
# port and dies.
|
|
69
|
+
deadline = Time.now + 10
|
|
70
|
+
sleep 0.2 until Time.now > deadline ||
|
|
71
|
+
!Client.info(base_url: "http://localhost:#{port}").is_a?(Hash) &&
|
|
72
|
+
port_free?(port + 1)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Graceful only. SIGKILL skips the TERM trap and the inner server
|
|
76
|
+
# teardown. The orphan still holds port + 1 and breaks the next
|
|
77
|
+
# boot. The pid comes off the wire, so only a positive integer is
|
|
78
|
+
# TERMed: 0 signals the caller's own process group, a negative
|
|
79
|
+
# one every process the user may signal.
|
|
80
|
+
def stop(pid)
|
|
81
|
+
return nil unless pid.is_a?(Integer) && pid > 1
|
|
82
|
+
Process.kill("TERM", pid)
|
|
83
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
84
|
+
nil
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Everything a boot needs on the machine, checked before the
|
|
88
|
+
# takeover stops the old daemon: a reload that cannot boot must
|
|
89
|
+
# leave the running daemon alive.
|
|
90
|
+
def preflight_install(install)
|
|
91
|
+
install.executable!
|
|
92
|
+
end
|
|
93
|
+
private_class_method :preflight_install
|
|
94
|
+
|
|
95
|
+
# `se serve --detached`: replace any se daemon on the port, then
|
|
96
|
+
# boot a plain foreground serve as a detached child - the same
|
|
97
|
+
# shape the auto-boot spawns. The parent returns once the
|
|
98
|
+
# handshake answers. nil means the child never came up, and the
|
|
99
|
+
# visible serve (same command, no flag) shows why. dev_log is
|
|
100
|
+
# forwarded so a detached daemon can still get a log file.
|
|
101
|
+
def start_detached(port: Client::DEFAULT_PORT, install: Install.from_env,
|
|
102
|
+
dev_log: nil)
|
|
103
|
+
preflight_install(install)
|
|
104
|
+
takeover(port)
|
|
105
|
+
# A foreign port holder survives the takeover untouched: fail
|
|
106
|
+
# in milliseconds like the foreground boot, not after the
|
|
107
|
+
# 90 s spawn timeout.
|
|
108
|
+
assert_port_free(port)
|
|
109
|
+
assert_port_free(port + 1)
|
|
110
|
+
args = ["--port", port.to_s]
|
|
111
|
+
args.concat(["--dev-log", dev_log]) if dev_log
|
|
112
|
+
pid = Client.spawn_daemon(*args)
|
|
113
|
+
# The child gives its inner server TIMEOUT_SECONDS to boot, and
|
|
114
|
+
# the handshake answers only after that: wait the child's own
|
|
115
|
+
# budget, not the default 90 s, or a slow boot reads as
|
|
116
|
+
# failure while the child is still coming up.
|
|
117
|
+
daemon = Client.wait_for(
|
|
118
|
+
seconds: SimpleEnglish::LanguageTool::TIMEOUT_SECONDS + 10
|
|
119
|
+
) { Client.info(base_url: "http://localhost:#{port}") }
|
|
120
|
+
return daemon if daemon.is_a?(Hash)
|
|
121
|
+
# Readiness failed. The child is either dead (InnerTimeout and
|
|
122
|
+
# InnerDied make it exit itself) or a straggler that later
|
|
123
|
+
# takes over the port from whatever daemon comes next.
|
|
124
|
+
# Stop it either way, and reap so nothing is left behind.
|
|
125
|
+
stop(pid)
|
|
126
|
+
begin
|
|
127
|
+
Process.wait(pid)
|
|
128
|
+
rescue SystemCallError
|
|
129
|
+
# The child already exited on its own (InnerTimeout,
|
|
130
|
+
# InnerDied): only the reap is left.
|
|
131
|
+
end
|
|
132
|
+
nil
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def start(port: Client::DEFAULT_PORT, install: Install.from_env, log: $stderr)
|
|
136
|
+
enabled_rules = SimpleEnglish::LanguageTool.rule_ids
|
|
137
|
+
daemon_info = {"version" => VERSION, "pid" => Process.pid,
|
|
138
|
+
"gem_digest" => Fingerprint.gem,
|
|
139
|
+
"rules_digest" => Fingerprint.sha(File.read(LanguageTool::RULES_FILE))}
|
|
140
|
+
# Preflight everything the replacement needs before the
|
|
141
|
+
# takeover stops the old daemon.
|
|
142
|
+
preflight_install(install)
|
|
143
|
+
takeover(port)
|
|
144
|
+
assert_port_free(port)
|
|
145
|
+
# The inner native server lives on port + 1. Guard it too so an occupied
|
|
146
|
+
# inner port raises in milliseconds instead of timing out later.
|
|
147
|
+
assert_port_free(port + 1)
|
|
148
|
+
# The inner native server's stderr goes to a file so failure messages can quote
|
|
149
|
+
# its first line. Only an explicitly opened dev log (a File) is reused
|
|
150
|
+
# for that. $stderr reports path "<STDERR>", so it creates a file
|
|
151
|
+
# by that name in the CWD. Anything else falls back to a temp file
|
|
152
|
+
# that Ruby unlinks when the process exits.
|
|
153
|
+
inner_log_path =
|
|
154
|
+
if log.is_a?(File)
|
|
155
|
+
log.path
|
|
156
|
+
else
|
|
157
|
+
inner_log = Tempfile.new("se-inner")
|
|
158
|
+
inner_log.close
|
|
159
|
+
inner_log.path
|
|
160
|
+
end
|
|
161
|
+
inner = spawn_inner(install: install, port: port + 1,
|
|
162
|
+
log_path: inner_log_path)
|
|
163
|
+
# TCPServer.new binds synchronously: construction means ready, so
|
|
164
|
+
# callers need no readiness polling. Closing the socket makes the
|
|
165
|
+
# serving loop's select raise and the loop exit.
|
|
166
|
+
server = TCPServer.new(port)
|
|
167
|
+
inner_died = false
|
|
168
|
+
# Closing the listening socket does not interrupt a syscall already
|
|
169
|
+
# blocked on the old fd, so the serving loop selects with a short
|
|
170
|
+
# timeout: the next IO.select on a closed socket raises IOError, so
|
|
171
|
+
# every shutdown path (trap, inner-death monitor) exits the loop
|
|
172
|
+
# within the timeout, on every platform.
|
|
173
|
+
stop_server = proc do
|
|
174
|
+
server.close
|
|
175
|
+
rescue IOError
|
|
176
|
+
# Already closed by another shutdown path.
|
|
177
|
+
end
|
|
178
|
+
# The monitor reaps the inner process, so nil it in the callback to
|
|
179
|
+
# keep the cleanup idempotent against the already-reaped pid.
|
|
180
|
+
monitor = monitor_inner(inner, on_death: proc do
|
|
181
|
+
inner = nil
|
|
182
|
+
inner_died = true
|
|
183
|
+
stop_server.call
|
|
184
|
+
end)
|
|
185
|
+
trap("INT") { stop_server.call }
|
|
186
|
+
trap("TERM") { stop_server.call }
|
|
187
|
+
(log.respond_to?(:puts) ? log : $stderr).puts "se listening on http://localhost:#{port}"
|
|
188
|
+
loop do
|
|
189
|
+
ready = IO.select([server], nil, nil, 0.5)
|
|
190
|
+
# Select timeout on an idle socket: loop back and select again.
|
|
191
|
+
next if ready.nil?
|
|
192
|
+
client = server.accept
|
|
193
|
+
Thread.new(client) { |c|
|
|
194
|
+
HTTP.handle_client(c, port: port,
|
|
195
|
+
enabled_rules: enabled_rules, info: daemon_info)
|
|
196
|
+
}
|
|
197
|
+
rescue IOError, Errno::EBADF
|
|
198
|
+
# A trap or the inner-death monitor closed the listener. This
|
|
199
|
+
# happens during the select, or between select and accept.
|
|
200
|
+
break
|
|
201
|
+
end
|
|
202
|
+
monitor.kill if monitor.alive?
|
|
203
|
+
raise InnerDied, "lint engine died. Rerun se serve." if inner_died
|
|
204
|
+
ensure
|
|
205
|
+
if inner
|
|
206
|
+
begin
|
|
207
|
+
Process.kill("TERM", inner)
|
|
208
|
+
Process.wait(inner)
|
|
209
|
+
rescue Errno::ESRCH, Errno::ECHILD
|
|
210
|
+
# The monitor thread already reaped the inner process.
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
monitor&.kill if monitor&.alive?
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
# Reaps the inner process and runs on_death (which stops the outer
|
|
217
|
+
# server) whenever it exits, so the daemon dies instead of serving
|
|
218
|
+
# without the lint engine behind it.
|
|
219
|
+
def monitor_inner(pid, on_death:)
|
|
220
|
+
Thread.new do
|
|
221
|
+
Process.wait(pid)
|
|
222
|
+
on_death.call
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# Boots the inner native server and blocks until it answers
|
|
227
|
+
# /v2/check. Raises InnerDied when it dies during startup, and
|
|
228
|
+
# InnerTimeout when it never becomes ready (the child is killed and
|
|
229
|
+
# reaped first, so a failed boot leaks no process). Both messages quote
|
|
230
|
+
# the inner log's first line.
|
|
231
|
+
def spawn_inner(install:, port:, log_path:)
|
|
232
|
+
executable = install.executable!
|
|
233
|
+
begin
|
|
234
|
+
pid = Process.spawn(executable, "--port", port.to_s,
|
|
235
|
+
out: File::NULL, err: log_path)
|
|
236
|
+
rescue Errno::ENOENT
|
|
237
|
+
raise ServerError, "cannot exec #{executable.inspect}."
|
|
238
|
+
end
|
|
239
|
+
ready = false
|
|
240
|
+
died = false
|
|
241
|
+
deadline = Time.now + SimpleEnglish::LanguageTool::TIMEOUT_SECONDS
|
|
242
|
+
until ready || died || Time.now > deadline
|
|
243
|
+
# A process that dies instantly must not be polled for the full timeout.
|
|
244
|
+
# WNOHANG reaps it here, so nothing else may wait on this pid after.
|
|
245
|
+
died = !!Process.wait(pid, Process::WNOHANG)
|
|
246
|
+
break if died
|
|
247
|
+
begin
|
|
248
|
+
Net::HTTP.post_form(URI("http://localhost:#{port}/v2/check"),
|
|
249
|
+
{"language" => "en",
|
|
250
|
+
"enabledRules" => SimpleEnglish::LanguageTool.rule_ids.first,
|
|
251
|
+
"enabledOnly" => "true",
|
|
252
|
+
"text" => "a"})
|
|
253
|
+
ready = true
|
|
254
|
+
rescue SystemCallError
|
|
255
|
+
sleep 0.5
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
if died
|
|
259
|
+
raise InnerDied,
|
|
260
|
+
"lint engine exited during startup. " \
|
|
261
|
+
"First log line: #{first_log_line(log_path)}"
|
|
262
|
+
end
|
|
263
|
+
unless ready
|
|
264
|
+
kill_and_reap(pid)
|
|
265
|
+
raise InnerTimeout,
|
|
266
|
+
"lint engine did not start " \
|
|
267
|
+
"within #{SimpleEnglish::LanguageTool::TIMEOUT_SECONDS} seconds. " \
|
|
268
|
+
"First log line: #{first_log_line(log_path)}"
|
|
269
|
+
end
|
|
270
|
+
pid
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def kill_and_reap(pid)
|
|
274
|
+
Process.kill("TERM", pid)
|
|
275
|
+
Process.wait(pid)
|
|
276
|
+
rescue SystemCallError
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
def first_log_line(path)
|
|
280
|
+
File.read(path).split("\n").first || "unknown"
|
|
281
|
+
rescue SystemCallError
|
|
282
|
+
"unknown"
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
end
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# LanguageTool AnnotatedText for code comments. Builds the annotation
|
|
4
|
+
# JSON and maps LT match offsets back to file line and UTF-16 column.
|
|
5
|
+
# Offsets are UTF-16 code units that count into the concatenation of
|
|
6
|
+
# all text and markup strings. interpretAs does not count. Semantics
|
|
7
|
+
# verified against a live LT 6.6 server on 2026-09-25.
|
|
8
|
+
|
|
9
|
+
require_relative "extractor"
|
|
10
|
+
require_relative "segment"
|
|
11
|
+
require_relative "result"
|
|
12
|
+
|
|
13
|
+
module SimpleEnglish
|
|
14
|
+
module AnnotatedText
|
|
15
|
+
# ASCII comment markers only (MARKER holds the set).
|
|
16
|
+
MARKER = /\A[\/#*;%-]+/
|
|
17
|
+
CLOSER = /\A.*?(\s*\*\/)\z/m
|
|
18
|
+
|
|
19
|
+
module_function
|
|
20
|
+
|
|
21
|
+
def build(source, spans)
|
|
22
|
+
segments = []
|
|
23
|
+
cursor = 0 # byte offset into source
|
|
24
|
+
emit = lambda do |text:, content:, file_char:, interpret_as: nil|
|
|
25
|
+
next if content.empty?
|
|
26
|
+
segments << Segment.new(text: text, content: content, file_char: file_char,
|
|
27
|
+
stream_start: segments.sum { |s| s.content.length }, interpret_as: interpret_as)
|
|
28
|
+
end
|
|
29
|
+
spans = spans.sort_by(&:start_byte)
|
|
30
|
+
spans.each do |span|
|
|
31
|
+
if cursor < span.start_byte
|
|
32
|
+
# Interior gaps break sentences between comments. Gaps with no
|
|
33
|
+
# checkable text before them (leading) change nothing.
|
|
34
|
+
emit.call(text: false, content: source.byteslice(cursor...span.start_byte),
|
|
35
|
+
file_char: char_index(source, cursor),
|
|
36
|
+
interpret_as: (segments.any? { |s| s.text }) ? "\n\n" : nil)
|
|
37
|
+
end
|
|
38
|
+
marker, body, closer = split(span.text)
|
|
39
|
+
file_char = char_index(source, span.start_byte)
|
|
40
|
+
emit.call(text: false, content: marker, file_char: file_char, interpret_as: " ")
|
|
41
|
+
emit.call(text: true, content: body, file_char: file_char + marker.length)
|
|
42
|
+
unless closer.empty?
|
|
43
|
+
emit.call(text: false, content: closer,
|
|
44
|
+
file_char: file_char + marker.length + body.length)
|
|
45
|
+
end
|
|
46
|
+
cursor = span.end_byte
|
|
47
|
+
end
|
|
48
|
+
emit.call(text: false, content: source.byteslice(cursor..),
|
|
49
|
+
file_char: char_index(source, cursor), interpret_as: nil)
|
|
50
|
+
Result.new(source: source, segments: segments,
|
|
51
|
+
stream: segments.map(&:content).join)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def data_json(result)
|
|
55
|
+
require "json"
|
|
56
|
+
JSON.generate("annotation" => result.segments.map do |segment|
|
|
57
|
+
if segment.text
|
|
58
|
+
{"text" => segment.content}
|
|
59
|
+
else
|
|
60
|
+
entry = {"markup" => segment.content}
|
|
61
|
+
entry["interpretAs"] = segment.interpret_as if segment.interpret_as
|
|
62
|
+
entry
|
|
63
|
+
end
|
|
64
|
+
end)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# -> [line, column], both 1-based. Column in UTF-16 code units.
|
|
68
|
+
def locate(result, utf16_offset)
|
|
69
|
+
char = utf16_to_char(result.stream, utf16_offset)
|
|
70
|
+
segment = result.segments.find do |s|
|
|
71
|
+
char >= s.stream_start && char < s.stream_start + s.content.length
|
|
72
|
+
end || result.segments.last
|
|
73
|
+
file_char = segment.file_char + (char - segment.stream_start)
|
|
74
|
+
starts = line_starts(result.source)
|
|
75
|
+
line = starts.rindex { |start| start <= file_char } + 1
|
|
76
|
+
[line, utf16_column(result.source, starts[line - 1], file_char)]
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def split(comment)
|
|
80
|
+
marker = comment[MARKER] || ""
|
|
81
|
+
body = comment[marker.length..] || comment
|
|
82
|
+
closer = body[CLOSER] || ""
|
|
83
|
+
body = body[0, body.length - closer.length] unless closer.empty?
|
|
84
|
+
[marker, body, closer]
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def char_index(source, byte)
|
|
88
|
+
# ponytail: O(filesize) per call. If large repos make this
|
|
89
|
+
# measurable, index byte->char once.
|
|
90
|
+
source.byteslice(0...byte).length
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def line_starts(source)
|
|
94
|
+
starts = [0]
|
|
95
|
+
source.chars.each_with_index { |c, i| starts << i + 1 if c == "\n" }
|
|
96
|
+
starts
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def utf16_to_char(stream, offset)
|
|
100
|
+
units = 0
|
|
101
|
+
stream.each_char.with_index do |char, index|
|
|
102
|
+
return index if units >= offset
|
|
103
|
+
units += (char.ord > 0xFFFF) ? 2 : 1
|
|
104
|
+
end
|
|
105
|
+
stream.length
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def utf16_column(source, from, to)
|
|
109
|
+
units = 1
|
|
110
|
+
source[from...to].each_char { |c| units += (c.ord > 0xFFFF) ? 2 : 1 }
|
|
111
|
+
units
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
private_class_method :split, :char_index, :line_starts,
|
|
115
|
+
:utf16_to_char, :utf16_column
|
|
116
|
+
end
|
|
117
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Counting rules: sentence and paragraph length. Pure text analysis.
|
|
4
|
+
|
|
5
|
+
module SimpleEnglish
|
|
6
|
+
module Counts
|
|
7
|
+
PROCEDURAL_WORD_LIMIT = 20
|
|
8
|
+
DESCRIPTIVE_WORD_LIMIT = 25
|
|
9
|
+
PARAGRAPH_SENTENCE_LIMIT = 6
|
|
10
|
+
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
# Takes Markdown-stripped text, returns findings for long
|
|
14
|
+
# sentences and long paragraphs.
|
|
15
|
+
def check(text)
|
|
16
|
+
Markdown.paragraphs(text).flat_map { |paragraph| findings_for(paragraph) }
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def findings_for(paragraph)
|
|
20
|
+
sentences = Markdown.sentences_of(paragraph.lines.join(" "))
|
|
21
|
+
limit = paragraph.procedural ? PROCEDURAL_WORD_LIMIT : DESCRIPTIVE_WORD_LIMIT
|
|
22
|
+
|
|
23
|
+
findings = sentences
|
|
24
|
+
.select { |sentence| sentence.split.size > limit }
|
|
25
|
+
.map do |_sentence|
|
|
26
|
+
Finding.new(line: paragraph.start_line, column: nil, rule: "SE_SENTENCE_TOO_LONG",
|
|
27
|
+
message: "Sentence has more than #{limit} words. Split it.")
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
if !paragraph.procedural && sentences.size > PARAGRAPH_SENTENCE_LIMIT
|
|
31
|
+
findings << Finding.new(line: paragraph.start_line, column: nil, rule: "SE_PARAGRAPH_TOO_LONG",
|
|
32
|
+
message: "Paragraph has more than #{PARAGRAPH_SENTENCE_LIMIT} " \
|
|
33
|
+
"sentences. Give one topic six sentences at most.")
|
|
34
|
+
end
|
|
35
|
+
findings
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private_class_method :findings_for
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Tree-sitter comment extraction. One walk, no per-language code:
|
|
4
|
+
# every grammar marks comments with a kind ending in "comment".
|
|
5
|
+
|
|
6
|
+
require_relative "span"
|
|
7
|
+
|
|
8
|
+
module SimpleEnglish
|
|
9
|
+
module Extractor
|
|
10
|
+
EXTENSION_LANGUAGES = {
|
|
11
|
+
".py" => "python",
|
|
12
|
+
".rb" => "ruby",
|
|
13
|
+
".js" => "javascript",
|
|
14
|
+
".mjs" => "javascript",
|
|
15
|
+
".ts" => "typescript",
|
|
16
|
+
".yaml" => "yaml",
|
|
17
|
+
".yml" => "yaml",
|
|
18
|
+
".go" => "go",
|
|
19
|
+
".rs" => "rust",
|
|
20
|
+
".java" => "java",
|
|
21
|
+
".sh" => "bash",
|
|
22
|
+
".kt" => "kotlin",
|
|
23
|
+
".cs" => "csharp",
|
|
24
|
+
".cpp" => "cpp",
|
|
25
|
+
".cc" => "cpp",
|
|
26
|
+
".cxx" => "cpp",
|
|
27
|
+
".hpp" => "cpp",
|
|
28
|
+
".h" => "cpp"
|
|
29
|
+
}.freeze
|
|
30
|
+
|
|
31
|
+
module_function
|
|
32
|
+
|
|
33
|
+
# nil for files we do not lint.
|
|
34
|
+
def language_for(path)
|
|
35
|
+
EXTENSION_LANGUAGES[File.extname(path)]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def comment_spans(source, language)
|
|
39
|
+
require "tree_sitter_language_pack"
|
|
40
|
+
root = TreeSitterLanguagePack.get_parser(language).parse(source).root_node
|
|
41
|
+
spans = []
|
|
42
|
+
stack = [root]
|
|
43
|
+
until stack.empty?
|
|
44
|
+
node = stack.pop
|
|
45
|
+
if comment?(node)
|
|
46
|
+
spans << Span.new(text: source.byteslice(node.start_byte...node.end_byte),
|
|
47
|
+
start_byte: node.start_byte, end_byte: node.end_byte)
|
|
48
|
+
end
|
|
49
|
+
(node.child_count - 1).downto(0) { |i| stack.push(node.child(i)) }
|
|
50
|
+
end
|
|
51
|
+
spans.sort_by!(&:start_byte)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def comment?(node)
|
|
55
|
+
node.kind.end_with?("comment") && !node.kind.start_with?("non")
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private_class_method :comment?
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# One lint result. Positions are 1-based. The end position is exclusive.
|
|
4
|
+
# Counting findings have no columns or end positions. Findings from an older
|
|
5
|
+
# daemon can have a start column without an end position.
|
|
6
|
+
|
|
7
|
+
module SimpleEnglish
|
|
8
|
+
Finding = Struct.new(:line, :column, :end_line, :end_column, :rule, :message) do
|
|
9
|
+
# One finding as a text line: `path:line:col-end: [rule] message`.
|
|
10
|
+
# The CLI text output, the MCP tool, and the hook feedback all share
|
|
11
|
+
# this format. Keep them in step by rendering through this method.
|
|
12
|
+
def to_line(path)
|
|
13
|
+
location = "#{path}:#{line}"
|
|
14
|
+
if column
|
|
15
|
+
location += ":#{column}"
|
|
16
|
+
if end_line && end_column
|
|
17
|
+
finish = (end_line == line) ? end_column : "#{end_line}:#{end_column}"
|
|
18
|
+
location += "-#{finish}"
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
"#{location}: [#{rule}] #{message}"
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# One decision, two callers: the CLI (SimpleEnglish.lint_file) and
|
|
4
|
+
# the daemon (Engine.lint) must agree on what kind of lint an input
|
|
5
|
+
# is. The CLI decides to skip comment-free files without a boot.
|
|
6
|
+
# The daemon decides again on its side of the wire. Both call this
|
|
7
|
+
# module, so the two tiers cannot drift.
|
|
8
|
+
|
|
9
|
+
module SimpleEnglish
|
|
10
|
+
module LintPlan
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
# Returns :prose when there is no language (the Markdown
|
|
14
|
+
# pipeline), nil for a comment-free code file (nothing to lint:
|
|
15
|
+
# no boot, no POST), or the comment spans for a code file.
|
|
16
|
+
def call(text, language)
|
|
17
|
+
return :prose unless language
|
|
18
|
+
spans = SimpleEnglish::Extractor.comment_spans(text, language)
|
|
19
|
+
spans.empty? ? nil : spans
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "paragraph"
|
|
4
|
+
|
|
5
|
+
# Markdown stripping and paragraph splitting. Block structure comes
|
|
6
|
+
# from the tree-sitter Markdown grammar shipped with the pack the
|
|
7
|
+
# extractor already uses. Inline fixes stay regular expressions.
|
|
8
|
+
# Stripping keeps the line count identical to the source.
|
|
9
|
+
|
|
10
|
+
module SimpleEnglish
|
|
11
|
+
module Markdown
|
|
12
|
+
# A terminator ends a sentence only before whitespace or at the
|
|
13
|
+
# end of the text, so periods inside URLs and file names do not
|
|
14
|
+
# split sentences.
|
|
15
|
+
SENTENCE_END = /(?<=[.!?])(?:\s+|$)/
|
|
16
|
+
# Non-prose blocks: their lines never reach the counting rules or
|
|
17
|
+
# LanguageTool.
|
|
18
|
+
BLANKED_BLOCKS = %w[fenced_code_block indented_code_block thematic_break].freeze
|
|
19
|
+
UNDERLINE = /\Asetext_h\d_underline\z/
|
|
20
|
+
|
|
21
|
+
module_function
|
|
22
|
+
|
|
23
|
+
# Blank out code and other non-prose blocks and neutralize inline
|
|
24
|
+
# code and heading markers. Line count and UTF-16 widths stay identical
|
|
25
|
+
# to the source so LanguageTool offsets remain source positions.
|
|
26
|
+
def strip(text)
|
|
27
|
+
blank = blank_rows(text).to_h { |row| [row, true] }
|
|
28
|
+
text.lines.each_with_index.map do |line, row|
|
|
29
|
+
ending = if line.end_with?("\r\n")
|
|
30
|
+
"\r\n"
|
|
31
|
+
elsif line.end_with?("\n")
|
|
32
|
+
"\n"
|
|
33
|
+
else
|
|
34
|
+
""
|
|
35
|
+
end
|
|
36
|
+
body = line.delete_suffix(ending)
|
|
37
|
+
(blank[row] ? spaces_for(body) : strip_line(body)) + ending
|
|
38
|
+
end.join
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def strip_line(line)
|
|
42
|
+
# Width-preserving replacements keep LanguageTool positions aligned
|
|
43
|
+
# with the source.
|
|
44
|
+
line
|
|
45
|
+
.gsub(/`[^`]*`/) { |code| "X" * utf16_length(code) }
|
|
46
|
+
.sub(/\A\#{1,6} /) { |marker| " " * marker.length }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def spaces_for(text)
|
|
50
|
+
" " * utf16_length(text)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def utf16_length(text)
|
|
54
|
+
text.each_char.sum { |char| (char.ord > 0xFFFF) ? 2 : 1 }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# A vertical list is not one paragraph: each list item is its own.
|
|
58
|
+
# Paragraph boundaries come from the tree-sitter Markdown grammar,
|
|
59
|
+
# so lazy continuations and interrupted lists match the spec
|
|
60
|
+
# instead of line-shape heuristics.
|
|
61
|
+
def paragraphs(text)
|
|
62
|
+
root = parse(text)
|
|
63
|
+
nodes = []
|
|
64
|
+
each_node(root) { |node| nodes << node if node.kind == "paragraph" }
|
|
65
|
+
nodes.sort_by!(&:start_byte)
|
|
66
|
+
nodes.filter_map do |node|
|
|
67
|
+
body = text.byteslice(node.start_byte, node.end_byte - node.start_byte)
|
|
68
|
+
lines = body.lines.map(&:strip).reject(&:empty?)
|
|
69
|
+
next if lines.empty?
|
|
70
|
+
Paragraph.new(lines: lines, start_line: node.start_position.row + 1,
|
|
71
|
+
procedural: inside_list_item?(node))
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def sentences_of(paragraph_body)
|
|
76
|
+
paragraph_body.split(SENTENCE_END).map(&:strip).reject(&:empty?)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# The rows (0-based) a node fully occupies. Tree-sitter end
|
|
80
|
+
# positions are exclusive. A node ending at column 0 does not
|
|
81
|
+
# cover that row.
|
|
82
|
+
def node_rows(node)
|
|
83
|
+
last = node.end_position.column.zero? ? node.end_position.row - 1 : node.end_position.row
|
|
84
|
+
(node.start_position.row..last)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def blank_rows(text)
|
|
88
|
+
rows = []
|
|
89
|
+
each_node(parse(text)) do |node|
|
|
90
|
+
next if node.kind == "paragraph" # prose, not a block to blank
|
|
91
|
+
rows.concat(node_rows(node).to_a) if
|
|
92
|
+
BLANKED_BLOCKS.include?(node.kind) || UNDERLINE.match?(node.kind)
|
|
93
|
+
end
|
|
94
|
+
rows
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def each_node(root)
|
|
98
|
+
stack = [root]
|
|
99
|
+
until stack.empty?
|
|
100
|
+
node = stack.pop
|
|
101
|
+
yield node
|
|
102
|
+
# No paragraphs live inside code blocks. Skip their subtrees.
|
|
103
|
+
next if BLANKED_BLOCKS.include?(node.kind)
|
|
104
|
+
node.child_count.times { |i| stack.push(node.child(i)) }
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def inside_list_item?(node)
|
|
109
|
+
return false unless node.parent
|
|
110
|
+
node.parent.kind == "list_item" || inside_list_item?(node.parent)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def parse(text)
|
|
114
|
+
require "tree_sitter_language_pack"
|
|
115
|
+
TreeSitterLanguagePack.get_parser("markdown").parse(text).root_node
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
private_class_method :strip_line, :spaces_for, :utf16_length, :node_rows,
|
|
119
|
+
:blank_rows, :each_node, :inside_list_item?, :parse
|
|
120
|
+
end
|
|
121
|
+
end
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SimpleEnglish
|
|
4
|
+
module Markdown
|
|
5
|
+
# A paragraph of prose: its lines, the first line's number in the
|
|
6
|
+
# source, and whether it starts with a list item (procedural text
|
|
7
|
+
# gets a tighter sentence limit).
|
|
8
|
+
Paragraph = Struct.new(:lines, :start_line, :procedural)
|
|
9
|
+
end
|
|
10
|
+
end
|