simple_english 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +61 -39
- data/docs/RULES.md +1 -1
- data/lib/simple_english/cli.rb +53 -15
- data/lib/simple_english/client/daemon.rb +225 -0
- data/lib/simple_english/client/language_tool.rb +95 -0
- data/lib/simple_english/daemon/engine.rb +40 -0
- data/lib/simple_english/{http.rb → daemon/http.rb} +10 -4
- data/lib/simple_english/{server.rb → daemon/server.rb} +113 -7
- data/lib/simple_english/{extractor.rb → lint/extractor.rb} +6 -1
- data/lib/simple_english/lint/finding.rb +9 -0
- data/lib/simple_english/lint/lint_plan.rb +22 -0
- data/lib/simple_english/{markdown.rb → lint/markdown.rb} +28 -10
- data/lib/simple_english/lint/plain_text.rb +30 -0
- data/lib/simple_english/{result.rb → lint/result.rb} +1 -1
- data/lib/simple_english/{config.rb → setup/config.rb} +3 -2
- data/lib/simple_english/setup/fingerprint.rb +38 -0
- data/lib/simple_english/{install.rb → setup/install.rb} +19 -9
- data/lib/simple_english/setup/languagetool.rb +226 -0
- data/lib/simple_english/version.rb +1 -1
- data/lib/simple_english.rb +25 -43
- data/rules/simple-english.xml +5 -1
- metadata +42 -36
- data/lib/simple_english/client.rb +0 -165
- data/lib/simple_english/engine.rb +0 -31
- data/lib/simple_english/finding.rb +0 -8
- data/lib/simple_english/languagetool.rb +0 -143
- data/lib/simple_english/plain_text.rb +0 -18
- /data/lib/simple_english/{annotated_text.rb → lint/annotated_text.rb} +0 -0
- /data/lib/simple_english/{counts.rb → lint/counts.rb} +0 -0
- /data/lib/simple_english/{paragraph.rb → lint/paragraph.rb} +0 -0
- /data/lib/simple_english/{segment.rb → lint/segment.rb} +0 -0
- /data/lib/simple_english/{span.rb → lint/span.rb} +0 -0
- /data/lib/simple_english/{suppressions.rb → lint/suppressions.rb} +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 706801ef7861107694a40cc9671788179c54ea4ac4755491c92df1dd174da41b
|
|
4
|
+
data.tar.gz: 162d45ec0d0c979c855b49f8f639f0d679e5575f4b3e789453f3f01287feda63
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 99b3510be0ae0327886b9cb8be7b4c008eaf8346091bf306423b5e2306c9896eea63d26c629b1a3b96cd2111d306834fedb368e7e1af3702bf5bf61bf3b1443a
|
|
7
|
+
data.tar.gz: 61840d3678edcc0e18a04db4c291935823a3d5aa139609ca5979a1b513c2420097645241aaa602cb5e2b009c31a1cc54da084f92641a0aba2795febefffe39dd
|
data/README.md
CHANGED
|
@@ -11,16 +11,18 @@ the slop it leaves behind. Every finding says what to write
|
|
|
11
11
|
instead.
|
|
12
12
|
|
|
13
13
|
```console
|
|
14
|
-
$ printf
|
|
14
|
+
$ printf "The config was written by setup — don't edit it; the daemon caches rules, making the first lint slow." > note.md
|
|
15
15
|
$ se note.md
|
|
16
|
-
note.md:1:
|
|
17
|
-
note.md:1:
|
|
18
|
-
note.md:1:
|
|
16
|
+
note.md:1:12-26: [SE_ACTIVE_VOICE] "was written by" - Use the active voice. Say who does the action.
|
|
17
|
+
note.md:1:73-81: [SE_ING_AFTER_COMMA] ", making" - Start a new sentence instead of the -ing phrase.
|
|
18
|
+
note.md:1:37-40: [SE_NO_CONTRACTIONS] "n't" - Write the words in full. No contractions.
|
|
19
|
+
note.md:1:33-34: [SE_NO_EMDASH] "—" - Write two sentences, or use a comma.
|
|
20
|
+
note.md:1:48-49: [SE_NO_SEMICOLON] ";" - Write two sentences, or name the relation.
|
|
19
21
|
```
|
|
20
22
|
|
|
21
23
|
Markdown prose plus code comments in Python, Ruby, JavaScript,
|
|
22
|
-
TypeScript, Go, Rust, Java, C#, Kotlin, bash, and YAML. Output as
|
|
23
|
-
text, JSON, or SARIF.
|
|
24
|
+
TypeScript, Go, Rust, Java, C#, C++, Kotlin, bash, and YAML. Output as
|
|
25
|
+
plain text, JSON, or SARIF.
|
|
24
26
|
|
|
25
27
|
## The rules
|
|
26
28
|
|
|
@@ -31,7 +33,7 @@ text, JSON, or SARIF.
|
|
|
31
33
|
- **Contractions:** write every word in full.
|
|
32
34
|
- **Sentence shape:** condition before command, no `-ing` phrase after a comma.
|
|
33
35
|
- **Word choice:** about 50 substitution rules, from `leverage` to `in conclusion`. `make sure that` keeps its "that".
|
|
34
|
-
- **Code comments:** same pattern rules, with line and column.
|
|
36
|
+
- **Code comments:** same pattern rules, with line and column range.
|
|
35
37
|
- **Counts (Markdown only):** 20 words per sentence in list items, 25 in paragraphs, six sentences per paragraph at most.
|
|
36
38
|
|
|
37
39
|
The full list, with a wrong and a right example for each rule:
|
|
@@ -39,10 +41,23 @@ The full list, with a wrong and a right example for each rule:
|
|
|
39
41
|
|
|
40
42
|
## Install
|
|
41
43
|
|
|
42
|
-
**Requirements:**
|
|
44
|
+
**Requirements:** On macOS, Homebrew covers everything. On Linux,
|
|
45
|
+
Windows, or CI, you need Ruby 3.3 or newer and Java 11 or newer for
|
|
43
46
|
LanguageTool. The Docker image bundles both.
|
|
44
47
|
|
|
45
|
-
###
|
|
48
|
+
### Homebrew (macOS)
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
brew install TonyCTHsu/tap/simple-english
|
|
52
|
+
brew services start simple-english
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Homebrew installs Ruby and Java alongside, so the machine needs
|
|
56
|
+
neither. The service keeps a background daemon running. Its first
|
|
57
|
+
start downloads LanguageTool (about 300 MB), and later lints take
|
|
58
|
+
milliseconds. Run `brew services stop simple-english` to stop it.
|
|
59
|
+
|
|
60
|
+
### Ruby gem (Linux, Windows, CI)
|
|
46
61
|
|
|
47
62
|
```bash
|
|
48
63
|
gem install simple_english
|
|
@@ -50,10 +65,7 @@ se setup # run once: downloads LanguageTool, locates Java, verifies both
|
|
|
50
65
|
se README.md
|
|
51
66
|
```
|
|
52
67
|
|
|
53
|
-
|
|
54
|
-
in your shell profile. Pass `--dir PATH` or set `SE_CACHE_DIR` to
|
|
55
|
-
put the cache somewhere else. If `java` is not on PATH, set `SE_JAVA`
|
|
56
|
-
to your java binary.
|
|
68
|
+
If `java` is not on PATH, set `SE_JAVA` to your java binary.
|
|
57
69
|
|
|
58
70
|
### Container
|
|
59
71
|
|
|
@@ -63,13 +75,6 @@ The image holds Ruby, Java, and LanguageTool, so it needs no setup:
|
|
|
63
75
|
docker run -v "$PWD":/work ghcr.io/tonycthsu/simple-english:latest docs/
|
|
64
76
|
```
|
|
65
77
|
|
|
66
|
-
Or keep the daemon in a container and lint through it:
|
|
67
|
-
|
|
68
|
-
```bash
|
|
69
|
-
docker run -d --name se-daemon -p 8181:8181 ghcr.io/tonycthsu/simple-english:latest serve
|
|
70
|
-
SE_SERVER_URL=http://localhost:8181 se lint docs/
|
|
71
|
-
```
|
|
72
|
-
|
|
73
78
|
### Git repository
|
|
74
79
|
|
|
75
80
|
Run `bundle install`, then use `bin/se`.
|
|
@@ -88,12 +93,16 @@ se - < notes.md # stdin (Markdown)
|
|
|
88
93
|
Lint only what changed:
|
|
89
94
|
|
|
90
95
|
```bash
|
|
91
|
-
git diff --name-only --diff-filter=ACM main | xargs se
|
|
96
|
+
git diff --name-only --diff-filter=ACM main | xargs -I{} se {}
|
|
92
97
|
```
|
|
93
98
|
|
|
94
99
|
### Outputs
|
|
95
100
|
|
|
96
|
-
|
|
101
|
+
Pattern findings print an exclusive column range as
|
|
102
|
+
`file:line:start-end: [RULE_ID] message`. A range that crosses lines ends
|
|
103
|
+
with `end-line:end-column`. Columns count UTF-16 code units. Counting findings
|
|
104
|
+
identify only the paragraph's first line. JSON and SARIF output expose the same
|
|
105
|
+
source ranges:
|
|
97
106
|
|
|
98
107
|
```bash
|
|
99
108
|
se --format json docs/
|
|
@@ -108,7 +117,7 @@ se --format sarif src/ > results.sarif
|
|
|
108
117
|
|
|
109
118
|
### CI
|
|
110
119
|
|
|
111
|
-
Gate the
|
|
120
|
+
Gate the prose in the pull request that changes it. The plain run
|
|
112
121
|
fails the build on findings, and the SARIF report puts them inline:
|
|
113
122
|
|
|
114
123
|
```yaml
|
|
@@ -166,24 +175,16 @@ In a code comment:
|
|
|
166
175
|
# Don't touch this constant. se: ignore=SE_NO_CONTRACTIONS
|
|
167
176
|
```
|
|
168
177
|
|
|
169
|
-
## The daemon
|
|
178
|
+
## The background daemon
|
|
170
179
|
|
|
171
|
-
The
|
|
172
|
-
|
|
173
|
-
|
|
180
|
+
The CLI runs a small background server on your machine. The first
|
|
181
|
+
lint starts it, which takes about 15 seconds. Later lints take
|
|
182
|
+
milliseconds.
|
|
174
183
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
Any tool or language can lint through its HTTP API:
|
|
180
|
-
|
|
181
|
-
```bash
|
|
182
|
-
curl -d "text=Don't do this." http://localhost:8181/lint
|
|
183
|
-
# [{"line":1,"column":null,"rule":"SE_NO_CONTRACTIONS","message":"..."}]
|
|
184
|
-
```
|
|
185
|
-
|
|
186
|
-
The full wire format: [docs/DAEMON.md](docs/DAEMON.md).
|
|
184
|
+
The first lint after a gem update prints a warning. Run `se serve --detached` then.
|
|
185
|
+
It stops the old daemon and starts the new one in the
|
|
186
|
+
background. You never talk to the daemon directly. Its HTTP
|
|
187
|
+
interface is internal and can change in any release.
|
|
187
188
|
|
|
188
189
|
## Scope
|
|
189
190
|
|
|
@@ -192,6 +193,27 @@ SimpleEnglish project. This tool does not check ASD-STE100 compliance.
|
|
|
192
193
|
This repo holds no ASD-STE100 text. If you need full compliance, read
|
|
193
194
|
the free standard at <https://www.asd-ste100.org/>.
|
|
194
195
|
|
|
196
|
+
## FAQ
|
|
197
|
+
|
|
198
|
+
### Have you thought about an agent skill?
|
|
199
|
+
|
|
200
|
+
The [SimpleEnglish project](https://github.com/AminBlg/SimpleEnglish)
|
|
201
|
+
ships one. Its skill guides an agent while it writes. This tool does
|
|
202
|
+
the other half of the work. It checks the result against fixed
|
|
203
|
+
rules. Use both: the skill helps the first draft, and the linter
|
|
204
|
+
catches what the agent missed.
|
|
205
|
+
|
|
206
|
+
### Why not Vale?
|
|
207
|
+
|
|
208
|
+
We ported all 67 rules to Vale and ran the corpus on both engines.
|
|
209
|
+
About 60 rules behave the same. Vale has no check for the em-dash
|
|
210
|
+
and semicolon rules: its checks see words, not punctuation. Its
|
|
211
|
+
tagger also mislabels verbs, so the condition-first rule stays
|
|
212
|
+
silent. The rules here are LanguageTool XML with examples that CI
|
|
213
|
+
verifies. Closing the Vale gaps needs scripts or an external
|
|
214
|
+
tagger, and that erases Vale's main advantage: one binary with no
|
|
215
|
+
service behind it.
|
|
216
|
+
|
|
195
217
|
## Develop
|
|
196
218
|
|
|
197
219
|
To change the linter, add rules, or run the tests, read [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md).
|
data/docs/RULES.md
CHANGED
data/lib/simple_english/cli.rb
CHANGED
|
@@ -28,6 +28,10 @@ module SimpleEnglish
|
|
|
28
28
|
start(argv)
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
+
# How each Install.java_source found its java, for `se setup`'s
|
|
32
|
+
# report line.
|
|
33
|
+
JAVA_SOURCES = {env: "SE_JAVA", path: "PATH", homebrew: "Homebrew fallback"}.freeze
|
|
34
|
+
|
|
31
35
|
default_task :lint
|
|
32
36
|
desc "lint FILE_OR_DIR...", "Lint Markdown prose and code comments (- reads stdin)"
|
|
33
37
|
method_option :format, type: :string, default: "text", enum: %w[text json sarif],
|
|
@@ -62,43 +66,62 @@ module SimpleEnglish
|
|
|
62
66
|
results.empty? ? 0 : 1
|
|
63
67
|
end
|
|
64
68
|
|
|
69
|
+
desc "version", "Print the gem version. The container image tag is v plus this."
|
|
70
|
+
def version
|
|
71
|
+
puts SimpleEnglish::VERSION
|
|
72
|
+
0
|
|
73
|
+
end
|
|
74
|
+
|
|
65
75
|
desc "serve", "Run the lint daemon in the foreground"
|
|
66
76
|
method_option :port, type: :numeric, default: SimpleEnglish::Client::DEFAULT_PORT,
|
|
67
77
|
desc: "Port to listen on"
|
|
68
78
|
method_option :"dev-log", type: :string, banner: "PATH",
|
|
69
79
|
desc: "Write daemon stderr to PATH"
|
|
80
|
+
method_option :detached, type: :boolean,
|
|
81
|
+
desc: "Replace any running se daemon and run in the background"
|
|
70
82
|
def serve
|
|
83
|
+
if options[:detached]
|
|
84
|
+
daemon = SimpleEnglish::Server.start_detached(port: options[:port],
|
|
85
|
+
install: SimpleEnglish::Install.from_env, dev_log: options[:"dev-log"])
|
|
86
|
+
if daemon
|
|
87
|
+
puts "se daemon detached, pid #{daemon["pid"]}, " \
|
|
88
|
+
"on http://localhost:#{options[:port]}"
|
|
89
|
+
return 0
|
|
90
|
+
end
|
|
91
|
+
warn "error: detached daemon did not come up. " \
|
|
92
|
+
"Run `se serve` and read its output."
|
|
93
|
+
return 2
|
|
94
|
+
end
|
|
71
95
|
log = options[:"dev-log"] ? File.open(options[:"dev-log"], "w") : $stderr
|
|
72
96
|
SimpleEnglish::Server.start(port: options[:port],
|
|
73
97
|
install: SimpleEnglish::Install.from_env, log: log)
|
|
74
98
|
0
|
|
75
|
-
rescue SimpleEnglish::Install::SetupError, SimpleEnglish::Server::ServerError
|
|
99
|
+
rescue SimpleEnglish::Install::SetupError, SimpleEnglish::Server::ServerError,
|
|
100
|
+
SimpleEnglish::Config::ConfigError => e
|
|
76
101
|
warn "error: #{e.message}"
|
|
77
102
|
2
|
|
78
103
|
end
|
|
79
104
|
|
|
80
105
|
desc "setup", "Download LanguageTool and locate Java. Idempotent."
|
|
81
|
-
method_option :dir, type: :string, banner: "PATH",
|
|
82
|
-
desc: "Install into PATH (default: the shared cache)"
|
|
83
106
|
def setup
|
|
84
107
|
install = SimpleEnglish::Install.from_env
|
|
85
|
-
|
|
86
|
-
return 2 unless
|
|
108
|
+
lt = SimpleEnglish::LanguageTool.install(install.cache_dir)
|
|
109
|
+
return 2 unless lt
|
|
110
|
+
puts "LanguageTool #{SimpleEnglish::LanguageTool::LT_VERSION}: " \
|
|
111
|
+
"#{lt["downloaded"] ? "downloaded to" : "already present at"} #{lt["dir"]}"
|
|
87
112
|
unless install.java?
|
|
88
|
-
warn "error:
|
|
89
|
-
"or set SE_JAVA to your java binary."
|
|
113
|
+
warn "error: #{install.java_message}"
|
|
90
114
|
return 2
|
|
91
115
|
end
|
|
116
|
+
puts "Java: #{install.java} (#{JAVA_SOURCES[install.java_source]})"
|
|
92
117
|
unless SimpleEnglish::LanguageTool.smoke(install)
|
|
93
118
|
warn "error: LanguageTool smoke test failed. The download may be corrupt. " \
|
|
94
|
-
"Delete #{
|
|
119
|
+
"Delete #{lt["dir"]} and rerun `se setup`."
|
|
95
120
|
return 2
|
|
96
121
|
end
|
|
97
|
-
puts
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
se finds it there automatically. Nothing to export.
|
|
101
|
-
SETUP
|
|
122
|
+
puts "Smoke test: passed"
|
|
123
|
+
puts
|
|
124
|
+
puts "se finds them automatically. Nothing to export."
|
|
102
125
|
0
|
|
103
126
|
end
|
|
104
127
|
|
|
@@ -122,6 +145,7 @@ module SimpleEnglish
|
|
|
122
145
|
when "json"
|
|
123
146
|
puts JSON.pretty_generate(results.map do |path, finding|
|
|
124
147
|
{"path" => path, "line" => finding.line, "column" => finding.column,
|
|
148
|
+
"end_line" => finding.end_line, "end_column" => finding.end_column,
|
|
125
149
|
"rule" => finding.rule, "message" => finding.message}
|
|
126
150
|
end)
|
|
127
151
|
when "sarif"
|
|
@@ -129,10 +153,17 @@ module SimpleEnglish
|
|
|
129
153
|
"version" => "2.1.0",
|
|
130
154
|
"$schema" => "https://json.schemastore.org/sarif-2.1.0.json",
|
|
131
155
|
"runs" => [{
|
|
156
|
+
"columnKind" => "utf16CodeUnits",
|
|
132
157
|
"tool" => {"driver" => {"name" => "se"}},
|
|
133
158
|
"results" => results.map do |path, finding|
|
|
134
159
|
region = {"startLine" => finding.line}
|
|
135
|
-
|
|
160
|
+
if finding.column
|
|
161
|
+
region["startColumn"] = finding.column
|
|
162
|
+
if finding.end_line && finding.end_column
|
|
163
|
+
region["endLine"] = finding.end_line
|
|
164
|
+
region["endColumn"] = finding.end_column
|
|
165
|
+
end
|
|
166
|
+
end
|
|
136
167
|
{"ruleId" => finding.rule, "level" => "error",
|
|
137
168
|
"message" => {"text" => finding.message},
|
|
138
169
|
"locations" => [{"physicalLocation" => {
|
|
@@ -145,7 +176,14 @@ module SimpleEnglish
|
|
|
145
176
|
else
|
|
146
177
|
results.each do |path, finding|
|
|
147
178
|
location = "#{path}:#{finding.line}"
|
|
148
|
-
|
|
179
|
+
if finding.column
|
|
180
|
+
location += ":#{finding.column}"
|
|
181
|
+
if finding.end_line && finding.end_column
|
|
182
|
+
finish = (finding.end_line == finding.line) ? finding.end_column :
|
|
183
|
+
"#{finding.end_line}:#{finding.end_column}"
|
|
184
|
+
location += "-#{finish}"
|
|
185
|
+
end
|
|
186
|
+
end
|
|
149
187
|
puts "#{location}: [#{finding.rule}] #{finding.message}"
|
|
150
188
|
end
|
|
151
189
|
end
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# HTTP client for the se daemon: probe, handshake, boot, lint. The
|
|
4
|
+
# LanguageTool wire protocol lives in lib/simple_english/client/language_tool.rb.
|
|
5
|
+
|
|
6
|
+
require "json"
|
|
7
|
+
require "net/http"
|
|
8
|
+
require "rbconfig"
|
|
9
|
+
require "fileutils"
|
|
10
|
+
require "uri"
|
|
11
|
+
|
|
12
|
+
require_relative "../setup/fingerprint"
|
|
13
|
+
require_relative "../version"
|
|
14
|
+
|
|
15
|
+
module SimpleEnglish
|
|
16
|
+
module Client
|
|
17
|
+
module_function
|
|
18
|
+
|
|
19
|
+
DEFAULT_PORT = 8181
|
|
20
|
+
REQUEST_TIMEOUT = 30
|
|
21
|
+
|
|
22
|
+
def url
|
|
23
|
+
ENV.fetch("SE_SERVER_URL") { "http://localhost:#{DEFAULT_PORT}" }
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def post(uri, params, read_timeout: REQUEST_TIMEOUT)
|
|
27
|
+
Net::HTTP.start(uri.host, uri.port, open_timeout: 5,
|
|
28
|
+
read_timeout: read_timeout) do |http|
|
|
29
|
+
request = Net::HTTP::Post.new(uri.request_uri)
|
|
30
|
+
request.set_form_data(params)
|
|
31
|
+
http.request(request)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private_class_method :post
|
|
36
|
+
|
|
37
|
+
# Full lint via the se daemon. Raw Markdown in, or code
|
|
38
|
+
# source with a language for the comment pipeline. nil when
|
|
39
|
+
# unreachable or the response is unusable, so callers can fall
|
|
40
|
+
# back without seeing a stack trace.
|
|
41
|
+
def lint(text, base_url: url, language: nil)
|
|
42
|
+
params = {"text" => text}
|
|
43
|
+
params["language"] = language if language
|
|
44
|
+
response = post(URI("#{base_url}/lint"), params)
|
|
45
|
+
return nil unless response.is_a?(Net::HTTPSuccess)
|
|
46
|
+
body = JSON.parse(response.body)
|
|
47
|
+
return nil unless body.is_a?(Array)
|
|
48
|
+
body.map do |hash|
|
|
49
|
+
Finding.new(line: hash.fetch("line"), column: hash["column"],
|
|
50
|
+
end_line: hash["end_line"], end_column: hash["end_column"],
|
|
51
|
+
rule: hash.fetch("rule"), message: hash.fetch("message"))
|
|
52
|
+
end
|
|
53
|
+
rescue SystemCallError, SocketError, Timeout::Error,
|
|
54
|
+
JSON::ParserError, TypeError
|
|
55
|
+
nil
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# The daemon handshake: GET / answers {version, pid, gem_digest,
|
|
59
|
+
# rules_digest}. Returns the handshake Hash for an se daemon,
|
|
60
|
+
# :foreign for a reachable responder that is not one (an older
|
|
61
|
+
# release, a foreign service) - those still lint, the caller
|
|
62
|
+
# cannot check them - or nil when nothing answers. Unreachable
|
|
63
|
+
# means boot. Foreign means lint but say so.
|
|
64
|
+
def info(base_url: url)
|
|
65
|
+
uri = URI(base_url)
|
|
66
|
+
response = Net::HTTP.start(uri.host, uri.port, open_timeout: 1,
|
|
67
|
+
read_timeout: 2) { |http| http.get("/") }
|
|
68
|
+
return :foreign unless response.is_a?(Net::HTTPSuccess)
|
|
69
|
+
data = JSON.parse(response.body)
|
|
70
|
+
# The whole contract or nothing: a responder with a string pid
|
|
71
|
+
# or a missing field is not a daemon we can reason about.
|
|
72
|
+
return :foreign unless data.is_a?(Hash) &&
|
|
73
|
+
data["version"].is_a?(String) && data["pid"].is_a?(Integer) &&
|
|
74
|
+
data["pid"].positive? && data["gem_digest"].is_a?(String) &&
|
|
75
|
+
data["rules_digest"].is_a?(String)
|
|
76
|
+
data
|
|
77
|
+
rescue Errno::ECONNRESET, Errno::EPIPE, EOFError, Net::ReadTimeout,
|
|
78
|
+
JSON::ParserError, TypeError
|
|
79
|
+
# Something holds the port but answers nothing usable. A read
|
|
80
|
+
# timeout is that too: the request went out, nothing came back.
|
|
81
|
+
:foreign
|
|
82
|
+
rescue Errno::ECONNREFUSED, SocketError, Timeout::Error
|
|
83
|
+
nil
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# True when the daemon answers. Starts it when the port is
|
|
87
|
+
# cold and we own the default URL. A running daemon is never
|
|
88
|
+
# replaced by a lint: `se serve` owns that. A custom
|
|
89
|
+
# SE_SERVER_URL belongs to someone else, so never spawn against
|
|
90
|
+
# it. It returns false when unusable. The caller owns the exit
|
|
91
|
+
# status. Diagnostics go to stderr here, where the cause is
|
|
92
|
+
# known.
|
|
93
|
+
def ensure_up(install: SimpleEnglish::Install.from_env)
|
|
94
|
+
daemon = info
|
|
95
|
+
return boot(install) if daemon.nil?
|
|
96
|
+
check_daemon(daemon)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# An answered probe, classified and said out loud: a foreign
|
|
100
|
+
# responder or a different build. Returns true: there is a daemon
|
|
101
|
+
# to lint against, so the lint proceeds. Every path that holds a
|
|
102
|
+
# handshake result goes through here, so no caller can lint
|
|
103
|
+
# silently.
|
|
104
|
+
def check_daemon(daemon)
|
|
105
|
+
# Reachable but not handshake-capable (an older release, a
|
|
106
|
+
# foreign service): lint against it, there is nothing to check
|
|
107
|
+
# or replace automatically. Say so: a silent stale lint is the
|
|
108
|
+
# bug the handshake exists to catch.
|
|
109
|
+
if daemon == :foreign
|
|
110
|
+
if ENV["SE_SERVER_URL"]
|
|
111
|
+
warn "se: daemon at #{url} is not a handshake-capable se daemon. " \
|
|
112
|
+
"Update it (pull a newer image or rebuild). " \
|
|
113
|
+
"It is not restarted automatically."
|
|
114
|
+
else
|
|
115
|
+
warn "se: daemon at #{url} is not a handshake-capable se daemon. " \
|
|
116
|
+
"Run `se serve` to replace it."
|
|
117
|
+
end
|
|
118
|
+
return true
|
|
119
|
+
end
|
|
120
|
+
return true if daemon["gem_digest"] == Fingerprint.gem
|
|
121
|
+
warn_mismatched_code(daemon)
|
|
122
|
+
true
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# The daemon is unreachable. Start it when we own the default
|
|
126
|
+
# URL, fail fast otherwise.
|
|
127
|
+
def boot(install)
|
|
128
|
+
if ENV["SE_SERVER_URL"]
|
|
129
|
+
warn "error: SE_SERVER_URL is set but #{url} does not answer."
|
|
130
|
+
return false
|
|
131
|
+
end
|
|
132
|
+
# Preflight everything the spawn needs: a missing prerequisite
|
|
133
|
+
# must fail fast with the real blocker named, not spawn a
|
|
134
|
+
# doomed child and wait out 90 s.
|
|
135
|
+
unless File.exist?(install.server_jar) && install.java?
|
|
136
|
+
warn(install.java? ? install.setup_error : install.java_message)
|
|
137
|
+
return false
|
|
138
|
+
end
|
|
139
|
+
# One boot at a time: a concurrent lint that also found the
|
|
140
|
+
# port cold waits here, re-checks, and skips its own spawn.
|
|
141
|
+
# Whatever answers on wake is classified like the first probe:
|
|
142
|
+
# the loser must get the same warnings, never a silent lint
|
|
143
|
+
# against the winner's rules or build.
|
|
144
|
+
with_spawn_lock do
|
|
145
|
+
daemon = info
|
|
146
|
+
unless daemon
|
|
147
|
+
warn "se: daemon not running; starting it (first lint takes ~15s)..."
|
|
148
|
+
spawn_daemon
|
|
149
|
+
daemon = wait_for { info }
|
|
150
|
+
end
|
|
151
|
+
if daemon
|
|
152
|
+
check_daemon(daemon)
|
|
153
|
+
else
|
|
154
|
+
warn "error: se daemon did not come up. Run `se serve` and read its output."
|
|
155
|
+
false
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# The daemon runs different code: older, or another build of
|
|
161
|
+
# this version (a dev checkout beside the installed gem). A
|
|
162
|
+
# lint never stops a daemon: `se serve` owns replacement. Say
|
|
163
|
+
# what runs and how to fix it, then lint against it: a stale
|
|
164
|
+
# lint with a warning beats a refused lint.
|
|
165
|
+
def warn_mismatched_code(daemon)
|
|
166
|
+
if newer_daemon?(daemon)
|
|
167
|
+
# `se serve` from this install boots an older daemon in its
|
|
168
|
+
# place. The fix is updating this gem, not the daemon.
|
|
169
|
+
warn "se: daemon at #{url} runs se #{daemon["version"]}, newer than this install " \
|
|
170
|
+
"(#{VERSION}). Update this gem. Linting against it meanwhile."
|
|
171
|
+
return
|
|
172
|
+
end
|
|
173
|
+
fix = ENV["SE_SERVER_URL"] ? "It is not restarted automatically." :
|
|
174
|
+
"Run `se serve --detached` to restart it."
|
|
175
|
+
warn "se: daemon at #{url} runs se #{daemon["version"]}, different code than this install " \
|
|
176
|
+
"(#{VERSION}). #{fix} Linting against it meanwhile."
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# A per-user lock keyed by the port, independent of the
|
|
180
|
+
# installation cache: daemon ownership is scoped to the port,
|
|
181
|
+
# so two lints with different SE_CACHE_DIR values still
|
|
182
|
+
# coordinate. flock releases when the block ends, even on
|
|
183
|
+
# failure.
|
|
184
|
+
def with_spawn_lock
|
|
185
|
+
dir = File.join(Dir.home, ".cache", "simple_english")
|
|
186
|
+
FileUtils.mkdir_p(dir)
|
|
187
|
+
port = URI(url).port
|
|
188
|
+
File.open(File.join(dir, "spawn-#{port}.lock"), "w") do |lock|
|
|
189
|
+
lock.flock(File::LOCK_EX)
|
|
190
|
+
yield
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Boots `se serve` as a detached child, output discarded: the
|
|
195
|
+
# daemon must outlive the caller. Extra arguments (a port, a
|
|
196
|
+
# dev-log path) are forwarded to the child serve.
|
|
197
|
+
def spawn_daemon(*serve_args)
|
|
198
|
+
bin = File.expand_path("../../../bin/se", __dir__)
|
|
199
|
+
Process.spawn(RbConfig.ruby, bin, "serve", *serve_args,
|
|
200
|
+
out: File::NULL, err: File::NULL)
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def wait_for(seconds: 90)
|
|
204
|
+
deadline = Time.now + seconds
|
|
205
|
+
until Time.now > deadline
|
|
206
|
+
result = yield
|
|
207
|
+
return result if result
|
|
208
|
+
sleep 0.5
|
|
209
|
+
end
|
|
210
|
+
false
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# The handshake reports the daemon's version: a digest mismatch
|
|
214
|
+
# alone cannot tell older from newer. A malformed version counts
|
|
215
|
+
# as not newer, so the warning path decides what to do with it.
|
|
216
|
+
def newer_daemon?(daemon)
|
|
217
|
+
Gem::Version.new(daemon["version"]) > Gem::Version.new(VERSION)
|
|
218
|
+
rescue ArgumentError
|
|
219
|
+
false
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
private_class_method :boot, :check_daemon, :warn_mismatched_code,
|
|
223
|
+
:newer_daemon?, :with_spawn_lock
|
|
224
|
+
end
|
|
225
|
+
end
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# The LanguageTool wire protocol: form posts to /v2/check, and
|
|
4
|
+
# matches mapped back to findings. Runs daemon-side (Engine calls it
|
|
5
|
+
# against the inner JVM) and in tests. The se daemon client - probe,
|
|
6
|
+
# trust, boot - lives in client/daemon.rb.
|
|
7
|
+
#
|
|
8
|
+
# The LanguageTool namespace is shared with setup/languagetool.rb,
|
|
9
|
+
# which holds the pinned-distro facts. This file adds the client
|
|
10
|
+
# that speaks to that distro.
|
|
11
|
+
|
|
12
|
+
require "json"
|
|
13
|
+
require "net/http"
|
|
14
|
+
require "uri"
|
|
15
|
+
|
|
16
|
+
require_relative "../lint/plain_text"
|
|
17
|
+
|
|
18
|
+
module SimpleEnglish
|
|
19
|
+
module LanguageTool
|
|
20
|
+
module Client
|
|
21
|
+
module_function
|
|
22
|
+
|
|
23
|
+
REQUEST_TIMEOUT = 30
|
|
24
|
+
|
|
25
|
+
def post(uri, params, read_timeout: REQUEST_TIMEOUT)
|
|
26
|
+
Net::HTTP.start(uri.host, uri.port, open_timeout: 5,
|
|
27
|
+
read_timeout: read_timeout) do |http|
|
|
28
|
+
request = Net::HTTP::Post.new(uri.request_uri)
|
|
29
|
+
request.set_form_data(params)
|
|
30
|
+
http.request(request)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private_class_method :post
|
|
35
|
+
|
|
36
|
+
# Pattern rules. `payload` is Markdown-stripped text (String) or an
|
|
37
|
+
# AnnotatedText::Result (code comments): one payload interface,
|
|
38
|
+
# #lt_params and #locate, either side of the daemon's LT request.
|
|
39
|
+
# The caller owns the enabled rule IDs (the daemon captures them
|
|
40
|
+
# at boot). There is no default, so no call silently lints with
|
|
41
|
+
# every rule disabled.
|
|
42
|
+
def check(payload, enabled_rules:, base_url:)
|
|
43
|
+
payload = to_payload(payload)
|
|
44
|
+
params = {"language" => "en",
|
|
45
|
+
"enabledRules" => Array(enabled_rules).join(","),
|
|
46
|
+
"enabledOnly" => "true"}.merge(payload.lt_params)
|
|
47
|
+
response = post(URI("#{base_url}/v2/check"), params)
|
|
48
|
+
parse_matches(JSON.parse(response.body).fetch("matches"), payload)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def parse_matches(matches, payload)
|
|
52
|
+
payload = to_payload(payload)
|
|
53
|
+
matches.map do |match|
|
|
54
|
+
offset = match.fetch("offset")
|
|
55
|
+
line, column = payload.locate(offset)
|
|
56
|
+
end_line, end_column = payload.locate(offset + match.fetch("length"))
|
|
57
|
+
Finding.new(line: line, column: column,
|
|
58
|
+
end_line: end_line, end_column: end_column,
|
|
59
|
+
rule: match.fetch("rule").fetch("id"),
|
|
60
|
+
message: with_context(match.fetch("message"), match))
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Prefix the offending text, so a finding says what to change,
|
|
65
|
+
# not only how. Context positions use Java UTF-16 code units.
|
|
66
|
+
def with_context(message, match)
|
|
67
|
+
context = match["context"] or return message
|
|
68
|
+
matched = utf16_slice(context.fetch("text", ""),
|
|
69
|
+
context.fetch("offset", 0).to_i, context.fetch("length", 0).to_i)
|
|
70
|
+
matched.empty? ? message : "\"#{matched}\" - #{message}"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def utf16_slice(text, offset, length)
|
|
74
|
+
first = utf16_index(text, offset)
|
|
75
|
+
last = utf16_index(text, offset + length)
|
|
76
|
+
text[first...last]
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def utf16_index(text, offset)
|
|
80
|
+
units = 0
|
|
81
|
+
text.each_char.with_index do |char, index|
|
|
82
|
+
return index if units >= offset
|
|
83
|
+
units += (char.ord > 0xFFFF) ? 2 : 1
|
|
84
|
+
end
|
|
85
|
+
text.length
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def to_payload(payload)
|
|
89
|
+
payload.is_a?(String) ? PlainText.new(payload) : payload
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
private_class_method :to_payload, :with_context, :utf16_slice, :utf16_index
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|