luckiest-co 1.0.62 → 1.0.64

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2388 @@
1
+ #!/usr/bin/env python3
2
+ """Deterministic linter for the structural STE rules in SKILL.md.
3
+
4
+ Checks only rules verifiable without ASD's dictionary. Deliberately never
5
+ flags hedges or modality (may/might/could): the skill treats confidence as
6
+ content, and a linter that pressures hedges out would rewrite claims.
7
+
8
+ Usage:
9
+ ste-lint.py FILE [FILE ...]
10
+ echo "text" | ste-lint.py [--json]
11
+ ste-lint.py --baseline 5 FILE # pass unless hard violations exceed 5
12
+ ste-lint.py --disable passive-voice,present-perfect FILE
13
+ ste-lint.py --selftest
14
+
15
+ Exit 1 when hard ("advisory-free") violations exceed the baseline (default 0).
16
+ Advisory findings (passive voice, compound tenses) never fail the run.
17
+ Exit 2 on a usage error (unknown flag, bad value, unknown rule, unreadable file).
18
+ """
19
+ import argparse
20
+ import bisect
21
+ import contextlib
22
+ import html.entities
23
+ import io
24
+ import json
25
+ import os
26
+ import re
27
+ import sys
28
+ import tempfile
29
+
30
+ # ponytail: regex heuristics, not a parser. No noun-cluster rule — needs POS
31
+ # tagging to avoid constant false positives; add spaCy-backed rule if ever needed.
32
+ # No ellipsis rule by owner's choice: technical writing sometimes earns one.
33
+ # Irregular past participles used by the present-perfect heuristic.
34
+ IRREGULAR_PARTICIPLES = "given|taken|made|done|found|seen|known|shown|written|built|sent|set|run|read|kept|held|left|put|cut|hit|let|shut|split|spread|begun|become|come|gone|got|gotten|lost|met|paid|said|sold|told|thought|brought|bought|caught|taught|won|worn|torn|born|drawn|grown|thrown|flown|driven|risen|chosen|broken|spoken|frozen|hidden|ridden|forgotten|fallen|eaten|beaten|understood|stood|struck|stuck|swung|hung|led|fed|bled|fled|sped|bound|wound|dug|spun|slid|bit|lit|quit"
35
+
36
+ # Preserve the original passive heuristic; perfects also include intransitive verbs.
37
+ PASSIVE_PARTICIPLES = "given|taken|made|done|found|seen|known|shown|written|built|sent|set|run|read|kept|held|left|put"
38
+ # Up to two words may stand between the modal and "have": "may not",
39
+ # "might already", "could not possibly", "may very well".
40
+ MODAL_PERFECT_PREFIX = re.compile(
41
+ r"\b(?:(?:may|might|could|should|would|must|can|will|shall)(?:n['’]t)?"
42
+ r"|cannot|can['’]t|won['’]t|shan['’]t)"
43
+ r"(?:\s+(?:not|already|also|ever|even|just|never|still|very|well|\w+ly)){0,2}\s+$",
44
+ re.I,
45
+ )
46
+
47
+ RULES = [
48
+ ("semicolon", "advisory-free",
49
+ re.compile(r";"),
50
+ "STE bans the semicolon (Rule 8.1). Split into separate sentences."),
51
+ ("phrasal-verb", "advisory-free",
52
+ re.compile(r"\b(spin(?:ning|s)? up|spun up|reach(?:ing|es|ed)? out|div(?:e|es|ing|ed) into|dove into|kick(?:ing|s|ed)? off|circl(?:e|es|ing|ed) back|touch(?:ing|es|ed)? base)\b", re.I),
53
+ "Soft phrasal verb. Use the single plain verb (start, contact, read, begin)."),
54
+ ("marketing-adjective", "advisory-free",
55
+ re.compile(r"\b(seamless(?:ly)?|robust(?:ly)?|cutting-edge|effortless(?:ly)?|blazing[- ]fast|world-class|state-of-the-art|game-chang(?:ing|er))\b", re.I),
56
+ "Marketing adjective. Delete, or replace with the measurement that earns the claim."),
57
+ ("nominalization", "advisory-free",
58
+ re.compile(r"\b(perform|performs|performed|conduct|conducts|conducted|carry out|carries out|carried out)\s+(?:a|an|the)\s+\w+(?:tion|sion|ment|ance|ence|ysis)\b", re.I),
59
+ "Action frozen into a noun. Use the verb (analyze, not perform an analysis of)."),
60
+ ("passive-voice", "advisory",
61
+ re.compile(r"\b(is|are|was|were|been|being)\s+(\w+ed|" + PASSIVE_PARTICIPLES + r")\b(?!\s+(?:to|for|by)\s+\w+ing)", re.I),
62
+ "Possible passive voice. Name the actor and use an active verb, unless the actor is unknown or irrelevant."),
63
+ ("present-perfect", "advisory",
64
+ # modal + perfect infinitive ("may have failed") is a protected hedge, not present perfect
65
+ re.compile(r"(?<!\bmay )(?<!\bmight )(?<!\bcould )(?<!\bshould )(?<!\bwould )(?<!\bmust )\b(has|have|had)\s+(?:been\s+)?(?:\w+(?:ed|en)|" + IRREGULAR_PARTICIPLES + r")\b", re.I),
66
+ "Compound tense. Use simple past/present unless current relevance is the point (then keep and flag)."),
67
+ ]
68
+
69
+ # One word, one meaning: groups of verbs commonly rotated for the same action.
70
+ # Only pairs where the members are genuinely interchangeable — error/fault/failure
71
+ # are distinct concepts and stay out.
72
+ SYNONYM_GROUPS = [
73
+ ("check", "verify", "confirm", "validate"),
74
+ ("delete", "remove", "erase"),
75
+ ("start", "launch", "begin", "initiate"),
76
+ ("stop", "halt", "terminate"),
77
+ ("show", "display"),
78
+ ("use", "utilize", "employ"),
79
+ ("fix", "repair", "correct"),
80
+ ("send", "transmit"),
81
+ ("get", "retrieve", "fetch", "obtain"),
82
+ ("change", "modify", "alter"),
83
+ ]
84
+
85
+ MAX_WORDS = 25 # descriptions cap; instructions cap is 20 but undetectable without context
86
+
87
+ CODE_FENCE = re.compile(r"^(`{3,}|~{3,})")
88
+ INLINE_CODE = re.compile(r"`[^`]*`")
89
+ BACKTICK_RUN = re.compile(r"`+")
90
+ # A list item start at any indent, to find the content indent of its item.
91
+ LIST_ITEM_INDENT = re.compile(r"^( *)([-*+]|[0-9]{1,9}[.)])( +|$)")
92
+ # Markup that is not prose: character references (&nbsp; &#39; &#x2192;) and
93
+ # autolinks (<https://...>). Inline link destinations ("](url") are found by
94
+ # _link_destination_end. A reference is markup only when GitHub reads it as one.
95
+ CHAR_REF = re.compile(r"&(?:([A-Za-z][A-Za-z0-9]{1,31})|#([0-9]{1,7})|#[xX]([0-9A-Fa-f]{1,6}));")
96
+ LINK_OPEN = re.compile(r"\]\(")
97
+ BRACKET = re.compile(r"[\[\]]")
98
+ # A list item or a fence that can interrupt a paragraph: a bullet or "1." with text.
99
+ PARAGRAPH_INTERRUPT = re.compile(r" {0,3}(?:[-+*][ \t]+\S|1[.)][ \t]+\S|`{3,}|~{3,})")
100
+ AUTOLINK = re.compile(r"<[A-Za-z][A-Za-z0-9+.-]{1,31}:[^<>\s]*>")
101
+ # An inline HTML open or closing tag on one line (CommonMark 6.6).
102
+ INLINE_TAG = re.compile(
103
+ r"<(?:(?P<name>[A-Za-z][A-Za-z0-9-]*)(?:[ \t]+[A-Za-z_:][A-Za-z0-9_.:-]*"
104
+ r"(?:[ \t]*=[ \t]*(?:[^\s\"'=<>`]+|'[^']*'|\"[^\"]*\"))?)*[ \t]*/?"
105
+ r"|/[A-Za-z][A-Za-z0-9-]*[ \t]*)>"
106
+ )
107
+ # Other inline HTML (CommonMark 6.6): a comment, a processing instruction, a
108
+ # declaration, a CDATA section. Each pair is (opener, closer).
109
+ INLINE_HTML_OTHER = ((re.compile(r"<!--"), "-->"), (re.compile(r"<\?"), "?>"),
110
+ (re.compile(r"<![A-Za-z]"), ">"), (re.compile(r"<!\[CDATA\["), "]]>"))
111
+ # GitHub shows these tags as text (GFM 6.11, the tagfilter extension).
112
+ FILTERED_TAGS = {"title", "textarea", "style", "xmp", "iframe", "noembed", "noframes", "script", "plaintext"}
113
+ ATTRIBUTE_VALUE = re.compile(r"=[ \t]*(?:\"[^\"]*\"|'[^']*'|[^\s\"'=<>`]+)")
114
+ # CommonMark 6.3 lets an implementation limit the nesting of parentheses in a
115
+ # link destination. The limit keeps the scan of a long line linear.
116
+ MAX_LINK_PARENS = 32
117
+ ASCII_PUNCTUATION = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
118
+ LIST_ITEM_START = re.compile(
119
+ r"^(?P<indent> {0,3})(?P<marker>[-*+]|[0-9]+[.)])(?P<gap> +)(?P<body>.*)$"
120
+ )
121
+ CONJUNCTION_END = re.compile(r"\b(?:and|or)\s*$", re.I)
122
+ TABLE_SEPARATOR_CELL = re.compile(r"^:?-+:?$")
123
+ # Headings, thematic breaks, front-matter delimiters, and HTML lines stand alone.
124
+ STANDALONE_LINE = re.compile(r"^ {0,3}(?:#|<|(?:[-*_=] *){3,}$)")
125
+ BLOCKQUOTE_PREFIX = re.compile(r"^ {0,3}(?:> ?)+")
126
+ # A GitHub alert marker, alone on the first line of a quote: "> [!NOTE]".
127
+ GITHUB_ALERT = re.compile(r"\[!(?:note|tip|important|warning|caution)\]", re.I)
128
+ # MkDocs admonition title lines. Four spaces allow a title inside a list item.
129
+ # Only ??? admonitions support the collapsible "+" marker.
130
+ ADMONITION_TITLE = re.compile(
131
+ r'^ {0,4}(?:!!!|\?\?\?\+?)[ \t]*[\w-]+'
132
+ r'(?:[ \t]+[\w-]+)*(?:[ \t]+".*")?[ \t]*$'
133
+ )
134
+ # Track ordinary block containers, not inline or void tags such as span or hr.
135
+ HTML_BLOCK_TAG = re.compile(
136
+ r"<(/?)(address|article|aside|blockquote|body|caption|center|dd|details|"
137
+ r"dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|h[1-6]|"
138
+ r"header|hgroup|html|iframe|legend|li|main|menu|nav|noframes|ol|"
139
+ r"optgroup|option|p|pre|script|search|section|style|summary|table|"
140
+ r"tbody|td|textarea|tfoot|th|thead|title|tr|ul)(?=[\s/>])[^>]*>", re.I,
141
+ )
142
+ # Any list marker at any depth starts a new prose block.
143
+ ANY_LIST_ITEM = re.compile(r"^\s*(?:[-*+]|[0-9]+[.)])(?:\s|$)")
144
+ # A letter or a digit: a token without one is punctuation, not a word.
145
+ WORD_CHAR = re.compile(r"[^\W_]")
146
+ # A token that ends in terminal punctuation, optionally followed by closing
147
+ # quotes, brackets, or Markdown emphasis markers ("**Lead.**").
148
+ SENTENCE_END = re.compile(r"[.!?][\"'”’)\]*_]*$")
149
+ OPENERS = "\"'“‘([*_"
150
+ # Abbreviations whose period never ends a sentence.
151
+ ABBREVIATIONS = {"vs.", "cf.", "approx.", "mr.", "mrs.", "ms.", "dr.", "fig.", "no."}
152
+ # Dotted initialisms such as "e.g." and "U.S." end a sentence only when the
153
+ # next word starts with a capital letter.
154
+ DOTTED_ABBREVIATION = re.compile(r"(?:[a-z]\.){2,}", re.I)
155
+
156
+
157
+ def _word_re(base):
158
+ return re.compile(r"\b" + base + r"(?:s|es|ed|d|ing)?\b", re.I)
159
+
160
+
161
+ def _count_words(text):
162
+ """Count whitespace-separated tokens that contain a letter or a digit.
163
+
164
+ Bare punctuation is not a word: a comma left behind by a masked code
165
+ span, or a spaced em dash, does not make a sentence longer.
166
+ """
167
+ return sum(1 for token in text.split() if WORD_CHAR.search(token))
168
+
169
+
170
+ def _mask_inline_code(text):
171
+ """Blank code spans with equal-length spaces so columns stay true."""
172
+ return INLINE_CODE.sub(lambda match: " " * len(match.group(0)), text)
173
+
174
+
175
+ def _list_item(rest, list_indent, in_paragraph):
176
+ """Return the match of the list item that starts REST, or None (CommonMark 5.2).
177
+
178
+ REST is a line after its quote prefix. A thematic break such as "* * *" is no
179
+ item. IN_PARAGRAPH: the line would otherwise continue a paragraph in the same
180
+ quote. Then an item to the left of LIST_INDENT is the next item of the open
181
+ list, and a new list starts only with a bullet or "1" and some text, and not
182
+ as indented code.
183
+ """
184
+ if NO_PARAGRAPH.match(rest):
185
+ return None
186
+ item = LIST_ITEM_INDENT.match(rest)
187
+ if not item or not in_paragraph or len(item.group(1)) < list_indent:
188
+ return item
189
+ marker = item.group(2)
190
+ if (len(item.group(1)) < list_indent + 4 and rest[item.end():].strip()
191
+ and (marker in "-*+" or int(marker[:-1]) == 1)):
192
+ return item
193
+ return None
194
+
195
+
196
+ def _breaks_paragraph(depth, line, list_indent):
197
+ """True when LINE ends a paragraph at quote DEPTH, as the main loop ends it.
198
+
199
+ These lines end it (CommonMark 4-5): a blank or quote-only line, a deeper quote,
200
+ a heading, a thematic break, a setext underline in the open item, a fence that is
201
+ no indented code, an HTML block of types 1 to 6, and a list item that can
202
+ interrupt a paragraph. A text line with fewer ">" is a lazy line and continues
203
+ the paragraph, but any list item there ends it. A code span ends there too.
204
+ """
205
+ line_depth, rest = _strip_quotes(line.expandtabs(4))
206
+ in_item = _indent_width(rest) >= list_indent
207
+ return (not rest.strip() or line_depth > depth
208
+ or bool(NO_PARAGRAPH.match(rest)
209
+ or (_fence_step(None, line)[0] and _indent_width(rest) < list_indent + 4)
210
+ or (line_depth == depth and in_item and SETEXT_UNDERLINE.match(rest))
211
+ or any(start.match(rest) for start, _ in HTML_BLOCK_KINDS)
212
+ or _list_item(rest, list_indent, line_depth == depth)))
213
+
214
+
215
+ def _paragraph_runs(lines, start, depth, list_indent):
216
+ """Return (end, last) for the paragraph that continues at LINES[START].
217
+
218
+ END is the index of the line that ends it. LAST maps each backtick run length
219
+ to the index of its last line before END. One scan serves every run in the
220
+ paragraph, so the cost stays linear in the line count.
221
+ """
222
+ last, end = {}, start
223
+ while end < len(lines) and not _breaks_paragraph(depth, lines[end], list_indent):
224
+ for run in BACKTICK_RUN.finditer(lines[end]):
225
+ last[len(run.group())] = end
226
+ end += 1
227
+ return end, last
228
+
229
+
230
+ def _mask_code_spans(line, open_run, closes_later, lines=(), next_index=0, depth=0, skip=0, carry=None,
231
+ brackets=None):
232
+ """Blank the code spans in LINE, also a span that wraps across lines.
233
+
234
+ OPEN_RUN is the backtick run length of a span still open from an earlier
235
+ line, or 0. LINES[NEXT_INDEX:] are the lines after LINE. As in CommonMark 6.1, a
236
+ span ends at the next run of the same length, and a run that no later run
237
+ in the paragraph closes is literal text. Outside code spans, fill inline
238
+ link destinations with underscores, blank link titles, and fill the
239
+ inline HTML that GitHub does not show. What starts first wins: a code
240
+ span that opens before "](" hides the link, and a backtick in a link, an
241
+ autolink or inline HTML is text.
242
+ DEPTH is the quote depth of the paragraph that LINE is in, or None when no
243
+ paragraph goes on after LINE, so a link must close in it. SKIP is where a
244
+ link from an earlier line ends in LINE. When a link goes on in a later
245
+ line, CARRY gets [lines ahead, column after its ")"]. BRACKETS carries the
246
+ open "[" marks from line to line of a paragraph. Return (masked line, open
247
+ run).
248
+ """
249
+ masked = list(line)
250
+ start = 0 if open_run else None
251
+ runs = list(BACKTICK_RUN.finditer(line))
252
+ last = {len(run.group()): i for i, run in enumerate(runs)} # last run of each length
253
+ opens = [m.end() for m in LINK_OPEN.finditer(line) if not _escaped(line, m.start())]
254
+ # Autolinks and inline HTML tags: a code span or a link in one is text.
255
+ # An item is (start, end, the parts of it that GitHub does not show).
256
+ atoms = [(m.start(), m.end(), ()) for m in AUTOLINK.finditer(line)]
257
+ for m in INLINE_TAG.finditer(line):
258
+ # GitHub shows no attribute value. A bare word stays, as in the
259
+ # placeholder "<one of the two>".
260
+ hide = ()
261
+ if m.group("name") and m.group("name").lower() not in FILTERED_TAGS:
262
+ hide = [value.span() for value in ATTRIBUTE_VALUE.finditer(line, m.end("name"), m.end())]
263
+ atoms.append((m.start(), m.end(), hide))
264
+ atoms += _inline_html_other(line)
265
+ atoms.sort(key=lambda atom: atom[:2])
266
+ atoms = [atom for atom in atoms if not _escaped(line, atom[0])]
267
+ marks = [m.start() for m in BRACKET.finditer(line) if not _escaped(line, m.start())]
268
+ if brackets is None:
269
+ brackets = [[], 0]
270
+ # BRACKETS[0] holds the "[" marks of the paragraph that no "]" closed yet,
271
+ # True for an image "![". A "[" below BRACKETS[1] is in a link, so it
272
+ # opens no link (CommonMark 6.3); an image "![" can still open an image.
273
+ stack = brackets[0]
274
+ a = 0
275
+ b = 0
276
+ k = 0
277
+ done = skip # the end of the last code span or link
278
+ _blank_title(masked, line, 0, skip) # a link title from an earlier line
279
+
280
+ def pop_bracket():
281
+ image = stack.pop()
282
+ active = image or len(stack) >= brackets[1]
283
+ brackets[1] = min(brackets[1], len(stack))
284
+ return image, active
285
+
286
+ def take_brackets(limit):
287
+ # Read the "[" and "]" marks before LIMIT, outside code, links and tags.
288
+ nonlocal b
289
+ while b < len(marks) and marks[b] < limit:
290
+ pos = marks[b]
291
+ b += 1
292
+ if pos < done:
293
+ continue
294
+ if line[pos] == "[":
295
+ stack.append(pos > 0 and line[pos - 1] == "!" and not _escaped(line, pos - 1))
296
+ elif line[pos + 1:pos + 2] != "(" and stack:
297
+ pop_bracket() # mask_links reads a "](" mark
298
+
299
+ def take_atoms(limit):
300
+ # Let the autolinks and tags that start before LIMIT start first.
301
+ nonlocal a, done
302
+ while a < len(atoms) and atoms[a][0] < limit:
303
+ begin, end, hide = atoms[a]
304
+ a += 1
305
+ if begin < done:
306
+ continue
307
+ take_brackets(begin)
308
+ done = end
309
+ # Keep the spaces, so the token count stays.
310
+ for left, right in hide:
311
+ masked[left:right] = [c if c.isspace() else "_" for c in line[left:right]]
312
+
313
+ def mask_links(limit):
314
+ # Fill the destinations after the "](" marks before LIMIT.
315
+ nonlocal k, done
316
+ while k < len(opens) and opens[k] <= limit:
317
+ pos = opens[k]
318
+ k += 1
319
+ take_atoms(pos - 2)
320
+ take_brackets(pos - 2)
321
+ if pos - 2 < done or not stack:
322
+ continue # no "[" opens link text, so "](" is text
323
+ image, active = pop_bracket()
324
+ if not active:
325
+ continue
326
+ later = (lines[index] for index in range(next_index, len(lines)))
327
+ link = _link_destination_end(line, pos, later, depth)
328
+ if link is None or (link[0] == pos and line[pos:link[1]].strip() != ")"):
329
+ continue # with no destination, a "title" is the destination: a gap this reads as no link
330
+ if not image:
331
+ brackets[1] = len(stack) # a link holds no link
332
+ masked[pos:link[0]] = "_" * (link[0] - pos)
333
+ _blank_title(masked, line, link[0], link[1])
334
+ done = link[1]
335
+ if carry is not None and link[2]:
336
+ carry[:] = link[2]
337
+
338
+ for i, run in enumerate(runs):
339
+ n = len(run.group())
340
+ if start is not None:
341
+ if n == open_run:
342
+ masked[start:run.end()] = " " * (run.end() - start)
343
+ start, open_run = None, 0
344
+ done = run.end()
345
+ continue
346
+ mask_links(run.start())
347
+ take_atoms(run.start())
348
+ if run.start() < done:
349
+ continue # a backtick in a link, an autolink or a tag is text
350
+ first = run.start()
351
+ if _escaped(line, first):
352
+ first, n = first + 1, n - 1 # an escaped backtick is text
353
+ if n and (last.get(n, -1) > i or closes_later(n)):
354
+ take_brackets(first)
355
+ start, open_run = first, n
356
+ if start is not None:
357
+ masked[start:] = " " * (len(line) - start)
358
+ else:
359
+ mask_links(len(line))
360
+ take_atoms(len(line))
361
+ take_brackets(len(line)) # a "[" that a later line can close
362
+ return "".join(masked), open_run
363
+
364
+
365
+ def _inline_html_other(line):
366
+ """Return (start, end, hidden parts) for each comment, processing
367
+ instruction, declaration and CDATA section that starts and ends in LINE.
368
+
369
+ GitHub shows none of a comment or a declaration. Its HTML sanitizer reads
370
+ a processing instruction and CDATA as a comment that ends at the first ">",
371
+ so it shows the text after that ">".
372
+ """
373
+ found = []
374
+ for opener, closer in INLINE_HTML_OTHER:
375
+ for m in opener.finditer(line):
376
+ if closer == "-->" and line.startswith(("<!-->", "<!--->"), m.start()):
377
+ end = line.index(">", m.end()) + 1
378
+ found.append((m.start(), end, [(m.start(), end)]))
379
+ continue
380
+ end = line.find(closer, m.end())
381
+ if end < 0:
382
+ break # no later opener closes either
383
+ end += len(closer)
384
+ shown = end if closer in ("-->", ">") else line.index(">", m.start()) + 1
385
+ found.append((m.start(), end, [(m.start(), shown)]))
386
+ return found
387
+
388
+
389
+ def _escaped(line, pos):
390
+ """True when an odd number of backslashes comes right before LINE[POS]."""
391
+ start = pos
392
+ while start and line[start - 1] == "\\":
393
+ start -= 1
394
+ return (pos - start) % 2 == 1
395
+
396
+
397
+ def _blank_title(masked, line, start, end):
398
+ """Blank the link title in MASKED[START:END] with spaces, but keep its ")".
399
+
400
+ GitHub shows a title in no text, so it is not prose. The ")" stays, so the
401
+ link stays one token for the word count, as it is when it has no title.
402
+ """
403
+ if start < end:
404
+ masked[start:end] = " " * (end - start)
405
+ if line[end - 1] == ")":
406
+ masked[end - 1] = ")"
407
+
408
+
409
+ def _char_ref_is_markup(match):
410
+ """True when the character reference MATCH renders as one character.
411
+
412
+ A named reference counts only when its name is in the HTML5 list. A
413
+ reference that renders as a semicolon is prose: GitHub shows a semicolon.
414
+ """
415
+ name, decimal, hexadecimal = match.groups()
416
+ if name is not None:
417
+ return name + ";" in html.entities.html5 and name != "semi"
418
+ value = int(decimal) if decimal is not None else int(hexadecimal, 16)
419
+ return value != ord(";")
420
+
421
+
422
+ def _paragraph_lines(later_lines, depth):
423
+ """Yield (text, column, lines ahead) for the lines that continue the paragraph.
424
+
425
+ TEXT is the line without its quote prefix, which ends at COLUMN.
426
+
427
+ DEPTH is the quote depth of the paragraph, or None when no paragraph goes
428
+ on. A blank line, a deeper quote and a line that starts a block end the
429
+ paragraph (CommonMark 4.1 to 4.6, 5.1 and 5.2). A line with fewer ">" is a
430
+ lazy line and continues it.
431
+ """
432
+ if depth is None:
433
+ return
434
+ for ahead, line in enumerate(later_lines, 1):
435
+ levels, rest = _strip_quotes(line)
436
+ if (levels > depth or not rest.strip() or NO_PARAGRAPH.match(rest)
437
+ or SETEXT_UNDERLINE.match(rest) or PARAGRAPH_INTERRUPT.match(rest)
438
+ or any(start.match(rest) for start, _ in HTML_BLOCK_KINDS)):
439
+ return
440
+ yield rest, len(line) - len(rest), ahead
441
+
442
+
443
+ def _skip_link_gap(item, pos, rest):
444
+ """Skip spaces and tabs from POS, with at most one line ending.
445
+
446
+ ITEM is (text, column, lines ahead), and REST yields the next items of the
447
+ paragraph. Return (item, pos, crossed), or None when the paragraph ends.
448
+ """
449
+ while pos < len(item[0]) and item[0][pos] in " \t":
450
+ pos += 1
451
+ if pos < len(item[0]):
452
+ return item, pos, False
453
+ item = next(rest, None)
454
+ if item is None:
455
+ return None
456
+ pos = 0
457
+ while pos < len(item[0]) and item[0][pos] in " \t":
458
+ pos += 1
459
+ return item, pos, True
460
+
461
+
462
+ def _skip_link_title(item, pos, rest):
463
+ """Return (item, pos) after a link title that starts at POS, or None.
464
+
465
+ ITEM is (text, column, lines ahead). The title can go on in the next lines
466
+ of the paragraph (REST), but it must close before the paragraph ends.
467
+ """
468
+ line = item[0]
469
+ closer = {'"': '"', "'": "'", "(": ")"}[line[pos]]
470
+ pos += 1
471
+ while True:
472
+ while pos < len(line):
473
+ if line[pos] == "\\" and pos + 1 < len(line) and line[pos + 1] in ASCII_PUNCTUATION:
474
+ pos += 2
475
+ continue
476
+ if line[pos] == closer:
477
+ return item, pos + 1
478
+ if closer == ")" and line[pos] == "(":
479
+ return None
480
+ pos += 1
481
+ item, pos = next(rest, None), 0
482
+ if item is None:
483
+ return None
484
+ line = item[0]
485
+
486
+
487
+ def _link_destination_end(text, start, later_lines=(), depth=None):
488
+ """Return (destination end, link end, carry) for the destination at START, or None.
489
+
490
+ START is after "](". CommonMark 6.3: the destination is in <...>, or it has
491
+ no space or control character and its parentheses are balanced. An
492
+ optional title and ")" must follow, in the same paragraph. LATER_LINES are
493
+ the lines after TEXT, and DEPTH is the quote depth of the paragraph (see
494
+ _paragraph_lines). Each gap of spaces and tabs may hold one line ending,
495
+ and the title may span lines. The link end is after ")" when the link
496
+ closes in TEXT, else the end of TEXT, and CARRY is then (lines ahead,
497
+ column after ")") of the line where it closes, else None.
498
+ """
499
+ pos = start
500
+ if pos < len(text) and text[pos] == "<":
501
+ pos += 1
502
+ while pos < len(text) and text[pos] not in "<>":
503
+ pos += 2 if text[pos] == "\\" and pos + 1 < len(text) else 1
504
+ if pos >= len(text) or text[pos] != ">":
505
+ return None
506
+ end = pos + 1
507
+ else:
508
+ parens = 0
509
+ while pos < len(text):
510
+ char = text[pos]
511
+ if char == "\\" and pos + 1 < len(text) and text[pos + 1] in ASCII_PUNCTUATION:
512
+ pos += 2
513
+ continue
514
+ if char <= " " or char == "\x7f":
515
+ break
516
+ if char == "(":
517
+ parens += 1
518
+ if parens > MAX_LINK_PARENS:
519
+ return None
520
+ elif char == ")":
521
+ if parens == 0:
522
+ break
523
+ parens -= 1
524
+ pos += 1
525
+ if parens:
526
+ return None
527
+ end = pos
528
+ rest = _paragraph_lines(later_lines, depth)
529
+ gap = _skip_link_gap((text, 0, 0), end, rest)
530
+ if gap is None:
531
+ return None
532
+ item, pos, crossed = gap
533
+ if (crossed or pos > end) and item[0][pos] in "\"'(":
534
+ title = _skip_link_title(item, pos, rest)
535
+ if title is None:
536
+ return None
537
+ gap = _skip_link_gap(*title, rest)
538
+ if gap is None:
539
+ return None
540
+ item, pos, _ = gap
541
+ if item[0][pos] != ")":
542
+ return None
543
+ if item[2] == 0:
544
+ return end, pos + 1, None
545
+ return end, len(text), (item[2], item[1] + pos + 1)
546
+
547
+
548
+ def _mask_non_prose(text):
549
+ """Fill character references and autolinks.
550
+
551
+ The fill is underscores, not spaces, so a token stays one token and word
552
+ counts do not change. An underscore is not a word character for _count_words.
553
+ Code spans and link destinations are blanked by _mask_code_spans.
554
+ """
555
+ out = []
556
+ pos = done = 0
557
+ while pos < len(text):
558
+ end = None
559
+ if text[pos] == "&":
560
+ ref = CHAR_REF.match(text, pos)
561
+ if ref and _char_ref_is_markup(ref):
562
+ end = ref.end()
563
+ if end is None and text[pos] == "<":
564
+ link = AUTOLINK.match(text, pos)
565
+ if link:
566
+ end = link.end()
567
+ if end is None:
568
+ pos += 1
569
+ continue
570
+ out.append(text[done:pos])
571
+ out.append("_" * (end - pos))
572
+ pos = done = end
573
+ out.append(text[done:])
574
+ return "".join(out)
575
+
576
+
577
+ def _sentence_starts(text):
578
+ """Return the offset of each sentence start in text.
579
+
580
+ A sentence ends at a word that ends in terminal punctuation, after any
581
+ closing quotes or brackets. Known abbreviations do not end a sentence.
582
+ One pass over the words keeps the cost linear in the text length.
583
+ """
584
+ words = list(re.finditer(r"\S+", text))
585
+ if not words:
586
+ return [0]
587
+ starts = [words[0].start()]
588
+ for word, next_word in zip(words, words[1:]):
589
+ end = SENTENCE_END.search(word.group())
590
+ if not end:
591
+ continue
592
+ core = word.group()[:end.start() + 1].lstrip(OPENERS)
593
+ if core.lower() in ABBREVIATIONS:
594
+ continue
595
+ if (DOTTED_ABBREVIATION.fullmatch(core)
596
+ and not next_word.group().lstrip(OPENERS)[:1].isupper()):
597
+ continue
598
+ starts.append(next_word.start())
599
+ return starts
600
+
601
+
602
+ def _long_sentence_findings(pieces, filename):
603
+ """Flag long sentences in one prose block.
604
+
605
+ pieces is a list of (lineno, source_column, text) tuples. The texts are
606
+ joined with spaces, so a sentence wrapped across source lines is counted
607
+ once, and each finding points at the line and column where it starts.
608
+ """
609
+ offsets, position = [], 0
610
+ for _, _, text in pieces:
611
+ offsets.append(position)
612
+ position += len(text) + 1
613
+ joined = " ".join(text for _, _, text in pieces)
614
+ findings = []
615
+ starts = _sentence_starts(joined)
616
+ for start, end in zip(starts, starts[1:] + [len(joined)]):
617
+ n = _count_words(joined[start:end])
618
+ if n <= MAX_WORDS:
619
+ continue
620
+ index = bisect.bisect_right(offsets, start) - 1
621
+ offset = offsets[index]
622
+ lineno, source_column, _ = pieces[index]
623
+ findings.append({"file": filename, "line": lineno,
624
+ "col": source_column + start - offset + 1,
625
+ "rule": "long-sentence", "level": "advisory-free",
626
+ "match": f"{n} words",
627
+ "message": f"Sentence has {n} words (cap {MAX_WORDS}). Split it."})
628
+ return findings
629
+
630
+
631
+ # One blockquote level: up to three spaces, ">", and one optional space.
632
+ QUOTE_LEVEL = re.compile(r" {0,3}> ?")
633
+
634
+
635
+ def _strip_quotes(line, depth=None):
636
+ """Return (levels, rest): the blockquote levels that start LINE and the text after them.
637
+
638
+ With DEPTH, remove at most DEPTH levels: a deeper ">" is content.
639
+ """
640
+ levels, pos = 0, 0
641
+ while depth is None or levels < depth:
642
+ level = QUOTE_LEVEL.match(line, pos)
643
+ if not level:
644
+ break
645
+ levels, pos = levels + 1, level.end()
646
+ return levels, line[pos:]
647
+
648
+
649
+ def _indent_width(text):
650
+ """Columns of the spaces and tabs that start TEXT, with a tab stop of 4."""
651
+ return len(text[:len(text) - len(text.lstrip(" \t"))].expandtabs(4))
652
+
653
+
654
+ def _fence_ends_with_quote(fence, line):
655
+ """True when LINE has fewer blockquote levels than the open FENCE."""
656
+ return fence is not None and _strip_quotes(line.expandtabs(4), fence[1])[0] < fence[1]
657
+
658
+
659
+ def _fence_step(fence, line):
660
+ """Return (is_fence_line, open_fence) after LINE.
661
+
662
+ FENCE is None, or the open fence as (run, quote depth, indent). This follows
663
+ CommonMark 4.5 for blockquotes and for the fence itself: a fence in a blockquote
664
+ ends when its quote ends. Only a run of the same character, at least as long,
665
+ with only spaces or tabs after it, closes the block. A backtick fence has no
666
+ backtick in its info string. List items are not tracked, so the indent of a
667
+ fence is not limited to three columns. A closing fence may have at most three
668
+ columns more indent than the opening fence, also at the top level.
669
+ """
670
+ # Tabs become spaces from the real column, so a tab after ">" counts correctly.
671
+ line = line.expandtabs(4)
672
+ if fence is None:
673
+ depth, rest = _strip_quotes(line)
674
+ content = rest.lstrip(" \t")
675
+ marker = CODE_FENCE.match(content)
676
+ if not marker:
677
+ return False, None
678
+ run = marker.group(1)
679
+ if run[0] == "`" and "`" in content[marker.end():]:
680
+ return False, None
681
+ return True, (run, depth, _indent_width(rest))
682
+ if _fence_ends_with_quote(fence, line):
683
+ # The quote that holds the fence ended, so the fence ended too.
684
+ return _fence_step(None, line)
685
+ run, depth, indent = fence
686
+ rest = _strip_quotes(line, depth)[1]
687
+ content = rest.lstrip(" \t")
688
+ marker = CODE_FENCE.match(content)
689
+ if (marker and marker.group(1)[0] == run[0] and len(marker.group(1)) >= len(run)
690
+ and _indent_width(rest) <= indent + 3
691
+ and not content[marker.end():].strip(" \t")):
692
+ return True, None
693
+ return False, fence
694
+
695
+
696
+ # CommonMark 4.6: how each kind of HTML block starts, and the text that ends it.
697
+ # None: the block ends at a blank line (types 6 and 7).
698
+ _BLOCK_TAGS = (
699
+ "address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|"
700
+ "details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|"
701
+ "h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|"
702
+ "optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|"
703
+ "tr|track|ul"
704
+ )
705
+ HTML_BLOCK_KINDS = [
706
+ (re.compile(r"^ {0,3}<(?:script|pre|style|textarea)(?:[ \t]|>|$)", re.I),
707
+ re.compile(r"</(?:script|pre|style|textarea)>", re.I)),
708
+ (re.compile(r"^ {0,3}<!--"), re.compile(r"-->")),
709
+ (re.compile(r"^ {0,3}<\?"), re.compile(r"\?>")),
710
+ (re.compile(r"^ {0,3}<!\[CDATA\["), re.compile(r"\]\]>")),
711
+ (re.compile(r"^ {0,3}<![A-Za-z]"), re.compile(r">")),
712
+ (re.compile(rf"^ {{0,3}}</?(?:{_BLOCK_TAGS})(?:[ \t]|/?>|$)", re.I), None),
713
+ ]
714
+ HTML_COMMENT_END = HTML_BLOCK_KINDS[1][1]
715
+ # Type 7: one complete open or closing tag alone on its line (CommonMark 6.6 grammar).
716
+ # It cannot interrupt a paragraph. In one line, CommonMark whitespace is a space or a
717
+ # tab: a no-break space or another Unicode space is text, so the line is no tag.
718
+ _ATTRIBUTE = (r"[ \t]+[A-Za-z_:][A-Za-z0-9_.:-]*"
719
+ r"(?:[ \t]*=[ \t]*(?:[^ \t\"'=<>`]+|'[^']*'|\"[^\"]*\"))?")
720
+ # The names of type 1 (pre, script, style, textarea) never start a type 7 block,
721
+ # as an open tag or as a closing tag. A custom name such as pre-x is no type 1 name.
722
+ _TYPE1_NAME = r"(?!(?i:pre|script|style|textarea)(?![A-Za-z0-9-]))"
723
+ HTML_TYPE7 = re.compile(
724
+ rf"^ {{0,3}}(?:<{_TYPE1_NAME}[A-Za-z][A-Za-z0-9-]*(?:{_ATTRIBUTE})*[ \t]*/?>"
725
+ rf"|</{_TYPE1_NAME}[A-Za-z][A-Za-z0-9-]*[ \t]*>)[ \t]*$")
726
+ # Lines that end a paragraph and are no paragraph line, after the quote prefix: an ATX
727
+ # heading and a thematic break of one repeated character. A quote line with no text
728
+ # also ends a paragraph.
729
+ NO_PARAGRAPH = re.compile(
730
+ r"^ {0,3}(?:#{1,6}(?:[ \t]|$)|(?:-[ \t]*){3,}$|(?:\*[ \t]*){3,}$|(?:_[ \t]*){3,}$)")
731
+ # A setext underline. It ends a paragraph only when a paragraph is open at the same
732
+ # quote depth. Else it is text, also as a lazy line after a quote (CommonMark 4.3).
733
+ SETEXT_UNDERLINE = re.compile(r"^ {0,3}(?:=+|-+)[ \t]*$")
734
+
735
+
736
+ def _html_block_step(block, line, in_paragraph):
737
+ """Return (in_html_block, open_block) for LINE.
738
+
739
+ BLOCK is None, False (an HTML block that ends at a blank line), or the pattern
740
+ that ends the open block. IN_PARAGRAPH: a paragraph is open before LINE. A ">"
741
+ in an HTML block is text, so it opens no fence.
742
+ """
743
+ if block is None:
744
+ for start, end in HTML_BLOCK_KINDS:
745
+ if start.match(line):
746
+ block = False if end is None else end
747
+ break
748
+ else:
749
+ if in_paragraph or not HTML_TYPE7.match(line):
750
+ return False, None
751
+ block = False
752
+ if block is False:
753
+ return bool(line.strip()), (block if line.strip() else None)
754
+ return True, (None if block.search(line) else block)
755
+
756
+
757
+ # GitHub shows the text of an HTML block of type 6 or 7. It also shows the text of a
758
+ # <script>, <style> or <textarea> block, because it escapes these tags (tagfilter). It
759
+ # shows a <pre> block as code, and it hides a comment and a "<?", "<!X" or CDATA block.
760
+ SHOWN_TYPE1 = re.compile(r"^ {0,3}<(?i:script|style|textarea)(?:[ \t]|>|$)")
761
+
762
+
763
+ def _block_shows_text(block, line):
764
+ """True when GitHub shows the text of the HTML BLOCK that starts on LINE as prose."""
765
+ return block is False or bool(SHOWN_TYPE1.match(line))
766
+
767
+
768
+ def _is_list_continuation(line, content_indent):
769
+ if not line.strip():
770
+ return True
771
+ if LIST_ITEM_START.match(line):
772
+ return False
773
+ return len(line[:len(line) - len(line.lstrip(" \t"))].expandtabs(4)) >= content_indent
774
+
775
+
776
+ def _split_table_row(line):
777
+ """Return trimmed table cells and their zero-based source columns.
778
+
779
+ A row needs a leading or trailing pipe, or a pipe between two cells.
780
+ Escaped pipes stay in their cell. This deliberately implements only the
781
+ ordinary Markdown table shape; it is enough to distinguish a table from
782
+ prose that happens to contain a pipe.
783
+ """
784
+ left = len(line) - len(line.lstrip())
785
+ right = len(line.rstrip())
786
+ content = line[left:right]
787
+ if "|" not in content:
788
+ return None
789
+ edge_pipe = False
790
+ if content.startswith("|"):
791
+ content = content[1:]
792
+ left += 1
793
+ edge_pipe = True
794
+ if content.endswith("|"):
795
+ content = content[:-1]
796
+ edge_pipe = True
797
+ raw_cells = re.split(r"(?<!\\)\|", content)
798
+ if len(raw_cells) < 2 and not edge_pipe:
799
+ return None
800
+
801
+ cells = []
802
+ column = left
803
+ for raw_cell in raw_cells:
804
+ leading = len(raw_cell) - len(raw_cell.lstrip())
805
+ cells.append((raw_cell.strip(), column + leading))
806
+ column += len(raw_cell) + 1
807
+ return cells
808
+
809
+
810
+ def _markdown_table_cells(lines):
811
+ """Map ordinary Markdown table rows to their prose cells.
812
+
813
+ The separator row anchors detection, so pipe-containing prose is not
814
+ treated as a table. Both leading-pipe and no-leading-pipe table styles are
815
+ accepted. As in GFM, the header and separator must have the same number of
816
+ cells, but a body row may have fewer or more. A line without a pipe ends
817
+ the table.
818
+ """
819
+ table_cells = {}
820
+ index = 1
821
+ while index < len(lines):
822
+ separator = _split_table_row(lines[index])
823
+ header = _split_table_row(lines[index - 1])
824
+ if (not separator or not header or len(separator) != len(header)
825
+ or not all(TABLE_SEPARATOR_CELL.fullmatch(cell)
826
+ for cell, _ in separator)):
827
+ index += 1
828
+ continue
829
+
830
+ table_cells[index - 1] = header
831
+ table_cells[index] = []
832
+ index += 1
833
+ while index < len(lines):
834
+ row = _split_table_row(lines[index])
835
+ if not row:
836
+ break
837
+ table_cells[index] = row
838
+ index += 1
839
+ return table_cells
840
+
841
+
842
+ def _dangling_conjunction_findings(text, filename):
843
+ lines = text.splitlines()
844
+ findings = []
845
+ fence = None
846
+ # The HTML block open outside the item loop below, as _html_block_step returns it,
847
+ # the content indent of the list item that holds it (0: no item), and whether
848
+ # GitHub shows its text.
849
+ html_block, html_container, html_shows = None, 0, False
850
+ # The content indents of the list items that can still hold a line, outermost first.
851
+ open_items = []
852
+ # A paragraph is open before the line: a text line with less indent continues it
853
+ # (a lazy line). PARAGRAPH_INDENT is the content indent of the list item that holds
854
+ # the paragraph (0: no item), or None in a quote. As in cmark-gfm, type 7 cannot
855
+ # interrupt the paragraph on a line that is in that item too.
856
+ in_paragraph, paragraph_indent = False, None
857
+
858
+ def close_items(indent):
859
+ """A line with INDENT ends the list items with more content indent."""
860
+ while open_items and open_items[-1] > indent:
861
+ open_items.pop()
862
+
863
+ index = 0
864
+ while index < len(lines):
865
+ line = lines[index]
866
+ if _fence_ends_with_quote(fence, line):
867
+ # The quote of the fence ended: judge this line as any other line.
868
+ fence = None
869
+ if fence is None:
870
+ # A line with less indent than the item that holds the block ends the item,
871
+ # and the block in it. GitHub still hides the text after an open comment up to
872
+ # "-->", as in the item loop below.
873
+ if (html_block is not None and html_block is not HTML_COMMENT_END
874
+ and line.strip(" \t") and _indent_width(line) < html_container):
875
+ html_block = None
876
+ open_block = html_block
877
+ continues = (in_paragraph and paragraph_indent is not None
878
+ and _indent_width(line) >= paragraph_indent)
879
+ # A block starts at the indent of the list item that holds the line, not at
880
+ # the start of the line.
881
+ line_indent = _indent_width(line)
882
+ block_indent = next((indent for indent in reversed(open_items)
883
+ if indent <= line_indent), 0)
884
+ block_line = line.expandtabs(4)[block_indent:]
885
+ in_html, html_block = _html_block_step(html_block, block_line, continues)
886
+ if in_html and open_block is None:
887
+ close_items(line_indent)
888
+ html_container = open_items[-1] if open_items else 0
889
+ html_shows = _block_shows_text(html_block, block_line)
890
+ # An HTML block holds literal text: no fence opens in it, and outside list items
891
+ # no list item starts in it. In a list item, the text that GitHub shows in the
892
+ # block is item text, so a list marker there starts an item, as before. The
893
+ # block stays open after that item, so a fence line in it is still text. The
894
+ # line that ends the block is in the block, so no item starts on it.
895
+ if (in_html and open_block is not None and html_block is not None
896
+ and html_shows and html_container and LIST_ITEM_START.match(line)):
897
+ in_html = False
898
+ if in_html:
899
+ in_paragraph = False
900
+ index += 1
901
+ continue
902
+ is_fence, fence = _fence_step(fence, line)
903
+ if is_fence or fence:
904
+ if is_fence and fence is not None:
905
+ close_items(_indent_width(line))
906
+ in_paragraph = False
907
+ index += 1
908
+ continue
909
+ start = LIST_ITEM_START.match(line)
910
+ if not start:
911
+ # A blank line has only spaces and tabs: a no-break space is text. A line
912
+ # with only quote markers is blank in its quote.
913
+ depth, content = _strip_quotes(line.expandtabs(4))
914
+ # A setext underline in the container of the paragraph makes it a heading,
915
+ # so the paragraph ends. As a lazy line it is text (CommonMark 4.3).
916
+ underline = (in_paragraph and paragraph_indent is not None and not depth
917
+ and _indent_width(line) >= paragraph_indent
918
+ and SETEXT_UNDERLINE.match(content))
919
+ text_line = (bool(content.strip(" \t")) and not NO_PARAGRAPH.match(content)
920
+ and not underline)
921
+ if line.strip(" \t") and not (in_paragraph and text_line and not depth):
922
+ close_items(_indent_width(line))
923
+ paragraph_indent = None if depth else (open_items[-1] if open_items else 0)
924
+ in_paragraph = text_line
925
+ index += 1
926
+ continue
927
+
928
+ # A list item that starts with a number other than 1, and an empty list item,
929
+ # cannot interrupt a paragraph (CommonMark 5.2): GitHub reads the line as text of
930
+ # the paragraph. An empty "-" item there is a setext underline, so the paragraph
931
+ # ends. The pass still reads the lines of the item, but with no HTML block in them
932
+ # (they are paragraph text), and after the item the paragraph stays open in its
933
+ # container. The item is not open for later lines.
934
+ marker, body = start.group("marker"), start.group("body")
935
+ outer_paragraph = (in_paragraph and paragraph_indent is not None
936
+ and _indent_width(line) >= paragraph_indent
937
+ and ((marker[:-1].isdigit() and marker[:-1].lstrip("0") != "1")
938
+ or (not body.strip(" \t") and marker != "-")))
939
+ outer_indent = paragraph_indent
940
+ close_items(len(start.group("indent")))
941
+ # After five or more spaces the text is indented code, and the content column
942
+ # is one space after the marker (CommonMark 5.2).
943
+ gap = len(start.group("gap"))
944
+ if gap > 4 and body.strip(" \t"):
945
+ gap = 1
946
+ content_indent = len(start.group("indent")) + len(start.group("marker")) + gap
947
+ # A list marker in an open HTML block around the item is text of the block on
948
+ # GitHub, so the item is not open for later lines (see the item loop below).
949
+ if not outer_paragraph and html_block is None:
950
+ open_items.append(content_indent)
951
+ # An HTML block can start on the line of the marker. It is in the item, and it
952
+ # ends when the item ends. After five or more spaces the text is indented code.
953
+ body_html, item_html, item_shows = False, None, False
954
+ # In an open HTML block around the item, the body is text of that block.
955
+ if html_block is None and not outer_paragraph and len(start.group("gap")) <= 4:
956
+ body_html, item_html = _html_block_step(None, body, False)
957
+ item_shows = _block_shows_text(item_html, body)
958
+ item_paragraph = outer_paragraph or (bool(body.strip(" \t")) and not body_html
959
+ and not NO_PARAGRAPH.match(body))
960
+ # A block of code or hidden text that stays open shows no item text (see below).
961
+ item_lines = [] if item_html and not item_shows else [(index, body)]
962
+ next_index = index + 1
963
+ item_fence = None
964
+ while next_index < len(lines):
965
+ candidate = lines[next_index]
966
+ # An open HTML block around this item (a list marker in it started the item,
967
+ # see above) goes on over the lines of the item, as raw item text, up to a
968
+ # blank line, its end pattern or a line with less indent than its container.
969
+ if html_block is not None:
970
+ if candidate.strip(" \t") and _indent_width(candidate) < html_container:
971
+ html_block = None
972
+ else:
973
+ in_outer_html, html_block = _html_block_step(html_block, candidate, False)
974
+ if html_block is None:
975
+ # The block ended. GitHub reads the next lines in its container.
976
+ content_indent = html_container
977
+ if in_outer_html:
978
+ # The line that ends the block is in it, also after a list marker.
979
+ if (html_block is not None
980
+ and not _is_list_continuation(candidate, content_indent)):
981
+ break
982
+ item_lines.append((next_index, candidate))
983
+ item_paragraph = False
984
+ next_index += 1
985
+ continue
986
+ if _fence_ends_with_quote(item_fence, candidate):
987
+ # The quote of the fence ended: judge this line as any other line.
988
+ item_fence = None
989
+ if item_html is HTML_COMMENT_END:
990
+ # GitHub hides the text from "<!--" up to "-->", also over the end of the
991
+ # item and over the next items. The text after "-->" joins this item. No
992
+ # item, block or fence starts in the comment.
993
+ end = HTML_COMMENT_END.search(candidate)
994
+ if end:
995
+ item_html = None
996
+ # spaces keep the columns
997
+ item_lines.append((next_index, " " * end.end() + candidate[end.end():]))
998
+ item_paragraph = False
999
+ next_index += 1
1000
+ continue
1001
+ in_html = hidden = False
1002
+ if item_fence is None and not outer_paragraph:
1003
+ if candidate.strip() and _indent_width(candidate) < content_indent:
1004
+ item_html = None # the line ends the item, and the HTML block in it
1005
+ else:
1006
+ # The item holds its lines from the content indent on.
1007
+ open_block = item_html
1008
+ rest = candidate.expandtabs(4)[content_indent:]
1009
+ in_html, item_html = _html_block_step(open_block, rest, item_paragraph)
1010
+ if in_html and open_block is None:
1011
+ item_shows = _block_shows_text(item_html, rest)
1012
+ # A <pre> block holds code, and GitHub hides a comment and a "<?", "<!X"
1013
+ # or CDATA block. As in a fence, the lines of such a block, from its
1014
+ # first line up to its last line, are no item text, and no list item
1015
+ # starts there. The last line can show text after the end of the block.
1016
+ # (An open end pattern is not None or False.)
1017
+ hidden = bool(item_html) and not item_shows
1018
+ if hidden:
1019
+ item_paragraph = False
1020
+ next_index += 1
1021
+ continue
1022
+ # A line outside the item ends it, also when it opens a fence. The line that ends
1023
+ # an HTML block of the item is in the block, also after a list marker.
1024
+ if (item_fence is None and not _is_list_continuation(candidate, content_indent)
1025
+ and not (in_html and item_html is None)):
1026
+ break
1027
+ # Fence delimiters are state markers, not meaningful item lines.
1028
+ # In an HTML block, a fence line is text.
1029
+ if not in_html:
1030
+ is_fence, item_fence = _fence_step(item_fence, candidate)
1031
+ if is_fence or item_fence:
1032
+ item_paragraph = False
1033
+ next_index += 1
1034
+ continue
1035
+ item_lines.append((next_index, candidate))
1036
+ item_paragraph = (not in_html and bool(candidate.strip(" \t"))
1037
+ and not NO_PARAGRAPH.match(candidate.expandtabs(4)[content_indent:]))
1038
+ next_index += 1
1039
+ # A list marker in a block of the item whose text GitHub shows ends the item, as
1040
+ # before, but on GitHub the block goes on. Hand it on, so that the lines after the
1041
+ # next item are still in it.
1042
+ if (item_html is not None and next_index < len(lines)
1043
+ and LIST_ITEM_START.match(lines[next_index])
1044
+ and _indent_width(lines[next_index]) >= content_indent):
1045
+ html_block, html_container, html_shows = item_html, content_indent, item_shows
1046
+ # The line after the item can continue the last paragraph of the item.
1047
+ in_paragraph = item_paragraph
1048
+ paragraph_indent = outer_indent if outer_paragraph else content_indent
1049
+
1050
+ meaningful = []
1051
+ for line_index, item_line in item_lines:
1052
+ # Preserve code spans as neutral operands while ignoring their contents.
1053
+ cleaned = INLINE_CODE.sub(" CODE ", item_line).strip()
1054
+ if cleaned:
1055
+ meaningful.append((line_index, cleaned))
1056
+ if meaningful:
1057
+ end_line_index, end_line = meaningful[-1]
1058
+ conjunction = CONJUNCTION_END.search(end_line)
1059
+ else:
1060
+ end_line_index, end_line, conjunction = None, None, None
1061
+ if conjunction:
1062
+ if end_line_index == index:
1063
+ finding_line = index + 1
1064
+ finding_col = start.start("marker") + 1
1065
+ else:
1066
+ raw_end_line = next(
1067
+ raw for line_index, raw in item_lines
1068
+ if line_index == end_line_index
1069
+ )
1070
+ masked_end_line = INLINE_CODE.sub(
1071
+ lambda match: " " * len(match.group(0)), raw_end_line
1072
+ )
1073
+ raw_conjunction = CONJUNCTION_END.search(masked_end_line)
1074
+ finding_line = end_line_index + 1
1075
+ finding_col = raw_conjunction.start() + 1 if raw_conjunction else 1
1076
+ findings.append({
1077
+ "file": filename,
1078
+ "line": finding_line,
1079
+ "col": finding_col,
1080
+ "rule": "dangling-conjunction",
1081
+ "level": "advisory-free",
1082
+ "match": end_line,
1083
+ "message": "List item ends with a coordinating conjunction. Complete the item or join it with the next item.",
1084
+ })
1085
+ index = next_index
1086
+ return findings
1087
+
1088
+
1089
+ # A line of YAML front matter: a key, a list item, an indented line, or a comment.
1090
+ # A key is quoted, or a plain word that starts with no YAML indicator; ": "
1091
+ # or ":" at the end of the line ends it. A plain key with a space is valid
1092
+ # YAML, but prose such as "Step 1: run it" looks the same, so it is Markdown.
1093
+ _YAML_KEY = (r"""(?:"(?:[^"\\]|\\.)*"|'(?:[^']|'')*'|[^\s#'"\[\]{},&*!|>%@`-]\S*?)"""
1094
+ r"[ \t]*:(?:[ \t]|$)")
1095
+ FRONT_MATTER_KEY = re.compile(_YAML_KEY)
1096
+ FRONT_MATTER_LINE = re.compile(_YAML_KEY + r"|-(?:[ \t]|$)|[ \t]+\S|#")
1097
+ # The key "description", plain or in quotes.
1098
+ DESCRIPTION_KEY = re.compile(r"""(?:description|"description"|'description')[ \t]*:(?:[ \t]+|$)""")
1099
+ # A block scalar indicator ("|", ">-", "|2+"), with an optional comment.
1100
+ BLOCK_SCALAR = re.compile(r"[|>][1-9+-]{0,2}(?:[ \t]+#.*)?")
1101
+ # A YAML comment in a plain scalar: "#" at the start or after a space or tab.
1102
+ YAML_COMMENT = re.compile(r"(?:^|[ \t])#")
1103
+
1104
+
1105
+ def _closing_quote(text, quote):
1106
+ """Return the index of the QUOTE that closes a quoted YAML scalar in TEXT, or None.
1107
+
1108
+ In single quotes, '' is a quote. In double quotes, a backslash escapes the
1109
+ next character.
1110
+ """
1111
+ pos = 0
1112
+ while pos < len(text):
1113
+ if quote == '"' and text[pos] == "\\":
1114
+ pos += 2
1115
+ continue
1116
+ if text[pos] == quote:
1117
+ if quote == "'" and text[pos + 1:pos + 2] == "'":
1118
+ pos += 2
1119
+ continue
1120
+ return pos
1121
+ pos += 1
1122
+ return None
1123
+
1124
+
1125
+ def _scalar_pieces(value, parts):
1126
+ """Return the prose pieces of a YAML scalar as (line index, column, text).
1127
+
1128
+ VALUE is the text after the key. PARTS are the lines of the scalar with
1129
+ their line index and the column of their text, the key line first when
1130
+ VALUE is not empty. A block scalar ("|", ">-") keeps each later line up to
1131
+ one with less indent than its first, also a "#". A quoted scalar ends at
1132
+ its closing quote. A plain scalar ends at a comment. A comment after the
1133
+ scalar is not prose.
1134
+ """
1135
+ pieces = []
1136
+ if BLOCK_SCALAR.fullmatch(value):
1137
+ # The first line sets the indent. A line with less indent, such as a
1138
+ # comment, ends the scalar.
1139
+ for piece in parts[1:]:
1140
+ if piece[1] < parts[1][1]:
1141
+ break
1142
+ pieces.append(piece)
1143
+ elif value[:1] in ("'", '"'):
1144
+ index, column, text = parts[0]
1145
+ parts = [(index, column + 1, text[1:])] + parts[1:]
1146
+ for index, column, text in parts:
1147
+ close = _closing_quote(text, value[0])
1148
+ pieces.append((index, column, text if close is None else text[:close]))
1149
+ if close is not None:
1150
+ break
1151
+ else:
1152
+ for index, column, text in parts:
1153
+ comment = YAML_COMMENT.search(text)
1154
+ pieces.append((index, column, text if comment is None else text[:comment.start()]))
1155
+ if comment is not None:
1156
+ break
1157
+ return [piece for piece in pieces if piece[2].strip()]
1158
+
1159
+
1160
+ def _front_matter(text):
1161
+ """Find a leading YAML front-matter block.
1162
+
1163
+ Return (text, description): TEXT with the block blanked, so line numbers
1164
+ stay true, and the value of its "description:" key as (line index,
1165
+ column, text) pieces. The block counts only when its first line that is
1166
+ not a comment is a key, and every line is a key, a list item, an indented
1167
+ line, a comment or a blank line. Else it is Markdown: a thematic break,
1168
+ prose and a second thematic break. Only the description is prose, because
1169
+ agents act on it.
1170
+ """
1171
+ lines = text.splitlines()
1172
+ if not lines or lines[0].rstrip() != "---":
1173
+ return text, []
1174
+ for end in range(1, len(lines)):
1175
+ if lines[end].rstrip() == "---":
1176
+ break
1177
+ else:
1178
+ return text, []
1179
+ body = [line for line in lines[1:end] if line.strip()]
1180
+ content = [line for line in body if not line.startswith("#")]
1181
+ if (body and (not content or not FRONT_MATTER_KEY.match(content[0]))
1182
+ or not all(FRONT_MATTER_LINE.match(line) for line in body)):
1183
+ return text, []
1184
+ pieces = []
1185
+ # "description" in quotes is the same key. The first key counts: a later line that
1186
+ # starts with the key can be in a quoted value that goes on over lines.
1187
+ for index in range(1, end):
1188
+ key = DESCRIPTION_KEY.match(lines[index])
1189
+ if key:
1190
+ break
1191
+ else:
1192
+ index = end
1193
+ if index < end:
1194
+ value = lines[index][key.end():].rstrip()
1195
+ parts = [(index, key.end(), value)] if value else []
1196
+ for later in range(index + 1, end):
1197
+ line = lines[later]
1198
+ if line.strip() and line[0] not in " \t":
1199
+ break
1200
+ lead = len(line) - len(line.lstrip(" \t"))
1201
+ if line[lead:].rstrip():
1202
+ parts.append((later, lead, line[lead:].rstrip()))
1203
+ pieces = _scalar_pieces(value, parts)
1204
+ return "\n" * (end + 1) + "\n".join(lines[end + 1:]), pieces
1205
+
1206
+
1207
+ def lint(text, filename="<stdin>"):
1208
+ text, description = _front_matter(text)
1209
+ findings = []
1210
+ words_total = 0
1211
+ fence = None
1212
+ html_block = None
1213
+ html_depth = 0
1214
+ in_paragraph = False
1215
+ # content indent of the open list item, so its indented lines are not code
1216
+ list_indent = 0
1217
+ # backtick run length of a code span that is open across lines, or 0
1218
+ open_span = 0
1219
+ paragraph_depth = 0
1220
+ # (line number, column) where a link from an earlier line ends, or None
1221
+ link_end = None
1222
+ html_containers = []
1223
+ lines = text.splitlines()
1224
+ table_cells = _markdown_table_cells(lines)
1225
+ # first occurrence of each synonym-group member: (group_idx, base) -> (line, col, match)
1226
+ seen_synonyms = {}
1227
+ # consecutive prose lines form one block, so wrapped sentences count once
1228
+ block = []
1229
+ in_quote = False
1230
+ after_boundary = True
1231
+
1232
+ # the end index and backtick run index of the paragraph that holds the current line
1233
+ span_end, span_runs = -1, {}
1234
+
1235
+ def flush():
1236
+ findings.extend(_long_sentence_findings(block, filename))
1237
+ block.clear()
1238
+
1239
+ def check(lineno, segment, source_column, line, skip, is_table,
1240
+ add_to_block=True):
1241
+ nonlocal words_total
1242
+ words_total += len(line.split())
1243
+ for rule_id, level, pattern, msg in RULES:
1244
+ for m in pattern.finditer(line):
1245
+ # the modal may end the line before: "may" / "have failed"
1246
+ before = (block[-1][2] + " " if block else "") + line[skip:m.start()]
1247
+ if rule_id == "present-perfect" and MODAL_PERFECT_PREFIX.search(before):
1248
+ continue
1249
+ findings.append({"file": filename, "line": lineno,
1250
+ "col": source_column + m.start() + 1,
1251
+ "rule": rule_id, "level": level,
1252
+ "match": m.group(0), "message": msg})
1253
+ for gi, group in enumerate(SYNONYM_GROUPS):
1254
+ for base in group:
1255
+ if (gi, base) in seen_synonyms:
1256
+ continue
1257
+ m = _word_re(base).search(line)
1258
+ if m:
1259
+ seen_synonyms[(gi, base)] = (
1260
+ lineno, source_column + m.start() + 1, m.group(0)
1261
+ )
1262
+ # This flag affects sentence blocks only; rule checks still scan the line.
1263
+ if add_to_block and line[skip:].strip():
1264
+ block.append((lineno, source_column + skip, line[skip:]))
1265
+
1266
+ # The description in the front matter is one paragraph of prose.
1267
+ span = 0
1268
+ values = [piece[2] for piece in description]
1269
+ last_piece = {} # the last piece with a run of each length
1270
+ for piece_index, piece_value in enumerate(values):
1271
+ for m in BACKTICK_RUN.finditer(piece_value):
1272
+ last_piece[len(m.group())] = piece_index
1273
+ piece_end = None # (piece, column) where a link from an earlier piece ends
1274
+ piece_brackets = [[], 0]
1275
+ for position, (index, column, value) in enumerate(description):
1276
+ skip = 0
1277
+ if piece_end and piece_end[0] > position:
1278
+ skip = len(value) # the whole piece is in the title of a link
1279
+ elif piece_end and piece_end[0] == position:
1280
+ skip = piece_end[1]
1281
+ carry = []
1282
+ masked, span = _mask_code_spans(value, span, lambda run: last_piece.get(run, -1) > position, values, position + 1,
1283
+ 0, skip, carry, piece_brackets)
1284
+ if carry:
1285
+ piece_end = (position + carry[0], carry[1])
1286
+ elif piece_end and piece_end[0] <= position:
1287
+ piece_end = None
1288
+ check(index + 1, value, column, _mask_non_prose(masked), 0, False)
1289
+ flush()
1290
+ def closes_later(run):
1291
+ """True when a run of length RUN closes a span on a later line of this paragraph."""
1292
+ nonlocal span_end, span_runs
1293
+ if not in_paragraph:
1294
+ return False
1295
+ if lineno > span_end:
1296
+ span_end, span_runs = _paragraph_runs(lines, lineno, paragraph_depth, list_indent)
1297
+ return span_runs.get(run, -1) >= lineno
1298
+
1299
+ for lineno, raw_line in enumerate(lines, 1):
1300
+ was_paragraph = in_paragraph
1301
+ if not was_paragraph or _breaks_paragraph(paragraph_depth, raw_line, list_indent):
1302
+ open_span = 0
1303
+ open_brackets = [[], 0] # the "[" marks that can open link text
1304
+ if _fence_ends_with_quote(fence, raw_line):
1305
+ # The quote that holds the fence ended: judge this line as any other line.
1306
+ fence = None
1307
+ in_html = False
1308
+ if fence is None:
1309
+ # An HTML block can sit in a block quote. It ends when its quote ends. An
1310
+ # open block strips only its own ">" levels: a deeper ">" is its text.
1311
+ if html_block is not None:
1312
+ line_depth, line_unquoted = _strip_quotes(raw_line.expandtabs(4), html_depth)
1313
+ if line_depth < html_depth:
1314
+ html_block = None
1315
+ if html_block is None:
1316
+ line_depth, line_unquoted = _strip_quotes(raw_line.expandtabs(4))
1317
+ # A deeper quote starts a block quote, which a paragraph outside it does not continue.
1318
+ continues = in_paragraph and line_depth <= paragraph_depth
1319
+ was_html = html_block is not None
1320
+ in_html, html_block = _html_block_step(html_block, line_unquoted, continues)
1321
+ if in_html and not was_html:
1322
+ html_depth = line_depth
1323
+ if in_html:
1324
+ # An HTML block holds literal text until it ends, so no fence opens in it.
1325
+ is_fence = False
1326
+ else:
1327
+ is_fence, fence = _fence_step(fence, raw_line)
1328
+ # A paragraph is open after a line of text that no other block takes.
1329
+ # The paragraph state reads the line after its quote prefix, so a line with only
1330
+ # quote markers is blank inside its quote.
1331
+ depth, unquoted = _strip_quotes(raw_line.expandtabs(4))
1332
+ prev_depth = paragraph_depth
1333
+ # an underline left of the open item's content is a lazy line, not an underline
1334
+ underline = (in_paragraph and depth == paragraph_depth
1335
+ and _indent_width(unquoted) >= list_indent
1336
+ and SETEXT_UNDERLINE.match(unquoted))
1337
+ in_paragraph = bool(unquoted.strip() and not in_html and not is_fence and not fence
1338
+ and not NO_PARAGRAPH.match(unquoted) and not underline)
1339
+ if not in_paragraph:
1340
+ paragraph_depth = 0
1341
+ elif (not was_paragraph or depth > paragraph_depth
1342
+ or (depth < paragraph_depth
1343
+ and _breaks_paragraph(paragraph_depth, raw_line, list_indent))):
1344
+ paragraph_depth = depth
1345
+ # A lazy line with fewer ">" keeps the depth of the paragraph. A list item
1346
+ # with fewer ">" is no lazy line: it ends the quote and starts a paragraph.
1347
+ if is_fence:
1348
+ flush()
1349
+ # A fence inside a quote does not start the quote: an alert marker after it is text.
1350
+ after_boundary = _strip_quotes(raw_line)[0] == 0
1351
+ continue
1352
+ if fence:
1353
+ continue
1354
+ # CommonMark 4.4: an indented code block. It cannot interrupt a paragraph.
1355
+ rest = _strip_quotes(raw_line.expandtabs(4))[1]
1356
+ indent = _indent_width(rest)
1357
+ if rest.strip() and not in_html:
1358
+ if indent >= list_indent + 4:
1359
+ if not was_paragraph:
1360
+ in_paragraph = False
1361
+ continue
1362
+ else:
1363
+ # An item line sets the content indent also after a text line or
1364
+ # in a tight list. A lazy line does not end the item.
1365
+ item = _list_item(rest, list_indent, was_paragraph and depth == prev_depth)
1366
+ if item:
1367
+ list_indent = len(item.group(0)) if item.group(3) else len(item.group(0)) + 1
1368
+ elif indent < list_indent and not (was_paragraph and in_paragraph):
1369
+ # only a lazy text line keeps the item open, not a heading or a break
1370
+ list_indent = 0
1371
+ if HTML_BLOCK_TAG.match(raw_line.lstrip()):
1372
+ for tag in HTML_BLOCK_TAG.finditer(_mask_inline_code(raw_line)):
1373
+ name = tag.group(2).lower()
1374
+ if tag.group(1):
1375
+ if name in html_containers:
1376
+ while html_containers:
1377
+ if html_containers.pop() == name:
1378
+ break
1379
+ elif not tag.group().endswith("/>"):
1380
+ html_containers.append(name)
1381
+ segments = table_cells.get(lineno - 1, [(raw_line, 0)])
1382
+ is_table = (lineno - 1) in table_cells
1383
+ quote = BLOCKQUOTE_PREFIX.match(raw_line)
1384
+ is_quote = bool(quote)
1385
+ # judge block boundaries on the text inside any blockquote prefix
1386
+ content = raw_line[quote.end():] if quote else raw_line
1387
+ # a GitHub alert marker is a label that stands alone, as a heading does.
1388
+ # Accept it only in an unindented quote of one level after a blank line,
1389
+ # a fence, or at the start of the text. Elsewhere a list item or a lazy
1390
+ # continuation line can hold the quote, and GitHub shows the marker as
1391
+ # text. Markers inside HTML containers are also plain text.
1392
+ alert = (not html_containers and raw_line.startswith(">") and after_boundary
1393
+ and quote.group().count(">") == 1
1394
+ and GITHUB_ALERT.fullmatch(content.strip()))
1395
+ admonition_title = bool(ADMONITION_TITLE.match(content))
1396
+ standalone = STANDALONE_LINE.match(content) or admonition_title or alert
1397
+ if (is_table or not content.strip() or standalone
1398
+ or ANY_LIST_ITEM.match(content) or is_quote != in_quote
1399
+ # a deeper quote starts a new paragraph, so the text before it is no context
1400
+ or depth > prev_depth):
1401
+ flush()
1402
+ in_quote = is_quote
1403
+ after_boundary = not raw_line.strip()
1404
+ for segment, source_column in segments:
1405
+ if in_html:
1406
+ # an HTML block has no Markdown links; backtick pairs stay blanked
1407
+ line = _mask_non_prose(_mask_inline_code(segment))
1408
+ elif is_table:
1409
+ # a table cell holds its own spans and links
1410
+ line = _mask_non_prose(_mask_code_spans(segment, 0, lambda run: False)[0])
1411
+ else:
1412
+ skip = 0
1413
+ if link_end and link_end[0] > lineno:
1414
+ skip = len(segment) # the whole line is in the title of a link
1415
+ elif link_end and link_end[0] == lineno:
1416
+ skip = link_end[1]
1417
+ carry = []
1418
+ masked, open_span = _mask_code_spans(segment, open_span, closes_later, lines, lineno,
1419
+ paragraph_depth if in_paragraph else None,
1420
+ skip, carry, open_brackets)
1421
+ if carry:
1422
+ link_end = (lineno + carry[0], carry[1])
1423
+ elif link_end and link_end[0] <= lineno:
1424
+ link_end = None
1425
+ line = _mask_non_prose(masked)
1426
+ # The blockquote prefix and list marker are markup.
1427
+ skip = quote.end() if quote and not is_table else 0
1428
+ marker = None if is_table else ANY_LIST_ITEM.match(segment[skip:])
1429
+ if marker:
1430
+ skip += marker.end()
1431
+ check(
1432
+ lineno, segment, source_column, line, skip, is_table,
1433
+ add_to_block=not admonition_title,
1434
+ )
1435
+ if is_table:
1436
+ flush() # each table cell is its own block
1437
+ if standalone:
1438
+ flush()
1439
+ flush()
1440
+ # synonym rotation: flag each member after the first, at its first occurrence
1441
+ for gi, group in enumerate(SYNONYM_GROUPS):
1442
+ present = [(seen_synonyms[(gi, b)], b) for b in group if (gi, b) in seen_synonyms]
1443
+ if len(present) > 1:
1444
+ present.sort() # document order
1445
+ first_base = present[0][1]
1446
+ for (lineno, col, match), base in present[1:]:
1447
+ findings.append({"file": filename, "line": lineno, "col": col,
1448
+ "rule": "synonym-rotation", "level": "advisory-free",
1449
+ "match": match,
1450
+ "message": f"'{base}' and '{first_base}' name the same action. Pick one and use it every time."})
1451
+ findings.extend(_dangling_conjunction_findings(text, filename))
1452
+ findings.sort(key=lambda f: (f["line"], f["col"]))
1453
+ return findings, words_total
1454
+
1455
+
1456
+ def report(findings, words_total, as_json, hard_count, baseline):
1457
+ rate = round(len(findings) * 100 / words_total, 1) if words_total else 0.0
1458
+ if as_json:
1459
+ print(json.dumps({"violations": findings, "count": len(findings),
1460
+ "hard_count": hard_count, "baseline": baseline,
1461
+ "words": words_total, "per_100_words": rate}, indent=2))
1462
+ return
1463
+ for f in findings:
1464
+ print(f"{f['file']}:{f['line']}:{f['col']} {f['rule']}: {f['message']} [{f['match']}]")
1465
+ print(f"\n{len(findings)} violations ({hard_count} hard, baseline {baseline}), "
1466
+ f"{words_total} words, {rate} per 100 words")
1467
+ print("Hedges/modality (may, might, could) are never flagged: confidence is content.")
1468
+
1469
+
1470
+ def selftest():
1471
+ bad = ("The panel is removed; spin up the job. "
1472
+ "Perform an analysis of the seamless log. "
1473
+ "We have received the report.")
1474
+ bad_findings, _ = lint(bad)
1475
+ rules = {f["rule"] for f in bad_findings}
1476
+ for expected in ("semicolon", "phrasal-verb", "nominalization",
1477
+ "marketing-adjective", "passive-voice", "present-perfect"):
1478
+ assert expected in rules, (expected, bad_findings)
1479
+
1480
+ # MkDocs admonition title variants, including multiple classes and no space.
1481
+ for title in (
1482
+ "!!! note",
1483
+ "!!!note",
1484
+ "!!! info inline end \"Title\"",
1485
+ "!!! danger highlight blink \"Title\"",
1486
+ "??? note",
1487
+ "???+ note",
1488
+ '!!! note "Quoted title"',
1489
+ ):
1490
+ assert ADMONITION_TITLE.fullmatch(title), title
1491
+ for title in ("!!!+ note", "!!!+note"):
1492
+ assert not ADMONITION_TITLE.fullmatch(title), title
1493
+
1494
+ # Long admonition titles are headings, not prose sentences.
1495
+ long_title = '!!! note "' + " ".join(["title"] * 30) + '"'
1496
+ findings, _ = lint(long_title)
1497
+ assert not any(f["rule"] == "long-sentence" for f in findings), findings
1498
+
1499
+ # A standalone title does not join the four-space-indented body sentence.
1500
+ body = " ".join(["word"] * 26) + "."
1501
+ findings, _ = lint('!!! note "Title"\n ' + body)
1502
+ long_sentences = [
1503
+ f for f in findings if f["rule"] == "long-sentence"
1504
+ ]
1505
+ assert len(long_sentences) == 1, findings
1506
+ assert long_sentences[0]["match"] == "26 words", findings
1507
+
1508
+ # A four-space-indented title inside a list item is also standalone.
1509
+ list_title = '!!! info inline end "' + " ".join(["title"] * 30) + '"'
1510
+ findings, _ = lint("- item\n " + list_title + "\n " + body)
1511
+ long_sentences = [
1512
+ f for f in findings if f["rule"] == "long-sentence"
1513
+ ]
1514
+ assert len(long_sentences) == 1, findings
1515
+ assert long_sentences[0]["line"] == 3, findings
1516
+ assert long_sentences[0]["match"] == "26 words", findings
1517
+
1518
+ # A long whitespace-only title candidate must fail quickly.
1519
+ assert not ADMONITION_TITLE.fullmatch("!!!" + " " * 32000)
1520
+ # Many class words followed by a broken title must also fail without backtracking.
1521
+ assert not ADMONITION_TITLE.fullmatch("!!! " + "info " * 16000 + '"')
1522
+ # hedges must never be flagged, including modal + perfect infinitive
1523
+ findings, _ = lint("The request may have failed. It could be a timeout. "
1524
+ "The disk might have filled.")
1525
+ assert findings == [], findings
1526
+ # irregular participles: "has run" is a compound tense as much as "has failed"
1527
+ findings, _ = lint("The task has run. The job has set the flag. We have begun.")
1528
+ assert sum(1 for f in findings if f["rule"] == "present-perfect") == 3, findings
1529
+ findings, _ = lint("The job may have run.")
1530
+ assert not any(f["rule"] == "present-perfect" for f in findings), findings
1531
+ # Modal perfects remain protected across negation and variable whitespace.
1532
+ for modal in ("may", "might", "could", "should", "would", "must"):
1533
+ for gap in (" ", " ", "\t", " not "):
1534
+ findings, _ = lint(f"The task {modal}{gap}have run.")
1535
+ assert not any(f["rule"] == "present-perfect" for f in findings), findings
1536
+ findings, _ = lint("The task couldn't have run. The task MAY NOT HAVE RUN.")
1537
+ assert not any(f["rule"] == "present-perfect" for f in findings), findings
1538
+ findings, _ = lint("It might already have finished. It could not possibly have run. "
1539
+ "It may never have started. It should probably have stopped.")
1540
+ assert not any(f["rule"] == "present-perfect" for f in findings), findings
1541
+ findings, _ = lint("It cannot have run. It won't have finished. It can't have stopped. "
1542
+ "It may well have failed. It may very well have failed.\n\n"
1543
+ "The request may\nhave failed.")
1544
+ assert not any(f["rule"] == "present-perfect" for f in findings), findings
1545
+ # a modal in an earlier paragraph does not protect the next one
1546
+ findings, _ = lint("It may\n\nhave failed.")
1547
+ assert sum(f["rule"] == "present-perfect" for f in findings) == 1, findings
1548
+ # the quote prefix and the list marker are markup: a modal at the end of the
1549
+ # line before still protects the perfect, also in a quote and in a list
1550
+ for text in ("> It may\n> have failed.", "> > It may\n> > have failed.",
1551
+ "It may\nhave failed.", "- It may\n have run.", "> - It may\n> have run."):
1552
+ findings, _ = lint(text)
1553
+ assert not any(f["rule"] == "present-perfect" for f in findings), (text, findings)
1554
+ for text in ("> It may\n>\n> have failed.", "> It may\n>> have failed."):
1555
+ findings, _ = lint(text)
1556
+ perfect_lines = [f["line"] for f in findings if f["rule"] == "present-perfect"]
1557
+ assert perfect_lines == [text.count("\n") + 1], (text, findings)
1558
+ # an adverb without a modal is not a hedge
1559
+ findings, _ = lint("The task already has run. They probably have finished.")
1560
+ assert sum(f["rule"] == "present-perfect" for f in findings) == 2, findings
1561
+ findings, _ = lint("The task has run. We have begun. The flag is set.")
1562
+ assert sum(f["rule"] == "present-perfect" for f in findings) == 2, findings
1563
+ assert any(f["rule"] == "passive-voice" for f in findings), findings
1564
+ findings, _ = lint("The task is gone.")
1565
+ assert not any(f["rule"] == "passive-voice" for f in findings), findings
1566
+ # code blocks skipped
1567
+ findings, _ = lint("```\nx = a; y = b\n```")
1568
+ assert findings == []
1569
+ # a fence closes only on the same character, at least as long, with nothing after it
1570
+ for text in ("````md\n```\n````\n\nProse; here.",
1571
+ "```\n~~~\n```\n\nProse; here.",
1572
+ "~~~~\n~~~\n~~~~\n\nProse; here."):
1573
+ findings, _ = lint(text)
1574
+ assert [f["rule"] for f in findings] == ["semicolon"], (text, findings)
1575
+ findings, _ = lint("```\n```sh\nx = a; y = b\n```")
1576
+ assert findings == [], findings
1577
+ # a fence inside a blockquote is a fence
1578
+ findings, _ = lint("Text.\n\n> ```\n> x = a; y = b\n> ```\n\nProse; here.")
1579
+ assert [(f["rule"], f["line"]) for f in findings] == [("semicolon", 7)], findings
1580
+ # the dangling-conjunction rule uses the same fence state
1581
+ findings, _ = lint("````md\n```\n- Do this and\n````")
1582
+ assert findings == [], findings
1583
+ findings, _ = lint("- Do this\n ````md\n ```\n code and\n ````\n- Do that")
1584
+ assert findings == [], findings
1585
+ # it tracks HTML blocks: there no fence opens and no list item starts
1586
+ for text, lines in (("<div>\n```\n</div>\n\n- Open the valve and\n", [5]),
1587
+ ("<!--\n```\n-->\n- Open the valve and\n", [4]),
1588
+ ("<div>\n- Open the valve and\n</div>\n", []),
1589
+ ("- x\n<div>\n- y and\n", []),
1590
+ # an HTML block in an item, also on the line of the marker
1591
+ ("- Open the valve\n\n <div>\n ```\n </div>\n\n- Close it and\n", [7]),
1592
+ ("- x\n - y\n <div>\n ```\n </div>\n\n- z and\n", [7]),
1593
+ ("- <pre>\n ```\n </pre>\n- y and\n", [4]),
1594
+ # a line with less indent ends the item, and the HTML block in it
1595
+ ("- x\n <div>\n- y and\n", [3]),
1596
+ ("- <pre>\n```\n- y and\n```\n", []),
1597
+ ("- a\n - b\n <div>\n- c and\n", [4]),
1598
+ # the lines in a <pre> block or a comment are no item text, and no
1599
+ # item starts there; GitHub hides the text after an open comment
1600
+ ("- Close it\n <pre>\n ```\n - Step and\n- Next\n", []),
1601
+ ("- a\n <!--\n x\n- b and\n", []),
1602
+ ("- a\n <?php\n x and\n- b\n", []),
1603
+ # an open comment joins the next items up to "-->"
1604
+ ("- Run tests and\n <!-- pending\n- Next:\n <!-- closes -->\n finish the checklist.\n", []),
1605
+ ("- Run tests and\n <!-- pending\n- Next\n", [1]),
1606
+ # also the first line of such a block, when the block stays open
1607
+ ("- Change summary:\n <!-- Explain the change and\n include test results.\n", []),
1608
+ ("- <pre>Expected an expression after and\n at parser.py:12\n", []),
1609
+ ("- Fix it\n <!-- old -->\n <pre>a</pre> and\n", [3]),
1610
+ # GitHub shows the text of a <script> block: it escapes the tag
1611
+ ("- a\n <script>\n x and\n- b\n", [3]),
1612
+ ("- a\n <!--\n- b and\n\n<div>\n-->\n</div>\n\n- c and\n", [9]),
1613
+ # in an item, the text of a <div> block is item text: a list marker
1614
+ # there starts an item, as before
1615
+ ("- <div>\n ```\n - Step and\n- Next\n", [3]),
1616
+ ("- a\n - b\n <div>\n - c and\n </div>\n", [4]),
1617
+ (" <!--\n ```\n- Open it and\n", []),
1618
+ # one tag alone after a text line continues the paragraph, so the
1619
+ # fence after it opens
1620
+ ("Text\n<span>\n```\n```\n- a and\n", [5]),
1621
+ ("- x\n <span>\n ```\n - y and\n ```\n", []),
1622
+ # a no-break space is no whitespace in a tag
1623
+ ("<div\u00a0class=\"note\">\n- Next and\n", [2]),
1624
+ ("<br/>\u00a0\n- Next and\n", [2]),
1625
+ # a line with a no-break space is text, also after "#"
1626
+ ("#\u00a0Notes\n<img src=\"a.png\">\n- Run checks and\n", [3]),
1627
+ ("Text\n\u00a0\n<img src=\"a.png\">\n- Run checks and\n", [4]),
1628
+ # a line with only ">" ends the paragraph, and a heading in a quote too
1629
+ (">\n<img src=\"a.png\">\n- Run checks and\n", []),
1630
+ ("> # Head\n<img src=\"a.png\">\n- Run checks and\n", []),
1631
+ # a tag alone interrupts a paragraph of a quote or of an item that
1632
+ # the line is not in, as in cmark-gfm
1633
+ ("> Note\n<img src=\"a.png\">\n- Run checks and\n", []),
1634
+ ("- a\n<span>\n- b and\n", []),
1635
+ ("- a\nlazy\n<span>\n- b and\n", []),
1636
+ ("- a\n - b\n text\n <span>\n ```\n ```\n- c and\n", [7]),
1637
+ ("- a\n - b\n\n text\n <span>\n ```\n ```\n- c and\n", [8]),
1638
+ # "2." cannot interrupt a paragraph, so the tag line is paragraph text
1639
+ ("Text\n2. Example\n<br>\n- Open the valve and\n", [4]),
1640
+ ("- a\n Text\n 2. Example\n <br>\n- Open the valve and\n", [5]),
1641
+ ("Text\n1. Example\n<br>\n- Open the valve and\n", []),
1642
+ ("Screenshot:\n2. <img src=\"example.png\">\n<br>\n- Run tests and\n", [4]),
1643
+ ("Text\n2. # Heading\n<br>\n- Run tests and\n", [4]),
1644
+ # an empty item cannot interrupt a paragraph; an empty "-" is a setext underline
1645
+ ("Text\n1. \n<br>\n- Run tests and\n", [4]),
1646
+ ("Text\n* \n<br>\n- Run tests and\n", [4]),
1647
+ ("Text\n- \n<br>\n- Run tests and\n", []),
1648
+ # its lines are paragraph text: no HTML block starts in them
1649
+ ("Example:\n2. Steps\n <details>\n ```markdown\n- Run tests and\n ```\n"
1650
+ " </details>\n", []),
1651
+ ("Text\n2. Steps and\n <details>\n", []),
1652
+ ("Text\n2. Steps\n <details>\n- Run tests and\n", [4]),
1653
+ # a setext underline ends the paragraph, so "2." starts a list
1654
+ ("Setup\n=====\n2. Install\n - Linux\n <div>Note</div>\n- Run tests and\n", [6]),
1655
+ # a lazy underline is text, and "<br>" after the "2." item is a block
1656
+ ("- Setup\n=====\n2. Install\n<br>\n- Run tests and\n", []),
1657
+ # an HTML block in an item goes on after a list marker in it
1658
+ ("- <div>\n ~~~\n - Note\n ~~~\n </div>\n\n- Open the valve and\n", [7]),
1659
+ ("- code\n <div>Note</div>\n- Open the valve and\n", [3]),
1660
+ ("1. <details>\n ~~~\n - Linux\n ~~~\n </details>\n\n2. Run tests and\n", [7]),
1661
+ ("- <div>\n - Note\n\n ~~~\n - x and\n ~~~\n- y and\n", [7]),
1662
+ ("- <div>\n ~~~\n - Note\n ~~~\n </div>\n\n- Open the valve and\n", [7]),
1663
+ # a block starts at the indent of the item that holds the line
1664
+ ("- Example:\n - Linux\n <div>\n ~~~\n </div>\n\n <div>\n ~~~\n </div>\n\n"
1665
+ "- Run tests and\n", [11]),
1666
+ # after the block, the lines are in the item that held the block
1667
+ ("- <div>\n ~~~\n - Note\n </div>\n\n <div>\n ~~~\n </div>\n\n- Run tests and\n",
1668
+ [10]),
1669
+ ("- <div>\n ~~~\n - Note\n\n <div>\n ~~~\n </div>\n\n- Run tests and\n", [9]),
1670
+ # a list marker on the line that ends the block is text of the block
1671
+ ("- <textarea>\n ~~~\n - Note </textarea>\n\n <div>\n ~~~\n </div>\n\n"
1672
+ "- Run tests and\n", [9]),
1673
+ ("- <pre>\n ~~~\n - Note </pre>\n\n <div>\n ~~~\n </div>\n\n- Run tests and\n", [9])):
1674
+ findings, _ = lint(text)
1675
+ assert [f["line"] for f in findings if f["rule"] == "dangling-conjunction"] == lines, (text, findings)
1676
+ # a quoted fence ends with its quote, and a deeper ">" in a fence is code
1677
+ for text, lines in (("> ```\n> code\n\nProse; here.", [4]),
1678
+ ("```\n> ```\nx; y\n```\n\nProse; here.", [6]),
1679
+ ("> > ```\n> > x; y\n> > ```\n\nProse; here.", [5]),
1680
+ ("> ```\n> ```\n> x; y\n> ```\n\nProse; here.", [6]),
1681
+ ("```foo`bar\nProse; here.", [2]),
1682
+ ("```\nx; y\n```` \t\n\nProse; here.", [5])):
1683
+ findings, _ = lint(text)
1684
+ assert [f["line"] for f in findings if f["rule"] == "semicolon"] == lines, (text, findings)
1685
+ # the end of a quoted fence in a list item does not join the next line to the item
1686
+ findings, _ = lint("- Item\n > ```\n > code\nProse and")
1687
+ assert findings == [], findings
1688
+ # a tab is indent up to the next tab stop of 4, from its real column, also after ">"
1689
+ for text, line in (("\t```\n\tx = 1\n\t```\n\nProse; here.", 5),
1690
+ ("```\nx\n\t```\ny; z\n```\n\nProse; here.", 7),
1691
+ ("> ```\n> x\n>\t```\n> Use a; b.", 4),
1692
+ ("> ```\n> x\n> \t```\n> Use a; b.", 4),
1693
+ (">\t> ```\n>\t> x; y\n>\t> ```\n\nProse; here.", 5),
1694
+ ("- Item\n ```\n\t```\n\nProse; here.", 5)):
1695
+ findings, _ = lint(text)
1696
+ assert [(f["rule"], f["line"]) for f in findings] == [("semicolon", line)], (text, findings)
1697
+ # a tab-indented fence stays in its list item
1698
+ findings, _ = lint("- Run the build and\n\t```\n\tmake\n\t```\n check the log.")
1699
+ assert findings == [], findings
1700
+ # a ">" line in an HTML block is text, not a quoted fence. A blank line ends a
1701
+ # block that starts with a tag, and a quoted fence after it is a fence.
1702
+ for text, lines in (("<div>\n> ```\n> Prose; here.\n> ```\n</div>", [3]),
1703
+ ("<custom>\n> ```\n> Prose; here.\n> ```\n</custom>", [3]),
1704
+ ("<!--\n> ```\n> Prose; here.\n> ```\n-->", [3]),
1705
+ ("<div>\n\n> ```\n> x; y\n> ```\n\nProse; here.", [7]),
1706
+ ("<details>\n\n> ```\n> a; b\n> ```\n\n</details>\n\nProse; here.", [9]),
1707
+ ("<pre>\n> ```\n> Prose; here.\n> ```\n</pre>", [3]),
1708
+ ("<?php\n> ```\n> Prose; here.\n> ```\n?>", [3]),
1709
+ ("<![CDATA[\n> ```\n> Prose; here.\n> ```\n]]>", [3]),
1710
+ # inline HTML is no HTML block: the quoted fence after it is a fence
1711
+ ("<span>Label</span>\n> ```\n> a; b\n> ```", []),
1712
+ ("<kbd>Enter</kbd> starts the job.\n> ```\n> a; b\n> ```", []),
1713
+ ("Text.\n<custom>\n> ```\n> a; b\n> ```", []),
1714
+ # a type 7 tag after a heading or a fence starts a block
1715
+ ("# Title\n<custom>\n> ```\n> Prose; here.\n> ```\n</custom>", [4]),
1716
+ ("```\nx\n```\n<img src=\"a.png\">\n> ```\n> Prose; here.\n> ```", [6]),
1717
+ # attribute values may hold "<" and ">", an invalid attribute is no tag
1718
+ ('<span title="a > b">\n> ```\n> Prose; here.\n> ```', [3]),
1719
+ ("<custom a=>\n> ```\n> a; b\n> ```", []),
1720
+ # a setext underline, an empty quote line and a thematic break of one
1721
+ # character end a paragraph, so a type 7 tag after them starts a block
1722
+ ("Title\n=====\n<custom>\n> ```\n> Prose; here.\n> ```\n</custom>", [5]),
1723
+ ("> Quote.\n>\n<custom>\n> ```\n> Prose; here.\n> ```\n</custom>", [5]),
1724
+ ("-*-\n<custom>\n> ```\n> a; b\n> ```", []),
1725
+ ("Title\n-\n<custom>\n> ```\n> Prose; here.\n> ```\n</custom>", [5]),
1726
+ ("> Quote.\n> >\n<custom>\n> ```\n> Prose; here.\n> ```\n</custom>", [5]),
1727
+ # an underline with no paragraph, or as a lazy line, is text
1728
+ ("Intro.\n\n==\n<custom>\n> ```\n> a; b\n> ```", []),
1729
+ ("> Title\n===\n<custom>\n> ```\n> a; b\n> ```", []),
1730
+ ("> > Title\n> lazy\n> ===\n<custom>\n> ```\n> a; b\n> ```", []),
1731
+ # a quoted heading or thematic break is no paragraph line
1732
+ ("> # Note\n> ===\n<custom>\n> ```\n> a; b\n> ```", []),
1733
+ ("> Note\n> ***\n<custom>\n> ```\n> Prose; here.\n> ```\n</custom>", [5]),
1734
+ # the line that ends a quoted fence can start an HTML block
1735
+ ("> ```\n> x\n<custom>\n> ```\n> Prose; here.", [5]),
1736
+ # a type 1 name is no type 7 tag
1737
+ ("<pre/>\n> ```\n> a; b\n> ```", []),
1738
+ ("</pre>\n> ```\n> a; b\n> ```", []),
1739
+ ("<pre-x>\n> ```\n> Prose; here.\n> ```", [3]),
1740
+ # no fence opens in an HTML block, also without ">"
1741
+ ("<pre>\n```\nsample\n</pre>\n\nThe paragraph is here; it must be checked.", [6]),
1742
+ ("<div>\n```\n\nProse; here.", [4]),
1743
+ ("<DIV class=x>text\n> ```\n> Prose; here.\n> ```", [3])):
1744
+ findings, _ = lint(text)
1745
+ assert [f["line"] for f in findings if f["rule"] == "semicolon"] == lines, (text, findings)
1746
+ # a fence at column 0 ends a list item
1747
+ findings, _ = lint("- Do this and\n```\nx\n```\n done")
1748
+ assert [(f["rule"], f["line"]) for f in findings] == [("dangling-conjunction", 1)], findings
1749
+ # a quoted fence also ends on a line with fewer quote levels that is not blank
1750
+ for text, line in (("> ```\n> x\nProse; here.", 3), ("> > ```\n> x; y", 2)):
1751
+ findings, _ = lint(text)
1752
+ assert [(f["rule"], f["line"]) for f in findings] == [("semicolon", line)], (text, findings)
1753
+ # a closing fence up to three columns deeper than the opening fence closes it
1754
+ findings, _ = lint("```\nx; y\n ```\n\nProse; here.")
1755
+ assert [(f["rule"], f["line"]) for f in findings] == [("semicolon", 5)], findings
1756
+ # a tilde fence opens with a backtick in its info string
1757
+ findings, _ = lint("~~~ a`b\nx; y\n~~~\n\nProse; here.")
1758
+ assert [(f["rule"], f["line"]) for f in findings] == [("semicolon", 5)], findings
1759
+ # an alert marker after a fence in the same quote is text, as GitHub shows it
1760
+ body = " ".join(["word"] * 25) + "."
1761
+ findings, _ = lint(f"> ```\n> x\n> ```\n> [!NOTE]\n> {body}")
1762
+ assert [f["rule"] for f in findings] == ["long-sentence"], findings
1763
+ # a quoted fence after a list item does not join the item
1764
+ findings, _ = lint("- Do this and\n\n> ```\n> code\n> ```\n done")
1765
+ assert [f["rule"] for f in findings] == ["dangling-conjunction"], findings
1766
+ # all supported list markers, case variants, and trailing whitespace
1767
+ findings, _ = lint(
1768
+ "- Confirm the target and\n"
1769
+ "* Record the result OR \n"
1770
+ "+ Close the panel\n"
1771
+ "1. Start the task and\n"
1772
+ "2) Stop the task OR"
1773
+ )
1774
+ dangling = [f for f in findings if f["rule"] == "dangling-conjunction"]
1775
+ assert len(dangling) == 4, dangling
1776
+ assert [f["line"] for f in dangling] == [1, 2, 4, 5], dangling
1777
+ assert [f["col"] for f in dangling] == [1, 1, 1, 1], dangling
1778
+ assert all(f["level"] == "advisory-free" for f in dangling), dangling
1779
+
1780
+ # valid continuation lines and standalone four-space code are ignored
1781
+ findings, _ = lint(" - Confirm the target and\n record the result.")
1782
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1783
+ findings, _ = lint("- Confirm the target and\n\trecord the result.")
1784
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings), findings
1785
+ findings, _ = lint("- Confirm the target\n and")
1786
+ dangling = [f for f in findings if f["rule"] == "dangling-conjunction"]
1787
+ assert len(dangling) == 1 and dangling[0]["line"] == 2, dangling
1788
+ assert dangling[0]["col"] == 3, dangling
1789
+ findings, _ = lint(" - code and")
1790
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1791
+ findings, _ = lint("> - Confirm the target and\n> - Record the result or")
1792
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1793
+ findings, _ = lint("- Do this and\n~~~\ncode and\n~~~")
1794
+ dangling = [f for f in findings if f["rule"] == "dangling-conjunction"]
1795
+ assert len(dangling) == 1 and dangling[0]["line"] == 1, dangling
1796
+ findings, _ = lint("```text\n- code and\n```")
1797
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1798
+
1799
+ # one- and three-space markers and ordered continuation width
1800
+ findings, _ = lint(" - Start the task and\n record the result.")
1801
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1802
+ findings, _ = lint("- Start the task and\n record the result.")
1803
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1804
+ findings, _ = lint("-\tStart the task and")
1805
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1806
+ findings, _ = lint(" - Start the task and", filename="fixture.md")
1807
+ dangling = [f for f in findings if f["rule"] == "dangling-conjunction"]
1808
+ assert len(dangling) == 1 and dangling[0]["col"] == 4, dangling
1809
+ assert dangling[0]["file"] == "fixture.md"
1810
+ assert dangling[0]["match"].endswith("and")
1811
+ assert "Complete the item" in dangling[0]["message"]
1812
+ findings, _ = lint("100. Start the task and\n unrelated text")
1813
+ dangling = [f for f in findings if f["rule"] == "dangling-conjunction"]
1814
+ assert len(dangling) == 1, dangling
1815
+ findings, _ = lint("- Start the task and.\n- Stop the task or,")
1816
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1817
+ findings, _ = lint("- Start the task and\n\n record the result.")
1818
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1819
+ findings, _ = lint("- Parent item and\n - Nested item or")
1820
+ dangling = [f for f in findings if f["rule"] == "dangling-conjunction"]
1821
+ assert [f["line"] for f in dangling] == [1, 2], dangling
1822
+
1823
+ # ordinary prose, inline code, and fenced code are ignored
1824
+ findings, _ = lint("The process may include steps and")
1825
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1826
+ findings, _ = lint("- Use `and` as a label")
1827
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1828
+ findings, _ = lint("- Combine `left` and `right`")
1829
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings), findings
1830
+ findings, _ = lint("~~~\n- code and\n~~~")
1831
+ assert not any(f["rule"] == "dangling-conjunction" for f in findings)
1832
+ findings, _ = lint(("word " * 30).strip() + ".")
1833
+ assert any(f["rule"] == "long-sentence" for f in findings)
1834
+ # Markdown table syntax is layout, not prose. Each cell stays lintable.
1835
+ short_cell = " ".join(f"term{number}" for number in range(1, 25)) + "."
1836
+ for table in (
1837
+ "| Label | Detail |\n"
1838
+ "| --- | --- |\n"
1839
+ f"| Clear | {short_cell} |",
1840
+ "Label | Detail\n"
1841
+ "--- | ---\n"
1842
+ f"Clear | {short_cell}"):
1843
+ findings, words_total = lint(table)
1844
+ assert not any(f["rule"] == "long-sentence" for f in findings), findings
1845
+ assert words_total == 27, words_total
1846
+ # GFM also accepts one-dash separators, one column, and ragged body rows.
1847
+ short_row = " ".join(f"term{number}" for number in range(1, 9))
1848
+ for table in (
1849
+ "| Label | Detail |\n|-|-|\n" + f"| {short_row} | {short_row} |\n" * 5,
1850
+ "| Label |\n| --- |\n" + f"| {short_row} |\n" * 8,
1851
+ "| Label | Detail |\n| --- | --- |\n| x |\n"
1852
+ + f"| {short_row} | {short_row} | extra |\n" * 2):
1853
+ findings, _ = lint(table)
1854
+ assert not any(f["rule"] == "long-sentence" for f in findings), (table, findings)
1855
+ long_cell = " ".join(f"term{number}" for number in range(1, 27)) + "."
1856
+ findings, _ = lint(
1857
+ "| Label | Detail |\n"
1858
+ "| --- | --- |\n"
1859
+ f"| Clear | {long_cell} |"
1860
+ )
1861
+ long_sentences = [f for f in findings if f["rule"] == "long-sentence"]
1862
+ assert len(long_sentences) == 1, long_sentences
1863
+ assert long_sentences[0]["match"] == "26 words", long_sentences
1864
+ # synonym rotation: second member flagged, first named as the keeper
1865
+ findings, _ = lint("Check the config file. Then verify the output. Verify twice.")
1866
+ rot = [f for f in findings if f["rule"] == "synonym-rotation"]
1867
+ assert len(rot) == 1 and "'verify' and 'check'" in rot[0]["message"], rot
1868
+ # single consistent term: no flag
1869
+ findings, _ = lint("Check the config. Check the output.")
1870
+ assert not any(f["rule"] == "synonym-rotation" for f in findings)
1871
+ # character references and link targets are markup, not prose
1872
+ findings, _ = lint("Press Ctrl&nbsp;C to stop &rarr; &#8594; &#x2192; now. "
1873
+ "See [the docs](https://x.io/a;b) or <https://x.io/a;b>.")
1874
+ assert not any(f["rule"] == "semicolon" for f in findings), findings
1875
+ findings, _ = lint('See [a](https://x.io/a;b "Title") and [b](<a b;c>) and [c](a(b;c)).')
1876
+ assert not any(f["rule"] == "semicolon" for f in findings), findings
1877
+ # a reference GitHub does not read as one, or that renders ";", is prose
1878
+ for text in ("We closed the Q&As; then we left.", "Use &unknownentity; here.",
1879
+ "Read this&#59; then stop.", "Read this&#x3b; then stop.", "Read&semi; this."):
1880
+ findings, _ = lint(text)
1881
+ assert [f["rule"] for f in findings] == ["semicolon"], (text, findings)
1882
+ # text in parentheses that is no link destination is prose
1883
+ findings, _ = lint("Read [this](seamless; now).")
1884
+ assert [f["rule"] for f in findings] == ["marketing-adjective", "semicolon"], findings
1885
+ for text in ("See [a](b;c d).", "See [a](b(c;d).", "See [a](<b;c).", "See [a](b;c \"t\" x)."):
1886
+ findings, _ = lint(text)
1887
+ assert [f["rule"] for f in findings] == ["semicolon"], (text, findings)
1888
+ # the link can go on in the next lines of its paragraph, and it must close there
1889
+ for text, cols in (("See [docs](https://x.io/a;b\n).", []),
1890
+ ("See [a](b;c \"multi\nline title\") now.", []),
1891
+ ("See [a](b;c\n\"title\") now.", []),
1892
+ ("See [a](b;c (paren\ntitle)) now.", []),
1893
+ ("> See [docs](https://x.io/a;b\n> ).", []),
1894
+ ("See [docs](https://x.io/a;b\n\n).", [(1, 26)]),
1895
+ ("See [a](b;c\n\n\"title\") now.", [(1, 10)]),
1896
+ ("See [a](b;c\nnext line) now.", [(1, 10)]),
1897
+ ("See [a](b(c);d \"oops).", [(1, 13)]),
1898
+ ("See [a](b(c);d \"oops).\n\nNext; para.", [(1, 13), (3, 5)]),
1899
+ ("See [a](b;c (paren (no)) now.", [(1, 10)]),
1900
+ # a block that interrupts the paragraph ends the link
1901
+ ("See [a](b;c \"x\n- y\") z.", [(1, 10)]),
1902
+ ("See [a](b;c \"x\n# y\") z.", [(1, 10)]),
1903
+ ("See [a](b;c \"x\n===\") z.", []),
1904
+ ("See [a](b;c \"x\n===\n\") z.", [(1, 10)]),
1905
+ ("See [a](b;c\n> )", [(1, 10)]),
1906
+ ("> See [a](b;c\n)", []),
1907
+ ("See [a](b;c \"x\n2. y\") z.", []),
1908
+ # a backtick in a destination is text, and what starts first wins
1909
+ ("See [a](https://x.io/a;b`c`tail).", []),
1910
+ ("See [x](a`b;c) and `d` e; f.", [(1, 25)]),
1911
+ ("Run `a [b](c;d` now.", []),
1912
+ ("A `` [a](b;c `` d) e; f", [(1, 21)]),
1913
+ ("| A | B |\n|-|-|\n| [a](b;c`d`e) | x |", []),
1914
+ # a lazy line continues the quoted paragraph of the link
1915
+ ("> First line\nSee [a](b;c\n> ) now.", []),
1916
+ # a backtick in a title is text, a code span after the link is a span
1917
+ ("See [a](b \"x`y\") and `z; w` now.", []),
1918
+ # in a table cell the link starts first, and an HTML block has no links
1919
+ ("| A | B |\n|-|-|\n| [a](b`c) d; e` f | x |", [(3, 13)]),
1920
+ ("<div>\nSee [a](b;c) now.\n</div>", [(2, 10)]),
1921
+ # a heading does not go on in the next line
1922
+ ("# See [a](b;c\n) now.", [(1, 12)]),
1923
+ # a title that goes on in the next line holds its backticks there too
1924
+ ("See [a](b \"x\ny`z\") and `w; v` q.", []),
1925
+ ("> See [a](b \"x\n> y\n> `z\") and `w; v` q.", []),
1926
+ ("See [a](b \"x\ny\") a; `b` c", [(2, 6)]),
1927
+ # a backtick that no code span holds is text, also after a link
1928
+ ("See [a](b \"y`z\") ``c ` d`` text; last `.", [(1, 32)]),
1929
+ ("See [a](b \"x\ny`z\") ``c ` d`` text; last `.", [(2, 21)]),
1930
+ ("Run ```a; b```` now.", [(1, 9)]),
1931
+ # "](" and a space start no destination
1932
+ ("]( ;)", [(1, 4)]),
1933
+ ("]( `);`", []),
1934
+ ("`[`]( ;)", [(1, 7)]),
1935
+ ("<aa:;[]( )>", [(1, 5)]),
1936
+ # an escaped backtick is text, and the rest of its run can open a span
1937
+ ("\\```;``", []),
1938
+ ("\\```;```", [(1, 5)]),
1939
+ ("> \\```;``", []),
1940
+ ("| A |\n|-|\n| \\```;`` |", []),
1941
+ # GitHub shows a link title in no text, so it is not prose
1942
+ ("[a](b \"`;`\")", []),
1943
+ ("[a](b \"`\n`;`\")", []),
1944
+ ("[a](b \"`;\")`", []),
1945
+ ("See [a](b \"x; y\") now; stop.", [(1, 22)]),
1946
+ ("See [a](b \"x\ny; z\nw\") now; stop.", [(3, 8)]),
1947
+ # an escaped "]" starts no link, and an autolink that starts first holds its backticks
1948
+ ("[label\\](url \"visible; prose\")", [(1, 22)]),
1949
+ ("[a\\](b;c) d", [(1, 7)]),
1950
+ ("<https://example.com/\\``> visible; prose `.", [(1, 34)]),
1951
+ ("<https://x.io/`a> b; c `.", [(1, 20)]),
1952
+ ("See <https://x.io/a`b> and `c; d` e.", []),
1953
+ ("`a <https://x` b; c>", [(1, 17)]),
1954
+ # "](" is a link only after a "[" of its paragraph that no "]" closed
1955
+ ("](b \";\")", [(1, 6)]),
1956
+ ("](b;c) d", [(1, 4)]),
1957
+ ("](b \"`;`\")", []),
1958
+ ("\\[a](b \";\")", [(1, 9)]),
1959
+ ("[a] ](b \";\")", [(1, 10)]),
1960
+ ("[a](x) ](b \";\")", [(1, 13)]),
1961
+ ("[a <b>](c \";\") d", []),
1962
+ ("[a\n](b \";\")", []),
1963
+ ("> [a\n> ](b \";\")", []),
1964
+ ("- [a\n ](b \";\")", []),
1965
+ ("[a\n\n](b \";\")", [(3, 6)]),
1966
+ # an image can be in a link; a link in a link is text
1967
+ ("[![x](y)](z \";\")", []),
1968
+ ("![a [b](c) d](e \";\") f", []),
1969
+ ("[a [b](c) d](e \";\") f", [(1, 17)]),
1970
+ # with no destination, the quoted text is the destination: no title
1971
+ ("[]( \"\n;\")", [(2, 1)]),
1972
+ # an autolink or an inline tag starts first; an escaped "<" starts neither
1973
+ ("\\<aa:`>;`", []),
1974
+ ("<aa:[a](b> \";\")", [(1, 13)]),
1975
+ ("<span title=\"`\"> text; `", [(1, 22)]),
1976
+ ("<span title=\"\\``\"> text; `", [(1, 24)]),
1977
+ ("<b title='`'>`c; d`", []),
1978
+ # GitHub shows no attribute of a tag
1979
+ ("x <b a=\"`;`\">", []),
1980
+ ("<a href=\"](x ';')\"> a; b", [(1, 22)]),
1981
+ ("x <b a=\";<b a=\"</b> y", [(1, 9)]),
1982
+ # GitHub shows a tag that its tagfilter escapes as text
1983
+ ("x <title a=\"x;y\"> b", [(1, 14)]),
1984
+ # a comment, a processing instruction, a declaration and CDATA hold their brackets
1985
+ ("[a <!-- ] --> b](c \"d;\")", []),
1986
+ ("[x <!-- [a](b) --> y](c \"d;\")", []),
1987
+ ("[a <? ] ?> b](c \"d;\")", []),
1988
+ ("[a <!X ] > b](c \"d;\")", []),
1989
+ ("[a <![CDATA[ ] ]]> b](c \"d;\")", []),
1990
+ ("x <!-- <a b=\" --> c; <d e=\"> f", [(1, 20)]),
1991
+ ("x <!-- ](a;b) --> y", []),
1992
+ ("x <? a; b ?> y", []),
1993
+ ("x <? a> b; c ?> d", [(1, 10)]),
1994
+ ("x <![CDATA[ a> b; c ]]> d", [(1, 17)])):
1995
+ findings, _ = lint(text)
1996
+ assert [(f["line"], f["col"]) for f in findings if f["rule"] == "semicolon"] == cols, (text, findings)
1997
+ findings, _ = lint("Press [stop](https://x.io); then wait. Use & ; here.")
1998
+ assert [f["col"] for f in findings if f["rule"] == "semicolon"] == [27, 46], findings
1999
+ # an indented code block is code, also in a quote and under a list item
2000
+ def semicolon_lines(text):
2001
+ return [f["line"] for f in lint(text)[0] if f["rule"] == "semicolon"]
2002
+ assert semicolon_lines("Intro text.\n\n x = 1; y = 2\n\tz; w") == []
2003
+ assert semicolon_lines("> Intro.\n>\n> x = 1; y = 2") == []
2004
+ assert semicolon_lines("- Item.\n\n x = 1; y = 2") == []
2005
+ # it cannot interrupt a paragraph, and a list item's own indent is prose
2006
+ assert semicolon_lines("Intro text\n goes on; here.") == [2]
2007
+ assert semicolon_lines("- Item.\n\n More; text.") == [3]
2008
+ assert semicolon_lines("1. Item.\n\n - Sub.\n\n More; text.") == [5]
2009
+ # the list ends at a line with less indent
2010
+ assert semicolon_lines("- Item.\n\nText.\n\n x; y") == []
2011
+ # a list item that follows a text line, or a nested item in a tight list, still
2012
+ # sets the content indent, so its indented continuation paragraph is prose
2013
+ assert semicolon_lines("Steps to follow:\n- Open the file.\n\n Then save it; close it.\n") == [4]
2014
+ assert semicolon_lines("- Open the file.\n - Pick a row.\n\n Then save it; close it.\n") == [4]
2015
+ assert semicolon_lines("- Item\ncontinued\n\n More; text.") == [4]
2016
+ assert semicolon_lines("Intro text.\n\n x = 1; y = 2\n") == []
2017
+ assert semicolon_lines("Steps:\n- Open the file.\n\n x = 1; y = 2\n") == []
2018
+ # in a paragraph, only a bullet or "1." with text starts a list, so its indent
2019
+ # does not apply after "2." or an empty bullet
2020
+ assert semicolon_lines("Intro\n2. item\n\n x = 1; y = 2") == []
2021
+ assert semicolon_lines("Intro\n*\n\n x = 1; y = 2") == []
2022
+ assert semicolon_lines("Intro\n1. item\n\n Then save it; close it.") == [4]
2023
+ # after a quote, any number starts a list; a thematic break and a number of ten
2024
+ # digits start none, and a very long number does not stop the run
2025
+ assert semicolon_lines("> Intro\n2. item\n\n Then save it; close it.") == [4]
2026
+ assert semicolon_lines("Intro text.\n* * *\n x = 1; y = 2") == []
2027
+ assert semicolon_lines("Text\n* * *\n\n x = 1; y = 2") == []
2028
+ assert semicolon_lines("* * *\n cont; `\n2. it; `") == [2]
2029
+ assert semicolon_lines("1234567890. item\n\n x = 1; y = 2") == []
2030
+ assert semicolon_lines("Intro\n" + "1" * 5000 + ". digits; here.") == [2]
2031
+ # a lazy line keeps the nested item open, so its paragraph after a blank line is prose
2032
+ assert semicolon_lines("- outer\n - nested\n outer again\n\n Save it; close it.") == [5]
2033
+ # a code span may wrap across lines of one paragraph
2034
+ assert semicolon_lines("Run `a;\nb; c` now; then") == [2]
2035
+ assert semicolon_lines("Run ``a`;\nb;`` now.") == []
2036
+ # a run that nothing closes in the paragraph is literal text
2037
+ assert semicolon_lines("A ` mark; here.\n\nAnd ` there; too.") == [1, 3]
2038
+ # a code span never crosses a block boundary: a new list item, a heading,
2039
+ # a fence, a quote-only line, or a change of quote depth
2040
+ assert semicolon_lines("- Press the ` key; it opens.\n- Run `x` now.\n") == [1]
2041
+ assert semicolon_lines("Press the ` key; it opens.\n# Run `x` now\n") == [1]
2042
+ assert semicolon_lines("Type ` to start; then stop.\n```\ncode\n```\nThe ` mark ends it.\n") == [1]
2043
+ assert semicolon_lines("> A ` mark; here.\n>\n> And ` there; too.\n") == [1, 3]
2044
+ assert semicolon_lines("# Title ` x\nText; with ` more\n") == [2]
2045
+ assert semicolon_lines("A ` mark; here.\n---\nAnd ` there; too.") == [1, 3]
2046
+ assert semicolon_lines("> A ` mark; here.\n> > And ` there; too.") == [1, 2]
2047
+ # a span still wraps inside a list item and a quote, also with a double run
2048
+ assert semicolon_lines("Run `a;\nb` now.\n") == []
2049
+ assert semicolon_lines("- Run `a;\n b` now.\n") == []
2050
+ assert semicolon_lines("> Run `a;\n> b` now.\n") == []
2051
+ assert semicolon_lines("Run ``a`;\nb;`` now.\n- Next `x;` y.") == []
2052
+ # a line ends a span only where it ends the paragraph: "#" with no space is text,
2053
+ # a lazy line with fewer ">" continues the quote, "2." cannot start a list in a
2054
+ # paragraph, and inline HTML is text
2055
+ for text in ("A `foo;\n#bar`; done.", "> Run `code;\ncontinued` text; here.",
2056
+ "Run `code;\n2. continued` text; here.",
2057
+ "Run `code;\n#hashtag continued` text; here.",
2058
+ "Run `code;\n<span>continued` text; here.",
2059
+ "> > Run `code;\n> lazy\n> > continued` text; here."):
2060
+ assert semicolon_lines(text) == [text.count("\n") + 1], text
2061
+ # a next item of the open list, a setext underline, and an HTML block end the span
2062
+ assert semicolon_lines("1. Run `a;\n2. b` now; x") == [1, 2]
2063
+ assert semicolon_lines("Run `a;\n===\nb` now; x") == [1, 3]
2064
+ assert semicolon_lines("Run `a;\n<div>\n\nb` now; x") == [1, 4]
2065
+ assert semicolon_lines("Run `a;\n<div>\nb` now; x") == [1, 3]
2066
+ assert semicolon_lines("# Run `a;\nb` x;") == [1, 2]
2067
+ # a lazy underline, a fence or an item that is indented code, and a tab after ">"
2068
+ assert semicolon_lines("> Run `a;\n===\nb` x;") == [3]
2069
+ assert semicolon_lines("- Run `a;\n===\nb` now; x") == [3]
2070
+ assert semicolon_lines("1. Press the ` key; it opens.\n===\nUse `x` here.") == []
2071
+ assert semicolon_lines("- item; `\n===\ntext; `") == [1]
2072
+ # a thematic break is no lazy line: it ends the item, so deeper text after it is code
2073
+ assert semicolon_lines("- Item; here.\n* * *\n x = 1; y = 2") == [1]
2074
+ assert semicolon_lines("Run `a;\n ```\nb` x;") == []
2075
+ assert semicolon_lines("Run `a;\n - b` x;") == [2]
2076
+ assert semicolon_lines("> Run `a;\n>\t- b` x;") == [1, 2]
2077
+ # any list item after a quote ends the quote: it is no lazy line
2078
+ assert semicolon_lines("> Press the ` key; it opens.\n2. Run `x` now; then stop.") == [1, 2]
2079
+ assert semicolon_lines("> Run `a;\n*\nb` x;") == [1, 3]
2080
+ assert semicolon_lines("> Quote text.\n- Press the ` key; it opens.\n> Run `x` now.") == [2]
2081
+ assert semicolon_lines("> > t w1; `\n> - q w2; `\n> > t w3; `") == [1, 2, 3]
2082
+ # each paragraph gets its own run index; a span in a quote uses the quote depth
2083
+ assert semicolon_lines("Run `a;\nb` x\n\nRun `c;\nd` y") == []
2084
+ assert semicolon_lines("> Run `a\n> b; c` now") == []
2085
+ # one scan of the paragraph finds the last line of each run length
2086
+ assert _paragraph_runs(["x ` y", "a `` b ` c", "", "e ` f"], 0, 0, 0) == (2, {1: 1, 2: 1})
2087
+ assert _mask_code_spans("a ` b;", 0, lambda run: False) == ("a ` b;", 0)
2088
+ assert _mask_code_spans("a ` b;", 0, lambda run: run == 1) == ("a ", 1)
2089
+ # per-file labels
2090
+ findings, _ = lint("a; b", filename="x.md")
2091
+ assert findings[0]["file"] == "x.md"
2092
+ # inline code is masked, not deleted, so columns match the source
2093
+ findings, _ = lint("Run `make all` then spin up the node.")
2094
+ phrasal = [f for f in findings if f["rule"] == "phrasal-verb"]
2095
+ assert len(phrasal) == 1 and phrasal[0]["col"] == 21, phrasal
2096
+ # a sentence ends after closing quotes and brackets
2097
+ twenty = " ".join(["word"] * 20)
2098
+ findings, _ = lint(f'He asked "{twenty}?" Then {twenty} ended. (It {twenty}.) Next.')
2099
+ assert not any(f["rule"] == "long-sentence" for f in findings), findings
2100
+ # abbreviations do not end a sentence
2101
+ findings, _ = lint(f"Use a tool, e.g. the {twenty} linter here now.")
2102
+ assert any(f["rule"] == "long-sentence" for f in findings), findings
2103
+ findings, _ = lint(f"Compare A vs. B in the {twenty} test.")
2104
+ assert any(f["rule"] == "long-sentence" for f in findings), findings
2105
+ # a sentence wrapped across lines is counted once, at its start
2106
+ findings, _ = lint(f"Intro text here.\nThe {twenty}\nwrapped tail words go here.")
2107
+ long_sentences = [f for f in findings if f["rule"] == "long-sentence"]
2108
+ assert len(long_sentences) == 1, long_sentences
2109
+ assert (long_sentences[0]["line"], long_sentences[0]["col"]) == (2, 1), long_sentences
2110
+ findings, _ = lint(f"- Item {twenty}\n continued in the same item.")
2111
+ assert any(f["rule"] == "long-sentence" for f in findings), findings
2112
+ # leading front matter is skipped except its description; line numbers after it stay true
2113
+ findings, words_total = lint(f"---\nname: x\ndescription: {twenty} {twenty}.\n"
2114
+ "tags:\n - a and\n---\nThe job is robust.")
2115
+ assert [(f["line"], f["rule"]) for f in findings] == [
2116
+ (3, "long-sentence"), (7, "marketing-adjective")], findings
2117
+ assert words_total == 44, words_total
2118
+ # the description is one paragraph, in a quoted, plain or block scalar
2119
+ for text, expected in (
2120
+ ("---\nname: x\ndescription: Applies to `thor/` only; the rules are in a file.\n---\n\nText.\n",
2121
+ [(3, 37, "semicolon")]),
2122
+ ('---\ndescription: "Use it; then stop."\n---\n', [(2, 21, "semicolon")]),
2123
+ ("---\ndescription: >-\n Use it\n robustly; then stop.\nname: x\n---\n",
2124
+ [(4, 5, "marketing-adjective"), (4, 13, "semicolon")]),
2125
+ ("---\ndescription: Use `a\n b; c` now; stop.\n---\n", [(3, 12, "semicolon")]),
2126
+ ("---\ndescription: ```\n---\n\nProse; here.\n", [(5, 6, "semicolon")]),
2127
+ ("---\n# note\nname: x\n---\n", []),
2128
+ ("---\nname: x\n# note; here\nsummary: a; b\n---\n", []),
2129
+ # a comment before the first key, and a comment after the value
2130
+ ("---\n# metadata\nname: robust-tool\ndescription: Read the file.\n---\n", []),
2131
+ ("---\ndescription: Read the file. # robust; note\n---\n", []),
2132
+ ("---\ndescription: \"Read the file.\" # robust; note\n---\n", []),
2133
+ ("---\ndescription: Use it\n # robust; note\n---\n", []),
2134
+ ("---\ndescription: \"Use it\n across; lines\" # robust\n---\n", [(3, 9, "semicolon")]),
2135
+ # a link title that goes on in the next lines of the description
2136
+ ("---\ndescription: See [a](b \"x\n y`z\") `w; v` q.\n---\n", []),
2137
+ ("---\ndescription: See [a](b \"x\n y\n z`w\") `p; q` r.\n---\n", []),
2138
+ ("---\ndescription: See [a](b \"x\n y\") c; d.\n---\n", [(3, 8, "semicolon")]),
2139
+ ("---\ndescription: See [a](b \"x\n y`z\") ``c ` d`` text; last `.\n---\n", [(3, 23, "semicolon")]),
2140
+ ("---\ndescription: ]( ;)\n---\n", [(2, 17, "semicolon")]),
2141
+ ("---\ndescription: See [a\n ](b \";\") c; d.\n---\n", [(3, 13, "semicolon")]),
2142
+ ("---\ndescription: \\```;``\n---\n", []),
2143
+ # a "#" in quotes, in a block scalar, or inside a word is text
2144
+ ("---\ndescription: \"Read # the; file.\"\n---\n", [(2, 25, "semicolon")]),
2145
+ ("---\ndescription: 'It''s robust' # c\n---\n", [(2, 21, "marketing-adjective")]),
2146
+ ("---\ndescription: >\n a # robust; b\n---\n", [(3, 7, "marketing-adjective"), (3, 13, "semicolon")]),
2147
+ ("---\ndescription: a#b; c\n---\n", [(2, 17, "semicolon")]),
2148
+ # the key "description" in quotes is the same key
2149
+ ("---\n\"description\": robust;\n---\n", [(2, 16, "marketing-adjective"), (2, 22, "semicolon")]),
2150
+ ("---\n'description' : a; b\n---\n", [(2, 18, "semicolon")]),
2151
+ ("---\n\"description\": |\n a; b\n---\n", [(3, 4, "semicolon")]),
2152
+ ("---\n\"descriptions\": a; b\n---\n", []),
2153
+ # a later line that starts with the key can be in a quoted value of another key
2154
+ ("---\ndescription: robust;\nexample: \"foo\ndescription: plain\"\n---\n",
2155
+ [(2, 14, "marketing-adjective"), (2, 20, "semicolon")]),
2156
+ # a comment after a block scalar has less indent than its text
2157
+ ("---\ndescription: |\n Use it.\n # robust; note\n---\n", []),
2158
+ # quoted keys, keys with spaces and keys with other characters are YAML
2159
+ ("---\n\"robust-kit\": patch\n---\n", []),
2160
+ ("---\nallowed-tools: Bash; Read\n---\n", []),
2161
+ ("---\n$schema: robust; y\n---\n", []),
2162
+ ("---\nrésumé: a; b\n---\n", [])):
2163
+ findings, _ = lint(text)
2164
+ assert [(f["line"], f["col"], f["rule"]) for f in findings] == expected, (text, findings)
2165
+ # a block that is not YAML is Markdown: thematic breaks around prose
2166
+ findings, _ = lint("---\n# Instructions\n\nStart; stop.\n\n- First.\n\n---\n")
2167
+ assert [(f["line"], f["rule"]) for f in findings] == [(4, "semicolon")], findings
2168
+ findings, _ = lint("---\nStart; stop.\n---\n")
2169
+ assert [(f["line"], f["rule"]) for f in findings] == [(2, "semicolon")], findings
2170
+ # a plain key with a space reads as prose, as in "Step 1: run it"
2171
+ findings, _ = lint("---\nStep 1: run it; then: stop.\n---\n")
2172
+ assert [(f["line"], f["rule"]) for f in findings] == [(2, "semicolon")], findings
2173
+ # a block of comments only is Markdown: a heading between thematic breaks
2174
+ findings, _ = lint("---\n# Note\n---\n\nText; here.\n")
2175
+ assert [(f["line"], f["rule"]) for f in findings] == [(5, "semicolon")], findings
2176
+ # a later --- pair is a thematic break, not front matter
2177
+ findings, _ = lint(f"Intro.\n\n---\n{twenty} {twenty}.\n---")
2178
+ assert any(f["rule"] == "long-sentence" for f in findings), findings
2179
+ # blank lines, headings, rules, list items, and quotes end a prose block
2180
+ for sep in ("\n\n", "\n# Heading\n", "\n---\n", "\n- ", "\n> "):
2181
+ findings, _ = lint(f"First {twenty} line{sep}Second {twenty} line.")
2182
+ assert not any(f["rule"] == "long-sentence" for f in findings), (sep, findings)
2183
+ # a GitHub alert marker on the first line of a quote is not a word of the body
2184
+ body = " ".join(["word"] * 25) + "."
2185
+ for alert in ("> [!NOTE]", "> [!tip]", ">[!Warning]", "> [!CAUTION] "):
2186
+ findings, _ = lint(f"{alert}\n> {body}")
2187
+ assert not any(f["rule"] == "long-sentence" for f in findings), (alert, findings)
2188
+ for text in (f"Text.\n\n> [!NOTE]\n> {body}", f"> Text.\n~~~\ncode\n~~~\n> [!NOTE]\n> {body}"):
2189
+ findings, _ = lint(text)
2190
+ assert not any(f["rule"] == "long-sentence" for f in findings), (text, findings)
2191
+ # Inside HTML containers the marker is text: marker plus body is 26 words.
2192
+ for opening, closing in (("<details>", "</details>"),
2193
+ ('<DIV class="note">', "</DIV>"),
2194
+ ("<section><details>", "</details></section>")):
2195
+ findings, _ = lint(f"{opening}\n\n> [!NOTE]\n> {body}\n\n{closing}")
2196
+ long_sentences = [f for f in findings if f["rule"] == "long-sentence"]
2197
+ assert [f["match"] for f in long_sentences] == ["26 words"], (opening, findings)
2198
+ findings, _ = lint(f"{opening}\n{closing}\n\n> [!NOTE]\n> {body}")
2199
+ assert not any(f["rule"] == "long-sentence" for f in findings), (opening, findings)
2200
+ findings, _ = lint(f"> [!NOTE]\n> {body}")
2201
+ assert not any(f["rule"] == "long-sentence" for f in findings), findings
2202
+ # An inner close or a fenced closing tag must not close the outer container.
2203
+ for content in ("<details>\n<div>\n</div>",
2204
+ "<details>\n\n~~~\n</details>\n~~~"):
2205
+ findings, _ = lint(f"{content}\n\n> [!NOTE]\n> {body}\n\n</details>")
2206
+ long_sentences = [f for f in findings if f["rule"] == "long-sentence"]
2207
+ assert [f["match"] for f in long_sentences] == ["26 words"], (content, findings)
2208
+ # Directly after <details>, ~~~ is HTML text, not a fence. So the </details> line after
2209
+ # it closes the container, and the alert after it is outside, as GitHub shows it.
2210
+ findings, _ = lint(f"<details>\n~~~\n</details>\n~~~\n\n> [!NOTE]\n> {body}\n\n</details>")
2211
+ assert not any(f["rule"] == "long-sentence" for f in findings), findings
2212
+ # Fenced, commented, inline, void, and self-closing tags do not open containers.
2213
+ for content in ("```html\n<details>\n```", "<!-- <details> -->",
2214
+ "<span>Text.</span>", "<hr>", "<div />",
2215
+ "<details></details>", "</details>"):
2216
+ findings, _ = lint(f"{content}\n\n> [!NOTE]\n> {body}")
2217
+ assert not any(f["rule"] == "long-sentence" for f in findings), (content, findings)
2218
+ # GitHub shows these markers as text, so they stay words. GitHub shows the
2219
+ # last two as alerts, but no blank line or fence proves that a quote starts.
2220
+ for text in (f"> [!FOO]\n> {body}", f"> [!NOTE] {body}", f"> Intro.\n> [!NOTE]\n> {body}",
2221
+ f"> > [!NOTE]\n> > {body}", f"> Intro.\ncontinuation.\n> [!NOTE]\n> {body}",
2222
+ f"- Item.\n\n > [!NOTE]\n > {body}",
2223
+ f"Text.\n> [!NOTE]\n> {body}", f" > [!NOTE]\n > {body}"):
2224
+ findings, _ = lint(text)
2225
+ long_sentences = [f for f in findings if f["rule"] == "long-sentence"]
2226
+ assert [f["match"] for f in long_sentences] == ["26 words"], (text, long_sentences)
2227
+ # a bold or italic lead sentence ends at its closing emphasis marker
2228
+ findings, _ = lint(f"**Lead.** The {twenty} more words here now.")
2229
+ assert not any(f["rule"] == "long-sentence" for f in findings), findings
2230
+ findings, _ = lint(f"_Note._ The {twenty} more words here now.")
2231
+ assert not any(f["rule"] == "long-sentence" for f in findings), findings
2232
+ findings, _ = lint(f"See **e.g.** the {twenty} linter here now.")
2233
+ assert any(f["rule"] == "long-sentence" for f in findings), findings
2234
+ # a dotted abbreviation ends a sentence before a capitalized word
2235
+ findings, _ = lint(f"The team shipped it in the U.S. Then {twenty} today.")
2236
+ assert not any(f["rule"] == "long-sentence" for f in findings), findings
2237
+ # nested list items, quoted list items, and HTML lines end a prose block
2238
+ for text in (f"- First item\n - First {twenty} line\n - Second {twenty} line",
2239
+ f"> - First {twenty} line\n> - Second {twenty} line",
2240
+ f"<!-- First {twenty} line -->\nSecond {twenty} line.",
2241
+ f"> First {twenty} line.\n>\n> Second {twenty} line."):
2242
+ findings, _ = lint(text)
2243
+ assert not any(f["rule"] == "long-sentence" for f in findings), (text, findings)
2244
+ # a blockquote prefix is not a word in a wrapped quoted sentence
2245
+ findings, _ = lint("> " + "\n> ".join(["word word word word word"] * 5) + ".")
2246
+ assert not any(f["rule"] == "long-sentence" for f in findings), findings
2247
+ # a list marker is not a word either, in every marker style
2248
+ twenty_five = " ".join(f"term{number}" for number in range(1, 26)) + "."
2249
+ for marker in ("-", "*", "+", "1.", "1)", " -", "> -"):
2250
+ findings, _ = lint(f"{marker} {twenty_five}")
2251
+ assert not any(f["rule"] == "long-sentence" for f in findings), (marker, findings)
2252
+ findings, _ = lint(f"- term0 {twenty_five}")
2253
+ long_sentences = [f for f in findings if f["rule"] == "long-sentence"]
2254
+ assert [(f["match"], f["col"]) for f in long_sentences] == [("26 words", 3)], long_sentences
2255
+ # a code span at the line start does not turn the next token into a list marker
2256
+ findings, _ = lint(f"`label` 1) {twenty_five}")
2257
+ long_sentences = [f for f in findings if f["rule"] == "long-sentence"]
2258
+ assert [f["match"] for f in long_sentences] == ["26 words"], long_sentences
2259
+ # bare punctuation is not a word: code-span separators and spaced dashes
2260
+ for separator in (", ", " ", " / "):
2261
+ names = separator.join(f"`a{number}`" for number in range(30))
2262
+ findings, _ = lint(f"The adapters are {names}.")
2263
+ assert not any(f["rule"] == "long-sentence" for f in findings), (separator, findings)
2264
+ findings, _ = lint(" ".join(f"term{number}" for number in range(1, 25)) + " — term25.")
2265
+ assert not any(f["rule"] == "long-sentence" for f in findings), findings
2266
+ findings, _ = lint(f"Use `make` — {twenty_five}")
2267
+ long_sentences = [f for f in findings if f["rule"] == "long-sentence"]
2268
+ assert [f["match"] for f in long_sentences] == ["26 words"], long_sentences
2269
+ # a long block stays linear: one short sentence per line, no blank lines
2270
+ findings, _ = lint("The agent read the log.\n" * 20000)
2271
+ assert not any(f["rule"] == "long-sentence" for f in findings)
2272
+ # a long line with many code spans, and many links that go on in the next line, stay linear
2273
+ findings, _ = lint("`x` " * 60000 + "c; d.")
2274
+ assert [(f["rule"], f["col"]) for f in findings] == [("semicolon", 240002)], findings
2275
+ findings, _ = lint("[a](x;y\n)\n" * 5000)
2276
+ assert not any(f["rule"] == "semicolon" for f in findings), findings
2277
+ # flags may appear between file paths
2278
+ with tempfile.TemporaryDirectory() as tmp:
2279
+ paths = [os.path.join(tmp, name) for name in ("a.md", "b.md")]
2280
+ for path in paths:
2281
+ with open(path, "w", encoding="utf-8") as fh:
2282
+ fh.write("Short line here.\n")
2283
+ out = io.StringIO()
2284
+ with contextlib.redirect_stdout(out):
2285
+ assert main([paths[0], "--json", paths[1]]) == 0
2286
+ assert json.loads(out.getvalue())["words"] == 6, out.getvalue()
2287
+ # repeated --disable flags add up instead of replacing each other
2288
+ with open(paths[0], "w", encoding="utf-8") as fh:
2289
+ fh.write("a; b. " + " ".join(["word"] * 30) + ".\n")
2290
+ out = io.StringIO()
2291
+ with contextlib.redirect_stdout(out):
2292
+ assert main([paths[0], "--json", "--disable", "semicolon",
2293
+ "--disable", "long-sentence"]) == 0
2294
+ assert json.loads(out.getvalue())["count"] == 0, out.getvalue()
2295
+ # command-line parsing rejects typos and bad values
2296
+ for bad in (["--basline", "3"], ["--baseline"], ["--baseline", "x"],
2297
+ ["--baseline", "-1"], ["--disable", "passive"], ["--base", "3"]):
2298
+ try:
2299
+ with contextlib.redirect_stderr(io.StringIO()):
2300
+ main(bad)
2301
+ except SystemExit as e:
2302
+ assert e.code == 2, (bad, e.code)
2303
+ else:
2304
+ raise AssertionError(bad)
2305
+ assert _rule_list("passive-voice,present-perfect") == {"passive-voice", "present-perfect"}
2306
+ # an HTML block in a block quote holds text, and it ends with its quote
2307
+ findings, _ = lint("> <div>\n> ```\n> This is prose; keep it.\n> ```\n> </div>")
2308
+ assert [(f["rule"], f["line"]) for f in findings] == [("semicolon", 3)], findings
2309
+ findings, _ = lint("> <div>\n\n```\nx; y\n```\n\nProse; here.")
2310
+ assert [(f["rule"], f["line"]) for f in findings] == [("semicolon", 7)], findings
2311
+ # a ">" line in an open HTML block is its text, not a quote end
2312
+ findings, _ = lint("<div>\n> Note.\n>\n> ```\n> a; b\n> ```\n</div>")
2313
+ assert [(f["rule"], f["line"]) for f in findings] == [("semicolon", 5)], findings
2314
+ findings, _ = lint("<div>\n>\n```\nText; here.\n```")
2315
+ assert [(f["rule"], f["line"]) for f in findings] == [("semicolon", 4)], findings
2316
+ # a quote after a paragraph starts a block quote, so a type 7 tag in it starts a block
2317
+ findings, _ = lint("Paragraph.\n> <widget>\n> ```\n> Text; here.\n> ```")
2318
+ assert [(f["rule"], f["line"]) for f in findings] == [("semicolon", 4)], findings
2319
+ print("selftest OK")
2320
+
2321
+
2322
+ RULE_IDS = {rule_id for rule_id, *_ in RULES} | {
2323
+ "long-sentence", "synonym-rotation", "dangling-conjunction"}
2324
+
2325
+
2326
+ def _rule_list(value):
2327
+ rules = {r for r in value.split(",") if r}
2328
+ unknown = rules - RULE_IDS
2329
+ if unknown:
2330
+ raise argparse.ArgumentTypeError(
2331
+ f"unknown rule(s): {', '.join(sorted(unknown))}. "
2332
+ f"Known rules: {', '.join(sorted(RULE_IDS))}")
2333
+ return rules
2334
+
2335
+
2336
+ def _non_negative_int(value):
2337
+ try:
2338
+ n = int(value)
2339
+ except ValueError:
2340
+ n = -1
2341
+ if n < 0:
2342
+ raise argparse.ArgumentTypeError(f"expected a non-negative integer, got {value!r}")
2343
+ return n
2344
+
2345
+
2346
+ def main(argv):
2347
+ parser = argparse.ArgumentParser(
2348
+ prog="ste-lint.py", allow_abbrev=False,
2349
+ description="Deterministic linter for the structural STE rules. "
2350
+ "Reads stdin when no FILE is given.")
2351
+ parser.add_argument("paths", nargs="*", metavar="FILE")
2352
+ parser.add_argument("--json", action="store_true", help="print findings as JSON")
2353
+ parser.add_argument("--baseline", type=_non_negative_int, default=0, metavar="N",
2354
+ help="pass unless hard violations exceed N (default 0)")
2355
+ parser.add_argument("--disable", type=_rule_list, action="append", default=[],
2356
+ metavar="RULES",
2357
+ help="comma-separated rule IDs to silence; may be repeated")
2358
+ parser.add_argument("--selftest", action="store_true", help="run the built-in tests")
2359
+ # intermixed parsing keeps flags valid between file paths
2360
+ args = parser.parse_intermixed_args(argv)
2361
+ if args.selftest:
2362
+ selftest()
2363
+ return 0
2364
+ as_json, baseline, paths = args.json, args.baseline, args.paths
2365
+ disabled = set().union(*args.disable)
2366
+
2367
+ findings, words_total = [], 0
2368
+ if paths:
2369
+ for p in paths:
2370
+ try:
2371
+ with open(p, encoding="utf-8") as fh:
2372
+ text = fh.read()
2373
+ except OSError as e:
2374
+ parser.error(f"cannot read {p}: {e.strerror}")
2375
+ f, w = lint(text, filename=p)
2376
+ findings.extend(f)
2377
+ words_total += w
2378
+ else:
2379
+ findings, words_total = lint(sys.stdin.read())
2380
+
2381
+ findings = [f for f in findings if f["rule"] not in disabled]
2382
+ hard_count = sum(1 for f in findings if f["level"] == "advisory-free")
2383
+ report(findings, words_total, as_json, hard_count, baseline)
2384
+ return 1 if hard_count > baseline else 0
2385
+
2386
+
2387
+ if __name__ == "__main__":
2388
+ sys.exit(main(sys.argv[1:]))