polytypo 1.11.0 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +20 -0
- data/lib/polytypo/data/VERSION +1 -1
- data/lib/polytypo/data/fixtures/bg.json +253 -0
- data/lib/polytypo/data/fixtures/cs.json +1 -1
- data/lib/polytypo/data/fixtures/de-CH.json +1 -1
- data/lib/polytypo/data/fixtures/de-DE.json +77 -1
- data/lib/polytypo/data/fixtures/el.json +1 -1
- data/lib/polytypo/data/fixtures/en-GB.json +89 -1
- data/lib/polytypo/data/fixtures/en-US.json +450 -1
- data/lib/polytypo/data/fixtures/es.json +1 -1
- data/lib/polytypo/data/fixtures/fi.json +1 -1
- data/lib/polytypo/data/fixtures/fr-CA.json +1 -1
- data/lib/polytypo/data/fixtures/fr.json +41 -1
- data/lib/polytypo/data/fixtures/hr.json +253 -0
- data/lib/polytypo/data/fixtures/hu.json +264 -0
- data/lib/polytypo/data/fixtures/it.json +1 -1
- data/lib/polytypo/data/fixtures/locale-resolution.json +1 -1
- data/lib/polytypo/data/fixtures/nl.json +1 -1
- data/lib/polytypo/data/fixtures/pl.json +1 -1
- data/lib/polytypo/data/fixtures/pt-BR.json +1 -1
- data/lib/polytypo/data/fixtures/pt-PT.json +1 -1
- data/lib/polytypo/data/fixtures/ro.json +234 -0
- data/lib/polytypo/data/fixtures/ru.json +1 -1
- data/lib/polytypo/data/fixtures/sk.json +1 -1
- data/lib/polytypo/data/fixtures/sv.json +1 -1
- data/lib/polytypo/data/fixtures/tr.json +1 -1
- data/lib/polytypo/data/fixtures/uk.json +1 -1
- data/lib/polytypo/data/locales/bg.json +82 -0
- data/lib/polytypo/data/locales/hr.json +82 -0
- data/lib/polytypo/data/locales/hu.json +88 -0
- data/lib/polytypo/data/locales/registry.json +6 -2
- data/lib/polytypo/data/locales/ro.json +82 -0
- data/lib/polytypo/data/rules/analyze.md +16 -4
- data/lib/polytypo/data/rules/dashes.md +2 -1
- data/lib/polytypo/data/rules/modes.md +262 -9
- data/lib/polytypo/data/rules/order.json +1 -1
- data/lib/polytypo/data/rules/quotes.md +9 -0
- data/lib/polytypo/data/rules/ranges.md +6 -5
- data/lib/polytypo/data/schema/fixtures.schema.json +160 -18
- data/lib/polytypo/data/schema/locale.schema.json +2 -2
- data/lib/polytypo/engine/origin.rb +13 -1
- data/lib/polytypo/engine/protect.rb +122 -0
- data/lib/polytypo/modes/runner.rb +0 -1
- data/lib/polytypo/version.rb +1 -1
- data/lib/polytypo.rb +34 -19
- metadata +10 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
3
|
"$id": "https://polytypo.org/schema/fixtures.schema.json",
|
|
4
4
|
"title": "polytypo conformance fixtures",
|
|
5
|
-
"description": "Executable form of the rule semantics. Every case is also an idempotency case: a runner must additionally assert transform(out) == out, passing the case's own options (mode, dialect, keys, frontmatterKeys, rules, narrowNbsp) on that second call exactly as on the first — a case is a fixed point under its own options, not under the defaults.",
|
|
5
|
+
"description": "Executable form of the rule semantics. Every case is also an idempotency case: a runner must additionally assert transform(out) == out, passing the case's own options (mode, dialect, keys, frontmatterKeys, rules, narrowNbsp, and, for `protect`, its `protectOut` counterpart, since the ranges move with the edits) on that second call exactly as on the first — a case is a fixed point under its own options, not under the defaults.",
|
|
6
6
|
"type": "object",
|
|
7
7
|
"additionalProperties": false,
|
|
8
8
|
"required": ["spec", "locale", "cases"],
|
|
@@ -11,7 +11,10 @@
|
|
|
11
11
|
"type": "string",
|
|
12
12
|
"description": "The current global spec contract version. Must equal spec/VERSION exactly — not an authoring-time stamp and not a per-case introduction version. scripts/validate-spec.mjs enforces this for every fixture root."
|
|
13
13
|
},
|
|
14
|
-
"locale": {
|
|
14
|
+
"locale": {
|
|
15
|
+
"type": "string",
|
|
16
|
+
"pattern": "^[a-z]{2}(-[A-Z]{2})?$"
|
|
17
|
+
},
|
|
15
18
|
"cases": {
|
|
16
19
|
"type": "array",
|
|
17
20
|
"minItems": 1,
|
|
@@ -21,22 +24,124 @@
|
|
|
21
24
|
"required": ["id", "rule", "mode", "in"],
|
|
22
25
|
"allOf": [
|
|
23
26
|
{
|
|
24
|
-
"if": {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
+
"if": {
|
|
28
|
+
"properties": {
|
|
29
|
+
"mode": {
|
|
30
|
+
"const": "markdown"
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
"required": ["mode"]
|
|
34
|
+
},
|
|
35
|
+
"then": {
|
|
36
|
+
"required": ["dialect"]
|
|
37
|
+
},
|
|
38
|
+
"else": {
|
|
39
|
+
"not": {
|
|
40
|
+
"required": ["dialect"]
|
|
41
|
+
}
|
|
42
|
+
}
|
|
27
43
|
},
|
|
28
44
|
{
|
|
29
|
-
"if": {
|
|
30
|
-
|
|
31
|
-
|
|
45
|
+
"if": {
|
|
46
|
+
"properties": {
|
|
47
|
+
"mode": {
|
|
48
|
+
"const": "yaml"
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
"required": ["mode"]
|
|
52
|
+
},
|
|
53
|
+
"then": {
|
|
54
|
+
"required": ["keys"]
|
|
55
|
+
},
|
|
56
|
+
"else": {
|
|
57
|
+
"not": {
|
|
58
|
+
"required": ["keys"]
|
|
59
|
+
}
|
|
60
|
+
}
|
|
32
61
|
},
|
|
33
62
|
{
|
|
34
|
-
"if": {
|
|
63
|
+
"if": {
|
|
64
|
+
"properties": {
|
|
65
|
+
"mode": {
|
|
66
|
+
"const": "markdown"
|
|
67
|
+
}
|
|
68
|
+
},
|
|
69
|
+
"required": ["mode"]
|
|
70
|
+
},
|
|
35
71
|
"then": true,
|
|
36
|
-
"else": {
|
|
72
|
+
"else": {
|
|
73
|
+
"not": {
|
|
74
|
+
"required": ["frontmatterKeys"]
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
"if": {
|
|
80
|
+
"required": ["protect", "out"]
|
|
81
|
+
},
|
|
82
|
+
"then": {
|
|
83
|
+
"properties": {
|
|
84
|
+
"protect": {
|
|
85
|
+
"type": "array",
|
|
86
|
+
"items": {
|
|
87
|
+
"type": "object",
|
|
88
|
+
"additionalProperties": false,
|
|
89
|
+
"required": ["start", "end"],
|
|
90
|
+
"properties": {
|
|
91
|
+
"start": {
|
|
92
|
+
"type": "integer"
|
|
93
|
+
},
|
|
94
|
+
"end": {
|
|
95
|
+
"type": "integer"
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
"required": ["protectOut"]
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"if": {
|
|
106
|
+
"required": ["throws"]
|
|
107
|
+
},
|
|
108
|
+
"then": {
|
|
109
|
+
"not": {
|
|
110
|
+
"required": ["protectOut"]
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
"if": {
|
|
116
|
+
"required": ["protect"],
|
|
117
|
+
"properties": {
|
|
118
|
+
"protect": {
|
|
119
|
+
"type": "array",
|
|
120
|
+
"minItems": 1
|
|
121
|
+
}
|
|
122
|
+
},
|
|
123
|
+
"not": {
|
|
124
|
+
"required": ["mode"],
|
|
125
|
+
"properties": {
|
|
126
|
+
"mode": {
|
|
127
|
+
"const": "text"
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
},
|
|
132
|
+
"then": {
|
|
133
|
+
"required": ["throws"]
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
],
|
|
137
|
+
"oneOf": [
|
|
138
|
+
{
|
|
139
|
+
"required": ["out"]
|
|
140
|
+
},
|
|
141
|
+
{
|
|
142
|
+
"required": ["throws"]
|
|
37
143
|
}
|
|
38
144
|
],
|
|
39
|
-
"oneOf": [{ "required": ["out"] }, { "required": ["throws"] }],
|
|
40
145
|
"properties": {
|
|
41
146
|
"id": {
|
|
42
147
|
"type": "string",
|
|
@@ -57,24 +162,34 @@
|
|
|
57
162
|
],
|
|
58
163
|
"description": "The rule under test. Required so a runtime can report partial conformance honestly."
|
|
59
164
|
},
|
|
60
|
-
"mode": {
|
|
165
|
+
"mode": {
|
|
166
|
+
"enum": ["text", "html", "markdown", "yaml"]
|
|
167
|
+
},
|
|
61
168
|
"dialect": {
|
|
62
169
|
"enum": ["commonmark", "mdx"],
|
|
63
170
|
"description": "Required when mode is \"markdown\" — spec/rules/modes.md §3.7.1 forbids detection, so a fixture must name the dialect exactly as a caller would. Meaningless for the other modes."
|
|
64
171
|
},
|
|
65
172
|
"keys": {
|
|
66
173
|
"type": "array",
|
|
67
|
-
"items": {
|
|
174
|
+
"items": {
|
|
175
|
+
"type": "string"
|
|
176
|
+
},
|
|
68
177
|
"description": "Required when mode is \"yaml\" — spec/rules/modes.md §3.8.2 gives the option no default, so a fixture must name the processable keys exactly as a caller would. An empty array is legal and processes nothing. Meaningless for the other modes."
|
|
69
178
|
},
|
|
70
179
|
"frontmatterKeys": {
|
|
71
180
|
"type": "array",
|
|
72
|
-
"items": {
|
|
181
|
+
"items": {
|
|
182
|
+
"type": "string"
|
|
183
|
+
},
|
|
73
184
|
"description": "Optional, spec 1.7.0, and only meaningful when mode is \"markdown\" — spec/rules/modes.md §3.7.4's opt-out from the frontmatter skip. Absent means the block is skipped whole, exactly as before 1.7.0; an empty array is legal and yields no spans. A runner passes it through to transform() as it passes dialect.",
|
|
74
185
|
"$comment": "The block is its own text unit (§3.7.4), so this option cannot change a byte of the body — every case released before 1.7.0 keeps its recorded output whatever it is set to."
|
|
75
186
|
},
|
|
76
|
-
"in": {
|
|
77
|
-
|
|
187
|
+
"in": {
|
|
188
|
+
"type": "string"
|
|
189
|
+
},
|
|
190
|
+
"out": {
|
|
191
|
+
"type": "string"
|
|
192
|
+
},
|
|
78
193
|
"throws": {
|
|
79
194
|
"enum": [
|
|
80
195
|
"POLYTYPO_UNKNOWN_LOCALE",
|
|
@@ -88,16 +203,43 @@
|
|
|
88
203
|
],
|
|
89
204
|
"description": "Expected error code instead of an output string. Messages are not part of the contract; codes are."
|
|
90
205
|
},
|
|
91
|
-
"note": {
|
|
206
|
+
"note": {
|
|
207
|
+
"type": "string"
|
|
208
|
+
},
|
|
92
209
|
"rules": {
|
|
93
210
|
"type": "object",
|
|
94
|
-
"additionalProperties": {
|
|
211
|
+
"additionalProperties": {
|
|
212
|
+
"type": "boolean"
|
|
213
|
+
},
|
|
95
214
|
"description": "Optional rule overrides for this case, same shape as the public `rules` option."
|
|
96
215
|
},
|
|
97
216
|
"narrowNbsp": {
|
|
98
217
|
"enum": ["narrow", "nbsp"],
|
|
99
218
|
"description": "Optional, spec 1.3.0. Same shape as the public `narrowNbsp` option (spec/rules/nbsp.md §3.1a): \"nbsp\" makes the engine emit U+00A0 wherever it would emit U+202F. Omitted means \"narrow\", the default, so every pre-1.3.0 case keeps its meaning. A runner passes this through to transform() exactly as it passes `rules`."
|
|
219
|
+
},
|
|
220
|
+
"protect": {
|
|
221
|
+
"description": "Optional, spec 1.13.0. The public `protect` option (spec/rules/modes.md §3.9): ranges of `in`, in Unicode code points, [start, end), that the engine never edits and reads as opaque content. A runner passes it to transform() as it passes `rules`. Unconstrained here because a `throws` case carries the malformed values §3.9.2 refuses; a case with `out` must carry a list of ranges, and a non-empty list with a mode other than \"text\" is only legal in a `throws` case."
|
|
222
|
+
},
|
|
223
|
+
"protectOut": {
|
|
224
|
+
"description": "Required when `protect` and `out` are both present, forbidden otherwise. The same protected runs located in `out`, in code points. Edits before a hole move it, so the idempotency call transform(out) must pass THESE ranges, not `protect`. scripts/validate-spec.mjs checks that the code points they select are exactly the ones `protect` selects in `in`. Measured from the engine, never written by hand.",
|
|
225
|
+
"type": "array",
|
|
226
|
+
"items": {
|
|
227
|
+
"type": "object",
|
|
228
|
+
"additionalProperties": false,
|
|
229
|
+
"required": ["start", "end"],
|
|
230
|
+
"properties": {
|
|
231
|
+
"start": {
|
|
232
|
+
"type": "integer"
|
|
233
|
+
},
|
|
234
|
+
"end": {
|
|
235
|
+
"type": "integer"
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
100
239
|
}
|
|
240
|
+
},
|
|
241
|
+
"dependentRequired": {
|
|
242
|
+
"protectOut": ["protect"]
|
|
101
243
|
}
|
|
102
244
|
}
|
|
103
245
|
}
|
|
@@ -39,11 +39,11 @@
|
|
|
39
39
|
"properties": {
|
|
40
40
|
"parenthetical": {
|
|
41
41
|
"enum": ["em-tight", "em-spaced", "en-tight", "en-spaced", "none"],
|
|
42
|
-
"description": "Dash used for a parenthetical break. `-tight` means no surrounding spaces, `-spaced` means a regular space on each side (the leading one may later be made non-breaking by the nbsp rule). `none` means the locale has no verified convention
|
|
42
|
+
"description": "Dash used for a parenthetical break. `-tight` means no surrounding spaces, `-spaced` means a regular space on each side (the leading one may later be made non-breaking by the nbsp rule). `none` means the rule must not substitute anything: either the locale has no verified convention, or the cited sources attest more than one form and rank none."
|
|
43
43
|
},
|
|
44
44
|
"range": {
|
|
45
45
|
"enum": ["em-tight", "em-spaced", "en-tight", "en-spaced", "none"],
|
|
46
|
-
"description": "Dash used between numbers and dates. Russian sets an em dash here. `none` means the locale has no verified convention
|
|
46
|
+
"description": "Dash used between numbers and dates. Russian sets an em dash here. `none` means the rule must not substitute anything: either the locale has no verified convention, or the cited sources attest more than one form and rank none — a missing citation is never a licence to guess."
|
|
47
47
|
}
|
|
48
48
|
}
|
|
49
49
|
},
|
|
@@ -54,6 +54,18 @@ module Polytypo
|
|
|
54
54
|
out
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
+
# Where a change ends in the input (analyze.md section 2). A non-empty edit ends one past
|
|
58
|
+
# the input offset of the last code point it replaced that has an origin. Scanning forward
|
|
59
|
+
# from edit.end instead, as origin_at does for a start, would carry the end across a skipped
|
|
60
|
+
# element or a protected run that follows -- the boundary marker has no origin -- and the
|
|
61
|
+
# reported range would contain text the rules never saw.
|
|
62
|
+
def self.end_of(origin, edit, input_length)
|
|
63
|
+
(edit.end - 1).downto(edit.start) do |i|
|
|
64
|
+
return origin[i] + 1 unless origin[i] == NO_ORIGIN
|
|
65
|
+
end
|
|
66
|
+
origin_at(origin, edit.end, input_length)
|
|
67
|
+
end
|
|
68
|
+
|
|
57
69
|
# One rule's edits, in the coordinates that rule saw, rendered as Changes in input
|
|
58
70
|
# coordinates. +before+ is the text this rule replaced and +after+ what it replaced it with
|
|
59
71
|
# (analyze.md section 2), so on a text two rules have both touched, +before+ is what the
|
|
@@ -64,7 +76,7 @@ module Polytypo
|
|
|
64
76
|
Polytypo::Change.new(
|
|
65
77
|
rule_id: rule_id,
|
|
66
78
|
start: origin_at(origin, edit.start, input_length),
|
|
67
|
-
end:
|
|
79
|
+
end: end_of(origin, edit, input_length),
|
|
68
80
|
before: Codepoints.from_codepoints(cp[edit.start...edit.end]),
|
|
69
81
|
after: Codepoints.from_codepoints(edit.replacement)
|
|
70
82
|
)
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "codepoints"
|
|
4
|
+
require_relative "origin"
|
|
5
|
+
require_relative "pipeline"
|
|
6
|
+
require_relative "sentinels"
|
|
7
|
+
require_relative "../errors"
|
|
8
|
+
require_relative "../modes/spans"
|
|
9
|
+
require_relative "../modes/runner"
|
|
10
|
+
|
|
11
|
+
module Polytypo
|
|
12
|
+
module Engine
|
|
13
|
+
# modes.md 3.9: the caller's own skipped regions, "text" mode only (spec 1.13.0). Each maximal
|
|
14
|
+
# protected run becomes ONE boundary marker in the code-point array -- at the very start or
|
|
15
|
+
# end too, which is the one difference from the html/markdown/yaml span concatenation -- the
|
|
16
|
+
# rules run once over that array, and the protected code points are put back at their markers.
|
|
17
|
+
module Protect
|
|
18
|
+
LINE_TERMINATORS = [0x0A, 0x0D, 0x0B, 0x0C, 0x85, 0x2028, 0x2029].freeze
|
|
19
|
+
|
|
20
|
+
def self.invalid(message)
|
|
21
|
+
raise Polytypo::Error.new(Polytypo::CODE_INVALID_OPTION, message)
|
|
22
|
+
end
|
|
23
|
+
private_class_method :invalid
|
|
24
|
+
|
|
25
|
+
# modes.md 3.9.2: validates +value+ against the input's length in code points and returns
|
|
26
|
+
# the protected runs as [start, end] pairs with touching ranges merged. nil and [] are the
|
|
27
|
+
# same call.
|
|
28
|
+
def self.resolve(value, length)
|
|
29
|
+
return [] if value.nil?
|
|
30
|
+
|
|
31
|
+
invalid("\"protect\" must be an Array of [start, end] pairs. Received #{value.inspect}.") unless value.is_a?(Array)
|
|
32
|
+
|
|
33
|
+
holes = []
|
|
34
|
+
previous_end = 0
|
|
35
|
+
value.each do |range|
|
|
36
|
+
unless range.is_a?(Array) && range.length == 2 && range.all?(Integer)
|
|
37
|
+
invalid("\"protect\" entries must be [start, end] pairs of Integers; received #{range.inspect}.")
|
|
38
|
+
end
|
|
39
|
+
s, e = range
|
|
40
|
+
invalid("protect range [#{s}, #{e}) is empty or negative.") if s.negative? || e <= s
|
|
41
|
+
invalid("protect range [#{s}, #{e}) overlaps or precedes the previous one.") if s < previous_end
|
|
42
|
+
invalid("protect range [#{s}, #{e}) runs past the end of the input (#{length}).") if e > length
|
|
43
|
+
|
|
44
|
+
if !holes.empty? && holes.last[1] == s
|
|
45
|
+
holes.last[1] = e
|
|
46
|
+
else
|
|
47
|
+
holes << [s, e]
|
|
48
|
+
end
|
|
49
|
+
previous_end = e
|
|
50
|
+
end
|
|
51
|
+
holes
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# modes.md 3.9.2: outside "text" mode a non-empty +protect+ is refused rather than ignored.
|
|
55
|
+
def self.refuse_outside_text(value, mode)
|
|
56
|
+
return if value.nil? || (value.is_a?(Array) && value.empty?)
|
|
57
|
+
|
|
58
|
+
invalid("\"protect\" is only available when mode is \"text\", not #{mode.inspect}.")
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# [S0] m [S1] m ... [Sk], a marker for every run, edges included, and the origin map.
|
|
62
|
+
def self.layout(cp, holes)
|
|
63
|
+
array = []
|
|
64
|
+
origin = []
|
|
65
|
+
cursor = 0
|
|
66
|
+
holes.each do |s, e|
|
|
67
|
+
(cursor...s).each do |i|
|
|
68
|
+
array << cp[i]
|
|
69
|
+
origin << i
|
|
70
|
+
end
|
|
71
|
+
line = (s...e).any? { |i| LINE_TERMINATORS.include?(cp[i]) }
|
|
72
|
+
array << (line ? Engine::LINE_MARKER : Engine::MARKER)
|
|
73
|
+
origin << Origin::NO_ORIGIN
|
|
74
|
+
cursor = e
|
|
75
|
+
end
|
|
76
|
+
(cursor...cp.length).each do |i|
|
|
77
|
+
array << cp[i]
|
|
78
|
+
origin << i
|
|
79
|
+
end
|
|
80
|
+
[array, origin]
|
|
81
|
+
end
|
|
82
|
+
private_class_method :layout
|
|
83
|
+
|
|
84
|
+
# Returns [text, moved] where +moved+ is the protected runs located in the output, the
|
|
85
|
+
# ranges a second call has to be given. Public API returns the text alone.
|
|
86
|
+
def self.run_mapped(cp, holes, plan, locale_data, ctx)
|
|
87
|
+
array, = layout(cp, holes)
|
|
88
|
+
out = Modes::Runner.run_rules_over_spans(array, plan, locale_data, ctx)
|
|
89
|
+
result = []
|
|
90
|
+
moved = []
|
|
91
|
+
index = 0
|
|
92
|
+
out.each do |value|
|
|
93
|
+
unless Engine.marker?(value)
|
|
94
|
+
result << value
|
|
95
|
+
next
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
s, e = holes[index]
|
|
99
|
+
moved << [result.length, result.length + (e - s)]
|
|
100
|
+
result.concat(cp[s...e])
|
|
101
|
+
index += 1
|
|
102
|
+
end
|
|
103
|
+
if index != holes.length
|
|
104
|
+
raise Polytypo::Error.new(Polytypo::CODE_RULE_CONTRACT, "boundary markers did not survive the pipeline")
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
[Codepoints.from_codepoints(result), moved]
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def self.run(cp, holes, plan, locale_data, ctx)
|
|
111
|
+
run_mapped(cp, holes, plan, locale_data, ctx).first
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def self.analyze(cp, holes, plan, locale_data, ctx)
|
|
115
|
+
array, origin = layout(cp, holes)
|
|
116
|
+
Pipeline.run_rules_recording(array, plan, locale_data, ctx, origin, cp.length) do |current, edits|
|
|
117
|
+
Modes::Spans.filter_boundary_edits(current, edits, Modes::Spans.span_ranges_of(current))
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
@@ -25,7 +25,6 @@ module Polytypo
|
|
|
25
25
|
end
|
|
26
26
|
current
|
|
27
27
|
end
|
|
28
|
-
private_class_method :run_rules_over_spans
|
|
29
28
|
|
|
30
29
|
# The output is the input with a set of disjoint substring replacements applied and nothing
|
|
31
30
|
# else (modes.md 4). A span whose content the rules did not change contributes no
|
data/lib/polytypo/version.rb
CHANGED
data/lib/polytypo.rb
CHANGED
|
@@ -5,6 +5,7 @@ require_relative "polytypo/errors"
|
|
|
5
5
|
require_relative "polytypo/engine/codepoints"
|
|
6
6
|
require_relative "polytypo/engine/origin"
|
|
7
7
|
require_relative "polytypo/engine/pipeline"
|
|
8
|
+
require_relative "polytypo/engine/protect"
|
|
8
9
|
require_relative "polytypo/engine/rules" # side effect: registers all 9 rules
|
|
9
10
|
require_relative "polytypo/modes/spans"
|
|
10
11
|
require_relative "polytypo/modes/runner"
|
|
@@ -24,7 +25,9 @@ module Polytypo
|
|
|
24
25
|
# "mdx" is not implemented by this runtime). +keys+ is required iff mode is "yaml" and names
|
|
25
26
|
# the mapping keys whose scalar values are prose (modes.md 3.8.2) -- it has no default, because
|
|
26
27
|
# nothing in YAML's syntax separates "description:" from "run:". +rules+ is an opt-out Hash
|
|
27
|
-
# keyed by rule id.
|
|
28
|
+
# keyed by rule id. +protect+ is "text" mode only: an Array of [start, end] pairs of
|
|
29
|
+
# code-point offsets (half-open, ascending, non-overlapping) into +input+ that are never edited
|
|
30
|
+
# and that the rules read as opaque content (modes.md 3.9). nil and [] are the same call.
|
|
28
31
|
#
|
|
29
32
|
# Pure: no I/O, no environment, no clock, no globals, no class-level mutable state --
|
|
30
33
|
# thread-safe and reentrant, callable concurrently from any number of Threads with no external
|
|
@@ -33,7 +36,7 @@ module Polytypo
|
|
|
33
36
|
# Only `mode: "text"`/`"html"` ever touch this file's own requires; `mode: "markdown"` lazily
|
|
34
37
|
# requires "commonmarker" from within Modes::Markdown, never at load time of this file.
|
|
35
38
|
def self.transform(input, locale:, mode: "text", dialect: nil, keys: nil, rules: nil, narrow_nbsp: nil,
|
|
36
|
-
frontmatter_keys: nil)
|
|
39
|
+
frontmatter_keys: nil, protect: nil)
|
|
37
40
|
resolved_mode = resolve_mode(mode)
|
|
38
41
|
narrow_target = Engine.resolve_narrow_target(narrow_nbsp)
|
|
39
42
|
|
|
@@ -42,19 +45,19 @@ module Polytypo
|
|
|
42
45
|
if dialect
|
|
43
46
|
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
|
|
44
47
|
end
|
|
45
|
-
transform_text(input, locale, rules, narrow_target)
|
|
48
|
+
transform_text(input, locale, rules, narrow_target, protect)
|
|
46
49
|
when "html"
|
|
47
50
|
if dialect
|
|
48
51
|
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
|
|
49
52
|
end
|
|
50
|
-
transform_html(input, locale, rules, narrow_target)
|
|
53
|
+
transform_html(input, locale, rules, narrow_target, protect)
|
|
51
54
|
when "yaml"
|
|
52
55
|
if dialect
|
|
53
56
|
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
|
|
54
57
|
end
|
|
55
|
-
transform_yaml(input, locale, keys, rules, narrow_target)
|
|
58
|
+
transform_yaml(input, locale, keys, rules, narrow_target, protect)
|
|
56
59
|
else # "markdown"
|
|
57
|
-
transform_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys)
|
|
60
|
+
transform_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys: frontmatter_keys, protect: protect)
|
|
58
61
|
end
|
|
59
62
|
end
|
|
60
63
|
|
|
@@ -71,7 +74,7 @@ module Polytypo
|
|
|
71
74
|
#
|
|
72
75
|
# Pure and thread-safe on the same terms as .transform.
|
|
73
76
|
def self.analyze(input, locale:, mode: "text", dialect: nil, keys: nil, rules: nil, narrow_nbsp: nil,
|
|
74
|
-
frontmatter_keys: nil)
|
|
77
|
+
frontmatter_keys: nil, protect: nil)
|
|
75
78
|
resolved_mode = resolve_mode(mode)
|
|
76
79
|
narrow_target = Engine.resolve_narrow_target(narrow_nbsp)
|
|
77
80
|
|
|
@@ -80,19 +83,19 @@ module Polytypo
|
|
|
80
83
|
if dialect
|
|
81
84
|
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
|
|
82
85
|
end
|
|
83
|
-
analyze_text(input, locale, rules, narrow_target)
|
|
86
|
+
analyze_text(input, locale, rules, narrow_target, protect)
|
|
84
87
|
when "html"
|
|
85
88
|
if dialect
|
|
86
89
|
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
|
|
87
90
|
end
|
|
88
|
-
analyze_html(input, locale, rules, narrow_target)
|
|
91
|
+
analyze_html(input, locale, rules, narrow_target, protect)
|
|
89
92
|
when "yaml"
|
|
90
93
|
if dialect
|
|
91
94
|
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
|
|
92
95
|
end
|
|
93
|
-
analyze_yaml(input, locale, keys, rules, narrow_target)
|
|
96
|
+
analyze_yaml(input, locale, keys, rules, narrow_target, protect)
|
|
94
97
|
else # "markdown"
|
|
95
|
-
analyze_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys)
|
|
98
|
+
analyze_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys: frontmatter_keys, protect: protect)
|
|
96
99
|
end
|
|
97
100
|
end
|
|
98
101
|
|
|
@@ -109,18 +112,22 @@ module Polytypo
|
|
|
109
112
|
end
|
|
110
113
|
private_class_method :resolve_mode
|
|
111
114
|
|
|
112
|
-
def self.transform_text(input, locale, rules, narrow_target)
|
|
115
|
+
def self.transform_text(input, locale, rules, narrow_target, protect)
|
|
113
116
|
_resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
|
|
114
117
|
cp = Engine::Codepoints.to_codepoints(input)
|
|
118
|
+
holes = Engine::Protect.resolve(protect, cp.length)
|
|
115
119
|
ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale,
|
|
116
120
|
narrow_target: narrow_target)
|
|
121
|
+
return Engine::Protect.run(cp, holes, plan, locale_data, ctx) unless holes.empty?
|
|
122
|
+
|
|
117
123
|
result = Engine::Pipeline.run_rules(cp, plan, locale_data, ctx)
|
|
118
124
|
Engine::Codepoints.from_codepoints(result)
|
|
119
125
|
end
|
|
120
126
|
private_class_method :transform_text
|
|
121
127
|
|
|
122
|
-
def self.transform_html(input, locale, rules, narrow_target)
|
|
128
|
+
def self.transform_html(input, locale, rules, narrow_target, protect)
|
|
123
129
|
resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
|
|
130
|
+
Engine::Protect.refuse_outside_text(protect, "html")
|
|
124
131
|
spans = Modes::Html.html_spans(input)
|
|
125
132
|
ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale,
|
|
126
133
|
narrow_target: narrow_target)
|
|
@@ -134,9 +141,10 @@ module Polytypo
|
|
|
134
141
|
# counterpart: with no declared grammar to violate, a file that is not YAML yields few spans or
|
|
135
142
|
# none and comes back byte for byte (modes.md 3.8.3). The only raise this mode adds is +keys+,
|
|
136
143
|
# which is about the call and not the input.
|
|
137
|
-
def self.transform_yaml(input, locale, keys, rules, narrow_target)
|
|
144
|
+
def self.transform_yaml(input, locale, keys, rules, narrow_target, protect)
|
|
138
145
|
resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
|
|
139
146
|
resolved_keys = Engine.resolve_yaml_keys(keys)
|
|
147
|
+
Engine::Protect.refuse_outside_text(protect, "yaml")
|
|
140
148
|
spans = Modes::Yaml.yaml_spans(input, resolved_keys)
|
|
141
149
|
ctx = Engine::RuleContext.new(mode: "yaml", dialect: nil, locale: resolved_locale,
|
|
142
150
|
narrow_target: narrow_target)
|
|
@@ -145,7 +153,7 @@ module Polytypo
|
|
|
145
153
|
end
|
|
146
154
|
private_class_method :transform_yaml
|
|
147
155
|
|
|
148
|
-
def self.transform_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys
|
|
156
|
+
def self.transform_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys: nil, protect: nil)
|
|
149
157
|
# Validation order is public, tested behaviour, identical across every runtime: rules (an
|
|
150
158
|
# unknown rule id), then locale (an unknown locale), then dialect, then frontmatter_keys, then
|
|
151
159
|
# parsing (modes.md 3.7.4).
|
|
@@ -153,6 +161,7 @@ module Polytypo
|
|
|
153
161
|
require_relative "polytypo/modes/markdown"
|
|
154
162
|
Modes::Markdown.resolve_dialect(dialect)
|
|
155
163
|
resolved_keys = Engine.resolve_frontmatter_keys(frontmatter_keys)
|
|
164
|
+
Engine::Protect.refuse_outside_text(protect, "markdown")
|
|
156
165
|
ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale,
|
|
157
166
|
narrow_target: narrow_target)
|
|
158
167
|
cp = Engine::Codepoints.to_codepoints(input)
|
|
@@ -171,17 +180,21 @@ module Polytypo
|
|
|
171
180
|
private_class_method :markdown_units
|
|
172
181
|
private_class_method :transform_markdown
|
|
173
182
|
|
|
174
|
-
def self.analyze_text(input, locale, rules, narrow_target)
|
|
183
|
+
def self.analyze_text(input, locale, rules, narrow_target, protect)
|
|
175
184
|
_resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
|
|
176
185
|
cp = Engine::Codepoints.to_codepoints(input)
|
|
186
|
+
holes = Engine::Protect.resolve(protect, cp.length)
|
|
177
187
|
ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale,
|
|
178
188
|
narrow_target: narrow_target)
|
|
189
|
+
return Engine::Protect.analyze(cp, holes, plan, locale_data, ctx) unless holes.empty?
|
|
190
|
+
|
|
179
191
|
Engine::Pipeline.run_rules_recording(cp, plan, locale_data, ctx, (0...cp.length).to_a, cp.length)
|
|
180
192
|
end
|
|
181
193
|
private_class_method :analyze_text
|
|
182
194
|
|
|
183
|
-
def self.analyze_html(input, locale, rules, narrow_target)
|
|
195
|
+
def self.analyze_html(input, locale, rules, narrow_target, protect)
|
|
184
196
|
resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
|
|
197
|
+
Engine::Protect.refuse_outside_text(protect, "html")
|
|
185
198
|
spans = Modes::Html.html_spans(input)
|
|
186
199
|
ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale,
|
|
187
200
|
narrow_target: narrow_target)
|
|
@@ -191,9 +204,10 @@ module Polytypo
|
|
|
191
204
|
|
|
192
205
|
# analyze.md section 1, "yaml" mode: offsets are into the document, not into a span
|
|
193
206
|
# (analyze.md section 6).
|
|
194
|
-
def self.analyze_yaml(input, locale, keys, rules, narrow_target)
|
|
207
|
+
def self.analyze_yaml(input, locale, keys, rules, narrow_target, protect)
|
|
195
208
|
resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
|
|
196
209
|
resolved_keys = Engine.resolve_yaml_keys(keys)
|
|
210
|
+
Engine::Protect.refuse_outside_text(protect, "yaml")
|
|
197
211
|
spans = Modes::Yaml.yaml_spans(input, resolved_keys)
|
|
198
212
|
ctx = Engine::RuleContext.new(mode: "yaml", dialect: nil, locale: resolved_locale,
|
|
199
213
|
narrow_target: narrow_target)
|
|
@@ -201,13 +215,14 @@ module Polytypo
|
|
|
201
215
|
end
|
|
202
216
|
private_class_method :analyze_yaml
|
|
203
217
|
|
|
204
|
-
def self.analyze_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys
|
|
218
|
+
def self.analyze_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys: nil, protect: nil)
|
|
205
219
|
# Validation order is public, tested behaviour and is shared with .transform: rules, then
|
|
206
220
|
# locale, then dialect, then frontmatter_keys, then parsing (analyze.md section 4, A1).
|
|
207
221
|
resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
|
|
208
222
|
require_relative "polytypo/modes/markdown"
|
|
209
223
|
Modes::Markdown.resolve_dialect(dialect)
|
|
210
224
|
resolved_keys = Engine.resolve_frontmatter_keys(frontmatter_keys)
|
|
225
|
+
Engine::Protect.refuse_outside_text(protect, "markdown")
|
|
211
226
|
ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale,
|
|
212
227
|
narrow_target: narrow_target)
|
|
213
228
|
Modes::Runner.analyze_over_units(Engine::Codepoints.to_codepoints(input),
|