polytypo 1.21.0 → 1.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/polytypo/data/VERSION +1 -1
- data/lib/polytypo/data/fixtures/bg.json +12 -4
- data/lib/polytypo/data/fixtures/cs.json +9 -1
- data/lib/polytypo/data/fixtures/de-CH.json +12 -4
- data/lib/polytypo/data/fixtures/de-DE.json +113 -13
- data/lib/polytypo/data/fixtures/el.json +28 -4
- data/lib/polytypo/data/fixtures/en-GB.json +23 -7
- data/lib/polytypo/data/fixtures/en-US.json +586 -162
- data/lib/polytypo/data/fixtures/es.json +1 -1
- data/lib/polytypo/data/fixtures/fi.json +10 -2
- data/lib/polytypo/data/fixtures/fr-CA.json +9 -1
- data/lib/polytypo/data/fixtures/fr.json +46 -13
- data/lib/polytypo/data/fixtures/hr.json +9 -1
- data/lib/polytypo/data/fixtures/hu.json +28 -4
- data/lib/polytypo/data/fixtures/it.json +1 -1
- data/lib/polytypo/data/fixtures/locale-resolution.json +1 -1
- data/lib/polytypo/data/fixtures/lt.json +9 -1
- data/lib/polytypo/data/fixtures/nl.json +17 -1
- data/lib/polytypo/data/fixtures/pl.json +17 -1
- data/lib/polytypo/data/fixtures/pt-BR.json +9 -1
- data/lib/polytypo/data/fixtures/pt-PT.json +9 -1
- data/lib/polytypo/data/fixtures/ro.json +2 -2
- data/lib/polytypo/data/fixtures/ru.json +38 -6
- data/lib/polytypo/data/fixtures/sk.json +9 -1
- data/lib/polytypo/data/fixtures/sl.json +9 -1
- data/lib/polytypo/data/fixtures/sr.json +12 -4
- data/lib/polytypo/data/fixtures/sv.json +12 -4
- data/lib/polytypo/data/fixtures/tr.json +9 -1
- data/lib/polytypo/data/fixtures/uk.json +1 -1
- data/lib/polytypo/data/locales/bg.json +9 -1
- data/lib/polytypo/data/locales/cs.json +2 -0
- data/lib/polytypo/data/locales/de-CH.json +9 -1
- data/lib/polytypo/data/locales/de-DE.json +9 -1
- data/lib/polytypo/data/locales/el.json +9 -1
- data/lib/polytypo/data/locales/en-GB.json +9 -4
- data/lib/polytypo/data/locales/en-US.json +11 -11
- data/lib/polytypo/data/locales/es.json +2 -0
- data/lib/polytypo/data/locales/fi.json +8 -3
- data/lib/polytypo/data/locales/fr-CA.json +9 -1
- data/lib/polytypo/data/locales/fr.json +3 -1
- data/lib/polytypo/data/locales/hr.json +3 -1
- data/lib/polytypo/data/locales/hu.json +9 -1
- data/lib/polytypo/data/locales/it.json +2 -0
- data/lib/polytypo/data/locales/lt.json +12 -5
- data/lib/polytypo/data/locales/nl.json +10 -2
- data/lib/polytypo/data/locales/pl.json +10 -2
- data/lib/polytypo/data/locales/pt-BR.json +4 -2
- data/lib/polytypo/data/locales/pt-PT.json +5 -3
- data/lib/polytypo/data/locales/registry.json +1 -1
- data/lib/polytypo/data/locales/ro.json +2 -0
- data/lib/polytypo/data/locales/ru.json +9 -17
- data/lib/polytypo/data/locales/sk.json +7 -5
- data/lib/polytypo/data/locales/sl.json +3 -1
- data/lib/polytypo/data/locales/sr.json +9 -2
- data/lib/polytypo/data/locales/sv.json +8 -6
- data/lib/polytypo/data/locales/tr.json +4 -2
- data/lib/polytypo/data/locales/uk.json +2 -0
- data/lib/polytypo/data/rules/modes.md +2 -2
- data/lib/polytypo/data/rules/nbsp.md +271 -74
- data/lib/polytypo/data/rules/order.json +1 -1
- data/lib/polytypo/data/rules/pipeline-idempotency.md +54 -7
- data/lib/polytypo/data/rules/quote-punctuation.md +3 -0
- data/lib/polytypo/data/rules/quotes.md +14 -5
- data/lib/polytypo/data/rules/ranges.md +7 -1
- data/lib/polytypo/data/rules/spaces.md +4 -2
- data/lib/polytypo/data/rules/symbols.md +2 -1
- data/lib/polytypo/data/schema/locale.schema.json +15 -1
- data/lib/polytypo/engine/rules/nbsp.rb +161 -11
- data/lib/polytypo/version.rb +1 -1
- metadata +1 -1
|
@@ -134,7 +134,11 @@ U+0020 in the wider set emits none in the narrower one.
|
|
|
134
134
|
|
|
135
135
|
**Who can violate it.** Only a rule that emits U+0020. In the current pipeline that is
|
|
136
136
|
`dashes` alone (`-spaced` forms). `nbsp` emits U+00A0 and U+202F, which are `CONTENT` to
|
|
137
|
-
`spaces` and are never touched. `
|
|
137
|
+
`spaces` and are never touched. `nbsp` also deletes: since spec 1.22.0 N5 removes the single
|
|
138
|
+
separator between a `DIGIT` and a `closedUnits` entry. That separator has a `DIGIT` on its left
|
|
139
|
+
and, by `nbsp.md` §3.7 U2, a code point outside `SPACELIKE` on its right, so it is a run of
|
|
140
|
+
length one with no space beside it, and removing it lengthens, shortens, merges or re-neighbours
|
|
141
|
+
no other U+0020 run (`nbsp.md` §5, composition obligation against `I₁`). `quotes` (0.3.0) deletes a code point inside a quote pair when
|
|
138
142
|
`innerSpace = "none"`, but never a U+0020 that survives the deletion — the maximal-run deletion
|
|
139
143
|
and the landing guard together ensure the two surviving neighbours are always `CONTENT`
|
|
140
144
|
(`quotes.md` 3.7, 5). Every other rule replaces code points one-for-one or shortens a run, and
|
|
@@ -189,7 +193,23 @@ whose two sides are a quote glyph and a `DELETE-LANDING` member — neither is a
|
|
|
189
193
|
`CLOSED-SYMBOL`, so no range member's `before`/`after` moves. `symbols` (R₇) emits only U+00A9,
|
|
190
194
|
U+00AE and U+2122 and deletes only a `(`…`)` span. `nbsp` (R₈) converts a space it found and
|
|
191
195
|
never inserts one where a guard reads (`nbsp.md` §3.3), and U+00A0/U+202F satisfy none of G1, G2
|
|
192
|
-
or G3.
|
|
196
|
+
or G3. N5's insertion and deletion (`nbsp.md` §3.7, 1.22.0) are the one place R₈ adds or removes
|
|
197
|
+
an `INLINE-SPACE` beside a `DIGIT` and a unit that may be a `CLOSED-SYMBOL`; guard G-R of that
|
|
198
|
+
section declines both whenever the first neighbour beyond the spaces, on either side, is a `DASH`
|
|
199
|
+
or `INERT-DASH`, which are the only positions where G1–G3 read that gap.
|
|
200
|
+
|
|
201
|
+
That discharge covers a unit whose first code point is in `CLOSED-SYMBOL` (`%`, `‰`, `°C`, the
|
|
202
|
+
currency signs), and only that: it is the only kind of unit that `ranges` §3.2a and T1's p1/p2
|
|
203
|
+
step over. A unit such as `km` or `kg` is **not read** by `ranges` or `dashes`. Neither walks over
|
|
204
|
+
a letter, so such a unit is never a range member and never a flank, and the only reads that reach
|
|
205
|
+
the gap before it are the effective neighbours of a digit run: `after` in G2 and G3 (a `DASH`,
|
|
206
|
+
an `INERT-DASH` or U+002F) and T1's first and second neighbour (a `DASH` or an `INERT-DASH`).
|
|
207
|
+
What those reads find at the two positions differs between the runs (the unit's first code
|
|
208
|
+
point, against a no-break space and then that code point; or a separator and then that code
|
|
209
|
+
point, against that code point alone), and the only code points involved are a space and the
|
|
210
|
+
unit's first code point, neither of which is in those sets: `nbsp.md` §3.7 U2 states that no unit
|
|
211
|
+
begins with one. For a unit whose first code point is outside `CLOSED-SYMBOL`, then, G-R is not
|
|
212
|
+
needed and is not applied, and the argument is this one.
|
|
193
213
|
|
|
194
214
|
### I₃ — `dashes`
|
|
195
215
|
|
|
@@ -216,14 +236,21 @@ replaces code points one-for-one with quote glyphs (also outside every `dashes`
|
|
|
216
236
|
`innerSpace = "none"`, deletes a maximal `INLINE-SPACE` run whose two sides are a quote glyph and
|
|
217
237
|
a `DELETE-LANDING` member — neither in `DASH ∪ INERT-DASH`, so no dash run is ever adjacent to a
|
|
218
238
|
deleted run and no token's `lsp`/`rsp` changes (`quotes.md` 3.7, 5, composition obligation
|
|
219
|
-
against `I₃`).
|
|
239
|
+
against `I₃`). `nbsp`'s N5 deletes a separator with a `DIGIT` on its left and a code point outside
|
|
240
|
+
`DASH` ∪ `INERT-DASH` on its right (`nbsp.md` §3.7 U2), so no dash run is adjacent to it; the
|
|
241
|
+
longer reads of T1 and of `ranges`' `after` are the ones discharged under `I₂ₐ`.
|
|
220
242
|
|
|
221
243
|
### I₄ — `hyphen`
|
|
222
244
|
|
|
223
245
|
`I₄(y)` requires no occurrence of a listed `hyphen` form whose hyphen is still U+002D. A later
|
|
224
246
|
rule violates it only by changing a form's word boundaries, i.e. by inserting or removing a
|
|
225
|
-
code point immediately beside a listed form. `nbsp` inserts
|
|
226
|
-
|
|
247
|
+
code point immediately beside a listed form. `nbsp` inserts next to listed punctuation or beside
|
|
248
|
+
a quote glyph, so the inserted space there never lands between two letters; its one other
|
|
249
|
+
insertion, N5's between a `DIGIT` and a `spacedUnits` entry, can land between a digit and a
|
|
250
|
+
letter, and is shown not to change a form's boundary in `nbsp.md` §5, composition obligation
|
|
251
|
+
against `I₄` (the unit ends in a letter or at the form's hyphen, which N5's right-context guard
|
|
252
|
+
declines); N5's deletion exposes a code point that is neither a letter nor a hyphen (U2, U3), and every form
|
|
253
|
+
begins with one or the other; `symbols`
|
|
227
254
|
deletes only `(`…`)` spans; `quotes` (0.3.0) deletes only a run whose landing is in
|
|
228
255
|
`ALNUM ∪ QUOTEMARK`, but the deletion always removes an `INLINE-SPACE` run that is *not itself*
|
|
229
256
|
inside a word — `hyphen`'s `WORDISH = ALNUM ∪ HYPHENISH` excludes both a U+0020 and a quote
|
|
@@ -239,7 +266,8 @@ space-like code point or the end. It is vacuous for a locale whose
|
|
|
239
266
|
**Who can violate it.** `apostrophe`, `symbols` and `nbsp`, by changing the class of `cp[i-1]` or
|
|
240
267
|
`cp[i+2]` or by putting a punctuation mark next to a primary close. The argument is in
|
|
241
268
|
`quote-punctuation.md` §5: none of them emits the primary close (QP-1), a letter or a
|
|
242
|
-
comma or full stop next to it, or anything that is not `SPACELIKE` where a space stood.
|
|
269
|
+
comma or full stop next to it, or anything that is not `SPACELIKE` where a space stood. `nbsp`'s N5
|
|
270
|
+
deletes a separator only after a `DIGIT`, never after a comma or full stop.
|
|
243
271
|
|
|
244
272
|
### I₅ … I₈
|
|
245
273
|
|
|
@@ -257,7 +285,10 @@ available. `quotes.md` §5's replacement is two lemmas plus a certification gate
|
|
|
257
285
|
own output rather than relying on an argument about it: **Lemma A** (glyph-blindness — no
|
|
258
286
|
candidate's verdict depends on *which* quote glyph a neighbour is) makes `apostrophe`'s emission
|
|
259
287
|
structurally inert as a neighbour (Corollary A1); **Lemma B** (space-inertness at every position
|
|
260
|
-
`nbsp` can reach) makes `nbsp`'s three insertion sites structurally inert
|
|
288
|
+
`nbsp` can reach) makes `nbsp`'s three insertion sites around quote marks structurally inert; N5's insertion and
|
|
289
|
+
deletion (`nbsp.md` §3.7) lie between a `DIGIT` and a unit that begins with no quotation glyph
|
|
290
|
+
and no U+0022 or U+0027 (`nbsp.md` §3.7 U1, U2), so they are not adjacent to any quote mark
|
|
291
|
+
(`quotes.md` §5, Lemma B item 4). Both are CO-S
|
|
261
292
|
discharges in the sense of §5.1a below, over a *reachable-position* alphabet rather than a
|
|
262
293
|
whole-emission alphabet for Lemma B specifically — see `quotes.md` §5 for why the alphabet alone
|
|
263
294
|
is not sufficient there.
|
|
@@ -450,6 +481,7 @@ The idempotency property test must include, in every runtime:
|
|
|
450
481
|
| **wide** | the consumed **and** emitted sets below | 0–5 |
|
|
451
482
|
| **deep** | a core of at least `1`, `-`, U+0020, `.`, U+2060 | 0–8 |
|
|
452
483
|
| **quotes** (spec 0.3.0) | `{ U+0022, U+0027, U+002D, U+0020, a }` ∪ every distinct quote glyph in `spec/locales/registry.json`'s locales | 0–6, per locale |
|
|
484
|
+
| **units** (spec 1.22.0, **mandatory**) | `{ 1, U+0020, U+00A0, x, х (U+0445), U+002D, U+2013, U+2014, U+2060, % }` ∪ one `nbsp.spacedUnits` entry ∪ one `nbsp.closedUnits` entry of that locale, each entry **one symbol** | 0–6, per locale |
|
|
453
485
|
|
|
454
486
|
**Why the quotes tier exists, separately from wide/deep.** `quotes` (0.3.0)'s worst witness —
|
|
455
487
|
`‘""-"-"` in `en-GB` — is seven code points over `{ ‘ " - }`, and the committed deep tier's
|
|
@@ -459,6 +491,21 @@ The idempotency property test must include, in every runtime:
|
|
|
459
491
|
to that single, self-consistent alphabet is the natural unit rather than folding it into wide
|
|
460
492
|
or deep, which are keyed to different rules' emission alphabets.
|
|
461
493
|
|
|
494
|
+
**Why the units tier exists, and what it must contain.** N5 inserts and deletes (`nbsp.md`
|
|
495
|
+
§3.7, spec 1.22.0), and the neighbourhood where it does so is the one place `nbsp` is read by
|
|
496
|
+
`ranges`, `dashes` and `symbols` at once: a digit run, a separator, a unit, a dash and a
|
|
497
|
+
multiplication letter. None of the other tiers contains a unit, so none can reach a defect in
|
|
498
|
+
that composition, and the two rules whose verdicts N5 can move (`ranges`/`dashes` through the
|
|
499
|
+
dash and `CLOSED-SYMBOL` reads, `symbols` through guard M3) are exactly the two guarded by G-R
|
|
500
|
+
and 5.5. The alphabet has the digit, both spaces N5 distinguishes (U+0020 and U+00A0), both
|
|
501
|
+
Latin and Cyrillic `MUL-LETTER` (M3), the hyphen, en and em dash and the joiner (G-R and T1),
|
|
502
|
+
and `%`, a `CLOSED-SYMBOL`; the two entries stand for the other unit kinds, and a locale
|
|
503
|
+
whose list is empty contributes none. A locale whose `spacedUnits` or `closedUnits` is
|
|
504
|
+
empty runs the tier without that entry (a `%` that is in no list is inert, but the dash and
|
|
505
|
+
chain reads are still exercised). Length 6 is the least that holds the guard-5.5 witness
|
|
506
|
+
`2␣x␣3km` (digit, space, letter, space, digit, unit), and the sweep must be run for both the
|
|
507
|
+
per-rule and the composition assertions (2) and (3) below, not only for the full pipeline. With twelve symbols the tier is about three million strings per locale.
|
|
508
|
+
|
|
462
509
|
**Why two tiers, and why the bound moved.** A single length-4 bound was normative until
|
|
463
510
|
`dashes` defect (e) — witness `1-1 - 1`, **seven characters** — shipped underneath it. Nothing
|
|
464
511
|
was excluded and no precondition hid it; the bound simply could not reach it, and the suite
|
|
@@ -217,6 +217,9 @@ both before and after the edit.
|
|
|
217
217
|
`Q` and the punctuation (N1/N2) only if the locale lists that punctuation in
|
|
218
218
|
`beforePunctuation` or `narrowBeforePunctuation`, which `en-US` does not; if a locale did, the
|
|
219
219
|
insertion would make `cp[i+1]` a space, which fails C2, so the pair is no longer a candidate.
|
|
220
|
+
`nbsp`'s N5 also deletes a single separator (`nbsp.md` §3.7, spec 1.22.0), but only one that
|
|
221
|
+
follows a `DIGIT`; C4 needs the space to follow a comma or full stop, so a deleted separator is
|
|
222
|
+
never the code point C4 reads.
|
|
220
223
|
|
|
221
224
|
**What is argued and what is measured.** The discharges against `ellipsis`, `ranges`, `dashes`,
|
|
222
225
|
`hyphen` and `nbsp` are class-membership arguments over the alphabet `{Q, U+002C, U+002E}` and
|
|
@@ -1161,12 +1161,14 @@ rest of this rule's determinism.
|
|
|
1161
1161
|
|
|
1162
1162
|
### Lemma B — space-inertness at every position `nbsp` can reach
|
|
1163
1163
|
|
|
1164
|
-
> Inserting a code point of `INLINE-SPACE`, or converting one `INLINE-SPACE`
|
|
1165
|
-
> another, at any position `nbsp` can act on, changes no capability of any
|
|
1164
|
+
> Inserting a code point of `INLINE-SPACE`, deleting one, or converting one `INLINE-SPACE`
|
|
1165
|
+
> code point to another, at any position `nbsp` can act on, changes no capability of any
|
|
1166
|
+
> candidate.
|
|
1166
1167
|
|
|
1167
1168
|
*Proof.* Conversion is trivial: every test reads class membership and U+0020, U+00A0, U+202F are
|
|
1168
|
-
all in `INLINE-SPACE`. For insertion, `nbsp` has exactly three insertion sites
|
|
1169
|
-
|
|
1169
|
+
all in `INLINE-SPACE`. For insertion, `nbsp` has exactly three insertion sites **around quote
|
|
1170
|
+
marks**, items 1 to 3 below (N8 on either side of a glyph, and N1/N2 before a listed mark), and
|
|
1171
|
+
one site that is not around a quote mark, N5's, which item 4 covers together with N5's deletion:
|
|
1170
1172
|
|
|
1171
1173
|
1. **N8, immediately right of an occurrence of `g ∈ SPACE-RIGHT`.**
|
|
1172
1174
|
- *`g` itself*: `canOpen` reads `Rskip` (skips) and `openLeft` (untouched); `canClose` reads
|
|
@@ -1187,7 +1189,14 @@ all in `INLINE-SPACE`. For insertion, `nbsp` has exactly three insertion sites (
|
|
|
1187
1189
|
`m ∈ CLOSEISH`, so both are accepted — same verdict. Its `Rskip` skips back to `m`. (`nbsp`'s
|
|
1188
1190
|
own step 3 already declines to insert after an opening glyph.)
|
|
1189
1191
|
|
|
1190
|
-
|
|
1192
|
+
4. **N5 (`spacedUnits` insertion, `closedUnits` deletion; `nbsp.md` §3.7).** The gap lies between
|
|
1193
|
+
a `DIGIT` and the first code point of a listed unit; a listed unit never begins or ends with one of the
|
|
1194
|
+
locale's own quote glyphs (`scripts/validate-spec.mjs` enforces it) and never begins with a
|
|
1195
|
+
straight U+0022 or U+0027 (`nbsp.md` §3.7 U2), so no
|
|
1196
|
+
quote mark is adjacent to the gap and no `Lskip` / `Rskip` read from a quote mark crosses it.
|
|
1197
|
+
No candidate's capability moves.
|
|
1198
|
+
|
|
1199
|
+
No other sub-rule inserts; N3, N4, N6, N7, N9, N10 only convert. ∎
|
|
1191
1200
|
|
|
1192
1201
|
**V1's `gapInsertable` clause (§3.2) is what extends this proof to V1 itself.** The four
|
|
1193
1202
|
capability tests above are the ones Lemma B's statement covers directly, but V1 is also a test a
|
|
@@ -487,4 +487,10 @@ machinery rather than range-specific stay in [dashes.md](dashes.md) §7 (its can
|
|
|
487
487
|
the locale sets **after a space** — `20 °C`, `20 %`. That is the spaced construction, out of
|
|
488
488
|
scope here; `15°-20°` and `35%-50%`, with the symbol closed up, are the other one. The two
|
|
489
489
|
never meet in the algorithm either: §3.2a's walk requires a `DIGIT` immediately beside the
|
|
490
|
-
symbol, so `15 %-20 %` is not a candidate at all.
|
|
490
|
+
symbol, so `15 %-20 %` is not a candidate at all. A locale's `nbsp.spacedUnits` and
|
|
491
|
+
`nbsp.closedUnits` (spec 1.22.0) are in the same position, with one addition: `nbsp` N5 can
|
|
492
|
+
insert or delete the space itself, and so turn the spaced construction into the closed one or
|
|
493
|
+
the reverse, after `ranges` has run. They still never meet, but the reason is `nbsp.md` §3.7
|
|
494
|
+
and not the walk: its guard G-R declines both the insertion and the deletion whenever a `DASH`
|
|
495
|
+
or `INERT-DASH` is the first code point beyond the number and its unit on either side (past
|
|
496
|
+
any spaces and U+2060), which is where a range member's symbol would be read.
|
|
@@ -478,8 +478,10 @@ without emitting any U+0020. None does: `ellipsis` writes only `DOTLIKE` code po
|
|
|
478
478
|
`dashes` and `hyphen` replace dashes, hyphens and spaces; `quotes` and `apostrophe` replace
|
|
479
479
|
quotation marks; `symbols` replaces a trademark literal, which starts with `(`, and a `MUL-LETTER`,
|
|
480
480
|
which is always preceded by a digit or a space and so never follows a dot directly; `nbsp` replaces
|
|
481
|
-
or inserts spaces, and inserts only beside a listed punctuation mark or a quote glyph,
|
|
482
|
-
a
|
|
481
|
+
or inserts spaces, and inserts only beside a listed punctuation mark or a quote glyph or (N5, for a
|
|
482
|
+
`spacedUnits` entry) between a digit and a unit, never between a dot and a letter, and deletes only
|
|
483
|
+
the single separator between a digit and a `closedUnits` entry, whose neighbours are a digit and a
|
|
484
|
+
non-letter. A new rule that rewrites letters or digits must re-check this.
|
|
483
485
|
|
|
484
486
|
---
|
|
485
487
|
|
|
@@ -386,7 +386,8 @@ glyph, or deletes one from the *right* of a mark, this paragraph must be re-deri
|
|
|
386
386
|
**A standing note for anyone adding a row here.** These rows become conformance fixtures
|
|
387
387
|
verbatim, so each one must exercise **only** the rule it documents. Choose the surrounding words
|
|
388
388
|
so that no _other_ rule fires: no unit or unit-like token (`mm`, `cm`, `см`, `km`, `%`, `°C`),
|
|
389
|
-
which `nbsp
|
|
389
|
+
which `nbsp` binds to a preceding number with U+00A0 (`beforeUnits`, `spacedUnits`), or whose
|
|
390
|
+
space it inserts or deletes (`spacedUnits`, `closedUnits`); no abbreviation ending in a
|
|
390
391
|
full stop, which `nbsp.beforeNumber` / `beforeWord` bind; and no short function word, which
|
|
391
392
|
`nbsp.afterShortWords` binds. A row reading `Tolerance ±5 mm` teaches a reader that `symbols`
|
|
392
393
|
produces the space it does not produce — the true pipeline output carries a U+00A0 that belongs
|
|
@@ -97,6 +97,8 @@
|
|
|
97
97
|
"afterShortWords",
|
|
98
98
|
"abbreviations",
|
|
99
99
|
"beforeUnits",
|
|
100
|
+
"spacedUnits",
|
|
101
|
+
"closedUnits",
|
|
100
102
|
"beforeNumber",
|
|
101
103
|
"beforeWord",
|
|
102
104
|
"afterSymbols",
|
|
@@ -127,7 +129,19 @@
|
|
|
127
129
|
"type": "array",
|
|
128
130
|
"items": { "type": "string", "minLength": 1 },
|
|
129
131
|
"uniqueItems": true,
|
|
130
|
-
"description": "Units and signs that bind to a preceding number with U+00A0. Example: \"%\", \"km\", \"°C\"."
|
|
132
|
+
"description": "Units and signs that bind to a preceding number with U+00A0 when the author already typed a space. Conversion only: a missing space is never inserted and an existing one is never removed. Use for a unit whose spacing the locale's sources leave to the author or make depend on context. Example: \"%\", \"km\", \"°C\". Disjoint from spacedUnits and closedUnits (scripts/validate-spec.mjs)."
|
|
133
|
+
},
|
|
134
|
+
"spacedUnits": {
|
|
135
|
+
"type": "array",
|
|
136
|
+
"items": { "type": "string", "minLength": 1 },
|
|
137
|
+
"uniqueItems": true,
|
|
138
|
+
"description": "Units and signs the locale writes with a space after the number. N5 binds an existing space with U+00A0 and inserts U+00A0 when the number and the unit touch (nbsp.md 3.7). Disjoint from beforeUnits, closedUnits and afterSymbols (scripts/validate-spec.mjs). Example: \"%\" in German, \"°C\" in Russian."
|
|
139
|
+
},
|
|
140
|
+
"closedUnits": {
|
|
141
|
+
"type": "array",
|
|
142
|
+
"items": { "type": "string", "minLength": 1 },
|
|
143
|
+
"uniqueItems": true,
|
|
144
|
+
"description": "Units and signs the locale writes closed up to the number. N5 deletes a single U+0020, U+00A0 or U+202F between the number and the unit (nbsp.md 3.7). Disjoint from beforeUnits, spacedUnits and afterSymbols (scripts/validate-spec.mjs). Example: \"%\" in English."
|
|
131
145
|
},
|
|
132
146
|
"beforeNumber": {
|
|
133
147
|
"type": "array",
|
|
@@ -127,6 +127,18 @@ module Polytypo
|
|
|
127
127
|
keyword_init: true,
|
|
128
128
|
)
|
|
129
129
|
|
|
130
|
+
# One N5 entry: the unit as code points and which list it came from.
|
|
131
|
+
UnitEntry = Struct.new(:w, :kind)
|
|
132
|
+
|
|
133
|
+
# nbsp.md 3.7: the three unit lists merged so the longest entry wins across all of them.
|
|
134
|
+
def self.prepare_units(data)
|
|
135
|
+
out = []
|
|
136
|
+
[["beforeUnits", :before], ["spacedUnits", :spaced], ["closedUnits", :closed]].each do |field, kind|
|
|
137
|
+
data[field].each { |entry| out << UnitEntry.new(entry.codepoints, kind) }
|
|
138
|
+
end
|
|
139
|
+
out.each_with_index.sort_by { |entry, i| [-entry.w.length, i] }.map(&:first)
|
|
140
|
+
end
|
|
141
|
+
|
|
130
142
|
def self.malformed(message)
|
|
131
143
|
raise Polytypo::Error.new(Polytypo::CODE_MALFORMED_LOCALE_DATA, message)
|
|
132
144
|
end
|
|
@@ -202,7 +214,7 @@ module Polytypo
|
|
|
202
214
|
narrow_before_punctuation: narrow_before_punctuation,
|
|
203
215
|
short_words: prepare_list(data["afterShortWords"]),
|
|
204
216
|
abbreviations: prepare_list(data["abbreviations"]),
|
|
205
|
-
units:
|
|
217
|
+
units: prepare_units(data),
|
|
206
218
|
before_number: prepare_list(data["beforeNumber"]),
|
|
207
219
|
before_word: prepare_list(data["beforeWord"]),
|
|
208
220
|
symbols: prepare_list(data["afterSymbols"]),
|
|
@@ -389,29 +401,167 @@ module Polytypo
|
|
|
389
401
|
end
|
|
390
402
|
end
|
|
391
403
|
|
|
392
|
-
#
|
|
404
|
+
# nbsp.md 3.7 step 5.1: what may stand left of a number that N5 inserts or deletes beside.
|
|
405
|
+
NUMBER_PREFIXES = [0x2B, 0xB1, 0xD7, 0x2212, 0x7E, 0x2248, 0x3C, 0x3E, 0x2060].freeze
|
|
406
|
+
HYPHEN_MINUS = 0x2D
|
|
407
|
+
NON_BREAKING_HYPHEN = 0x2011
|
|
408
|
+
LOW_LINE = 0x5F
|
|
409
|
+
COMMA = 0x2C
|
|
410
|
+
WORD_JOINER = 0x2060
|
|
411
|
+
|
|
412
|
+
# ranges.md 3.2a CLOSED-SYMBOL, mirrored (this rule does not depend on `ranges`).
|
|
413
|
+
def self.closed_symbol?(cp)
|
|
414
|
+
cp == 0x24 || (cp >= 0xA2 && cp <= 0xA5) || (cp >= 0x20A0 && cp <= 0x20CF) ||
|
|
415
|
+
cp == 0x25 || cp == 0x2030 || cp == 0x2031 || cp == 0xB0
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
ANY_DASH = [
|
|
419
|
+
0x2D, 0x2010, 0x2013, 0x2014, 0x2212, 0xAD, 0x2011, 0x2012, 0x2015, 0xFE58, 0xFE63, 0xFF0D,
|
|
420
|
+
].freeze
|
|
421
|
+
|
|
422
|
+
# dashes.md 3.1 DASH + INERT-DASH.
|
|
423
|
+
def self.any_dash?(cp)
|
|
424
|
+
ANY_DASH.include?(cp)
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
# 3.7 step 5.4: the first code point past a run of non-breaking-line spaces and joiners.
|
|
428
|
+
def self.nearest_beyond(cp, from, step)
|
|
429
|
+
i = from
|
|
430
|
+
while i >= 0 && i < cp.length
|
|
431
|
+
c = cp[i]
|
|
432
|
+
if c == WORD_JOINER || (space_like?(c) && !break?(c))
|
|
433
|
+
i += step
|
|
434
|
+
next
|
|
435
|
+
end
|
|
436
|
+
return c
|
|
437
|
+
end
|
|
438
|
+
NONE
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
# Start of the number whose last digit is at `end_index`, extended over a decimal or
|
|
442
|
+
# thousands separator (3.7 step 5.1).
|
|
443
|
+
def self.number_start(cp, end_index)
|
|
444
|
+
b = end_index
|
|
445
|
+
b -= 1 while b - 1 >= 0 && digit?(cp[b - 1])
|
|
446
|
+
while (at(cp, b - 1) == COMMA || at(cp, b - 1) == FULL_STOP) && digit?(at(cp, b - 2))
|
|
447
|
+
b -= 2
|
|
448
|
+
b -= 1 while b - 1 >= 0 && digit?(cp[b - 1])
|
|
449
|
+
end
|
|
450
|
+
b
|
|
451
|
+
end
|
|
452
|
+
|
|
453
|
+
MUL_LETTERS = [0x78, 0x58, 0x445, 0x425].freeze
|
|
454
|
+
|
|
455
|
+
# symbols.md 3.1 SPACE | NOBREAK-SPACE: U+0020, U+00A0 and U+202F.
|
|
456
|
+
def self.mul_space?(cp)
|
|
457
|
+
cp == SPACE || cp == NBSP || cp == NNBSP
|
|
458
|
+
end
|
|
459
|
+
|
|
460
|
+
# 3.7 step 5.5: DIGIT [space] MUL-LETTER [space] immediately before the number start.
|
|
461
|
+
def self.right_operand_of_multiplication?(cp, b)
|
|
462
|
+
i = b - 1
|
|
463
|
+
i -= 1 if mul_space?(at(cp, i))
|
|
464
|
+
return false unless MUL_LETTERS.include?(at(cp, i))
|
|
465
|
+
|
|
466
|
+
i -= 1
|
|
467
|
+
i -= 1 if mul_space?(at(cp, i))
|
|
468
|
+
digit?(at(cp, i))
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
def self.number_start_clean?(cp, prep, b)
|
|
472
|
+
c = at(cp, b - 1)
|
|
473
|
+
return true if c == NONE || space_like?(c) || openish?(prep, c) || sentence_dash?(c)
|
|
474
|
+
return true if NUMBER_PREFIXES.include?(c)
|
|
475
|
+
return false if c != HYPHEN_MINUS
|
|
476
|
+
|
|
477
|
+
before = at(cp, b - 2)
|
|
478
|
+
before == NONE || space_like?(before) || openish?(prep, before)
|
|
479
|
+
end
|
|
480
|
+
|
|
481
|
+
# beforeUnits: convert one existing space after a number, with the letter guard
|
|
482
|
+
# ("H2 O", "A4", "MP3" are not measurements).
|
|
483
|
+
def self.convert_before_unit(cp, a, left, claims)
|
|
484
|
+
return if left != SPACE
|
|
485
|
+
return unless digit?(at(cp, a - 2))
|
|
486
|
+
|
|
487
|
+
b = a - 2
|
|
488
|
+
b -= 1 while b - 1 >= 0 && digit?(cp[b - 1])
|
|
489
|
+
return if UnicodeUtil.letter?(at(cp, b - 1))
|
|
490
|
+
|
|
491
|
+
claim_conversion(claims, a - 1, NBSP)
|
|
492
|
+
end
|
|
493
|
+
|
|
494
|
+
# nbsp.md 3.7 step 5: the guards on inserting or deleting the separator.
|
|
495
|
+
def self.unit_edit_allowed?(cp, entry, a, start, closed)
|
|
496
|
+
k = entry.w.length
|
|
497
|
+
# A closed-up symbol beside a dash decides whether `ranges` and `dashes` read the
|
|
498
|
+
# token as a range; editing the space between the number and the symbol would change
|
|
499
|
+
# that verdict on the next pass (step 5.3).
|
|
500
|
+
if closed_symbol?(entry.w[0]) &&
|
|
501
|
+
(any_dash?(nearest_beyond(cp, start - 1, -1)) || any_dash?(nearest_beyond(cp, a + k, 1)))
|
|
502
|
+
return false
|
|
503
|
+
end
|
|
504
|
+
return false if !closed && right_operand_of_multiplication?(cp, start)
|
|
505
|
+
|
|
506
|
+
following = at(cp, a + k)
|
|
507
|
+
return false if following == HYPHEN_MINUS || following == NON_BREAKING_HYPHEN || following == LOW_LINE
|
|
508
|
+
|
|
509
|
+
!(following == FULL_STOP && alnum?(at(cp, a + k + 1)))
|
|
510
|
+
end
|
|
511
|
+
|
|
512
|
+
# N5 -- nbsp.md 3.7. beforeUnits converts an existing space, spacedUnits converts and
|
|
513
|
+
# inserts, closedUnits deletes. The longest entry across the three lists wins at `a`.
|
|
393
514
|
def self.units_sub_rule(cp, prep, claims)
|
|
394
515
|
return if prep.units.empty?
|
|
395
516
|
|
|
396
517
|
(0...cp.length).each do |a|
|
|
397
|
-
|
|
398
|
-
next if
|
|
518
|
+
entry = prep.units.find { |candidate| match_exact?(cp, a, candidate.w) }
|
|
519
|
+
next if entry.nil?
|
|
399
520
|
|
|
400
|
-
k = w.length
|
|
521
|
+
k = entry.w.length
|
|
401
522
|
next if alnum?(at(cp, a + k))
|
|
402
523
|
|
|
403
524
|
left = at(cp, a - 1)
|
|
404
|
-
next if left == NBSP # already correct
|
|
405
|
-
next if left != SPACE
|
|
406
525
|
|
|
407
|
-
|
|
526
|
+
if entry.kind == :before
|
|
527
|
+
convert_before_unit(cp, a, left, claims)
|
|
528
|
+
next
|
|
529
|
+
end
|
|
530
|
+
|
|
531
|
+
closed = entry.kind == :closed
|
|
532
|
+
separated = left == SPACE || left == NBSP || (closed && left == NNBSP)
|
|
533
|
+
touching = digit?(left)
|
|
534
|
+
next if !separated && !(touching && !closed)
|
|
535
|
+
next if !closed && left == NBSP
|
|
536
|
+
|
|
537
|
+
end_index = separated ? a - 2 : a - 1
|
|
538
|
+
next unless digit?(at(cp, end_index))
|
|
408
539
|
|
|
409
|
-
b =
|
|
540
|
+
b = end_index
|
|
410
541
|
b -= 1 while b - 1 >= 0 && digit?(cp[b - 1])
|
|
411
|
-
# The letter guard: "H2 O", "A4", "MP3" are not measurements.
|
|
412
542
|
next if UnicodeUtil.letter?(at(cp, b - 1))
|
|
413
543
|
|
|
414
|
-
|
|
544
|
+
# Converting an existing space keeps beforeUnits' behaviour; the guards below protect
|
|
545
|
+
# only what N5 would otherwise insert or delete (3.7 step 5).
|
|
546
|
+
if !closed && separated
|
|
547
|
+
claim_conversion(claims, a - 1, NBSP) if left == SPACE
|
|
548
|
+
next
|
|
549
|
+
end
|
|
550
|
+
|
|
551
|
+
start = number_start(cp, end_index)
|
|
552
|
+
next unless number_start_clean?(cp, prep, start)
|
|
553
|
+
|
|
554
|
+
next unless unit_edit_allowed?(cp, entry, a, start, closed)
|
|
555
|
+
|
|
556
|
+
following = at(cp, a + k)
|
|
557
|
+
|
|
558
|
+
if closed
|
|
559
|
+
next if following == SPACE && digit?(at(cp, a + k + 1))
|
|
560
|
+
|
|
561
|
+
claims[a - 1] = Edit.new(a - 1, a, [], "nbsp") if claims[a - 1].nil?
|
|
562
|
+
else
|
|
563
|
+
claim_insertion(claims, a, NBSP)
|
|
564
|
+
end
|
|
415
565
|
end
|
|
416
566
|
end
|
|
417
567
|
|
data/lib/polytypo/version.rb
CHANGED