polytypo 1.8.1 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d0c292d4d4d7848c37d1b537a94a9b4ed57f2b19f02976572f87c17679b6653
-  data.tar.gz: 8ae273d9d48fe370fd406549bcfeffb3ad43dc2f4791a29cee83d791e1e481d5
+  metadata.gz: e84bade0cfecbed7a333cf29ef6bb7ea793a349805cfbd7b8f1f18f7d5ccd721
+  data.tar.gz: 9ba16ce6e90eb2f47c3f7478ef541bad6287c48fa6b968d54e7b9f59102c15bc
 SHA512:
-  metadata.gz: d805f12ef4d099d3080d1d2af38406ef031e55006d6a31aba70cd546b771f3c944d4e00ace219e7535aa8f641fa95ad20c1e529f1ea183cc52863dcd00202778
-  data.tar.gz: 3e20237ab9491e16ac14f07826a227e74b72d882c966630839bcec5c2cceb8bb3707bdf975fa538b8b3ec6cd8a21dc1ea47e90d9b3e5a9abb14ba4d498e233c6
+  metadata.gz: ea98e7ac865b6d023a77381c07592961be171b73a9349062b7e4b83758d89447062932642d45d0e337408d4f5888c8e231fda775e5ecee81cc83e7f10c8ef144
+  data.tar.gz: '0298aaf8bca95858b5735f407031b86c1a893bb6ca2d3275061b4ef48aaf5367ecad2359565fa937bf05be5a0352c2e77e7a77dae2a8f84ff5d89dabc1ac05e3'

data/README.md

@@ -104,9 +104,9 @@ Polytypo.transform(
 ```
 
 Without `frontmatter_keys:` nothing in the block changes, and a key you do not name never does. The
-block is read with the same scan `yaml` mode uses, so it refuses the same constructs — a
-single-quoted scalar containing `''` among them, which is how an apostrophe is written inside single
-quotes, so `title: 'It''s a test'` comes back untouched. The block is also processed separately from
+block is read with the same scan `yaml` mode uses, so it refuses the same constructs, and
+reads a doubled apostrophe in a single-quoted scalar as an opaque unit: in `title: 'It''s a test'`
+the `''` is left alone and the text on either side is processed. The block is also processed separately from
 the body, which shows up in exactly one place: an unbalanced quotation mark in `title` cannot pair
 with a mark in your first paragraph.
 

data/lib/polytypo/data/VERSION

@@ -1 +1 @@
-1.8.1
+1.10.0

data/lib/polytypo/data/fixtures/cs.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "cs",
   "cases": [
     {

data/lib/polytypo/data/fixtures/de-CH.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "de-CH",
   "cases": [
     {

data/lib/polytypo/data/fixtures/de-DE.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "de-DE",
   "cases": [
     {
@@ -776,6 +776,17 @@
       "in": "---\ntitle: Er sagte \"hallo\" laut\nsummary: Noch \"eins\" hier\nslug: \"er-sagte-hallo\"\n---\n\nIm Text \"so\" hier.\n",
       "out": "---\ntitle: Er sagte „hallo“ laut\nsummary: Noch „eins“ hier\nslug: \"er-sagte-hallo\"\n---\n\nIm Text „so“ hier.\n",
       "note": "modes.md §3.7.4: two listed plain scalars in a locale whose quote glyphs differ from English, with an unlisted slug between the block and the body. Each listed scalar is its own span and the marks pair inside it, so the German pair appears three times and the slug keeps its straight quotes."
+    },
+    {
+      "id": "de-de-yaml-single-quoted-escape-growth-declined",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "description"
+      ],
+      "in": "description: 'a''--''b'\n",
+      "out": "description: 'a''--''b'\n",
+      "note": "modes.md §3.8.6 and §3.4: de-DE is a `-spaced` locale, so the dash would grow against a unit and the edge-growth rule discards it, as in fr. A second locale so that an implementation that does not split cannot pass on one cell alone."
     }
   ]
 }

data/lib/polytypo/data/fixtures/el.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "el",
   "cases": [
     {

data/lib/polytypo/data/fixtures/en-GB.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "en-GB",
   "cases": [
     {

data/lib/polytypo/data/fixtures/en-US.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "en-US",
   "cases": [
     {
@@ -1974,16 +1974,16 @@
       "note": "modes.md §3.7.4 inherits every bail of §3.8.6: a double-quoted scalar whose content contains U+005C yields no spans, because the source spells the content with more characters than it has. The listed key is not enough on its own."
     },
     {
-      "id": "en-us-markdown-commonmark-frontmatter-keys-single-quote-bail",
-      "rule": "apostrophe",
+      "id": "en-us-markdown-commonmark-frontmatter-keys-single-quote-escape",
+      "rule": "ellipsis",
       "mode": "markdown",
       "dialect": "commonmark",
       "frontmatterKeys": [
         "title"
       ],
-      "in": "---\ntitle: 'It''s a test'\n---\n\nBody it's here.\n",
-      "out": "---\ntitle: 'It''s a test'\n---\n\nBody it’s here.\n",
-      "note": "modes.md §3.7.4, and the cost §7.13 measures: a single-quoted scalar containing two consecutive U+0027 yields no spans, and an apostrophe in a single-quoted scalar is written exactly that way. Half the prose values of the §3.7.4.1 corpus bail here when re-emitted in this style."
+      "in": "---\ntitle: 'It''s a test...'\n---\n\nBody it's here.\n",
+      "out": "---\ntitle: 'It''s a test…'\n---\n\nBody it’s here.\n",
+      "note": "modes.md §3.7.4 inherits §3.8.6's split of `''` (spec 1.9.0): the value converts on both sides of the unit and the apostrophe it spells stays straight, while the body's own apostrophe converts. Until 1.9.0 the whole value yielded no spans."
     },
     {
       "id": "en-us-markdown-commonmark-frontmatter-keys-separate-unit",
@@ -2545,15 +2545,70 @@
       "note": "modes.md §3.8.6: a single-quoted scalar's content is one span; the quotes themselves are outside it."
     },
     {
-      "id": "en-us-yaml-single-quoted-escape-untouched",
+      "id": "en-us-yaml-single-quoted-escape-split",
       "rule": "ellipsis",
       "mode": "yaml",
       "keys": [
         "description"
       ],
       "in": "description: 'it''s one...two'\n",
-      "out": "description: 'it''s one...two'\n",
-      "note": "modes.md §3.8.6: `''` makes the source spell the content with more characters than it has, so the offset model of §3.1 cannot hold and the scalar yields no spans."
+      "out": "description: 'it''s one…two'\n",
+      "note": "modes.md §3.8.6 (spec 1.9.0): `''` is one apostrophe of content spelled with two code points, so it is an opaque skipped unit and the scalar is split around it. Everything on either side converts; the apostrophe it spells is outside every span and stays U+0027. Until 1.9.0 the whole scalar yielded no spans."
+    },
+    {
+      "id": "en-us-yaml-single-quoted-escape-quotes",
+      "rule": "quotes",
+      "mode": "yaml",
+      "keys": [
+        "description"
+      ],
+      "in": "description: 'It''s -- he said \"hi\" to me...'\n",
+      "out": "description: 'It''s—he said “hi” to me…'\n",
+      "note": "modes.md §3.8.6: the same scalar with a double-quoted run after the unit. The run pairs and converts, the dash and ellipsis convert, and the apostrophe stays straight."
+    },
+    {
+      "id": "en-us-yaml-single-quoted-escape-at-ends",
+      "rule": "ellipsis",
+      "mode": "yaml",
+      "keys": [
+        "description"
+      ],
+      "in": "description: '''one...two'''\n",
+      "out": "description: '''one…two'''\n",
+      "note": "modes.md §3.8.6: a unit directly after the opening quote and directly before the closing one. The scan claims each pair as it is met, so `'''` is the unit followed by content and `'''` at the end is the unit followed by the closing quote."
+    },
+    {
+      "id": "en-us-yaml-single-quoted-escape-only",
+      "rule": "ellipsis",
+      "mode": "yaml",
+      "keys": [
+        "description"
+      ],
+      "in": "description: ''''\n",
+      "out": "description: ''''\n",
+      "note": "modes.md §3.8.6: a scalar that is nothing but one unit yields no non-empty segment, so no span, and comes back byte for byte."
+    },
+    {
+      "id": "en-us-yaml-single-quoted-escape-contraction-at-unit",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "description"
+      ],
+      "in": "description: 'a''--''b'\n",
+      "out": "description: 'a''—''b'\n",
+      "note": "modes.md §3.8.6 and §3.4: the dash sits between two units, so it is a whole span. A contraction (2 → 1) is applied at a span extremity, exactly as it is against `:` and `#` in a plain scalar."
+    },
+    {
+      "id": "en-us-yaml-single-quoted-escape-unterminated",
+      "rule": "ellipsis",
+      "mode": "yaml",
+      "keys": [
+        "description"
+      ],
+      "in": "description: 'it''s one...two\n",
+      "out": "description: 'it''s one...two\n",
+      "note": "modes.md §3.8.6: after a unit the scan keeps looking for a closing quote on the same line, finds none, and the value yields no spans — a trailing unit is not mistaken for a close."
     },
     {
       "id": "en-us-yaml-double-quoted-escape-untouched",
@@ -3090,6 +3145,136 @@
       "in": "---\rtitle: a \"b\"\r---\r\rBody \"q\".\r",
       "out": "---\rtitle: a \"b\"\r---\r\rBody “q”.\r",
       "note": "modes.md §3.7.3a steps 1 and 5 composed: a byte-order mark in front of a document whose lines end with lone U+000D. Each clause is pinned alone and their combination is where one runtime lost the body entirely — found by an adversarial pass over a finished port, not by any fixture then in the suite. The block is located and skipped, its content is declined for the U+000D, and the body converts."
+    },
+    {
+      "id": "en-us-yaml-lone-cr-plain-untouched",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "title",
+        "slug",
+        "summary"
+      ],
+      "in": "title: a\rb -- c\n",
+      "out": "title: a\rb -- c\n",
+      "note": "modes.md §3.8.4 step 9 (spec 1.10.0): a U+000D that is not a line's terminator would land inside the scalar's span, and line terminators are never inside a span. The line yields no spans and comes back byte for byte. Until 1.10.0 this converted, with the carriage return inside the span."
+    },
+    {
+      "id": "en-us-yaml-lone-cr-quoted-untouched",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "title",
+        "slug",
+        "summary"
+      ],
+      "in": "title: \"a\rb -- c\"\n",
+      "out": "title: \"a\rb -- c\"\n",
+      "note": "modes.md §3.8.4 step 9: the same line with a double-quoted scalar. The decline is on the line, not on the scalar form."
+    },
+    {
+      "id": "en-us-yaml-lone-cr-block-scalar-untouched",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "title",
+        "slug",
+        "summary"
+      ],
+      "in": "title: |\n  a -- b\n  c\rd -- e\n",
+      "out": "title: |\n  a -- b\n  c\rd -- e\n",
+      "note": "modes.md §3.8.5 (spec 1.10.0): a U+000D on any line of a block scalar's run makes the whole block yield no spans, as a tab does — the first line, which is clean, is not converted either."
+    },
+    {
+      "id": "en-us-yaml-lone-cr-value-run-consumed",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "title",
+        "slug",
+        "summary"
+      ],
+      "in": "summary: a\rb\n  title: c -- d\n",
+      "out": "summary: a\rb\n  title: c -- d\n",
+      "note": "modes.md §3.8.4 step 9: the declined line's value run is still consumed, so the indented `title: c -- d` stays a continuation of `summary` and is not read as a mapping line of its own. Declining without consuming would convert a key that is not a key."
+    },
+    {
+      "id": "en-us-yaml-trailing-cr-is-a-terminator",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "title",
+        "slug",
+        "summary"
+      ],
+      "in": "title: a -- b\r",
+      "out": "title: a—b\r",
+      "note": "modes.md §3.8.4: a U+000D that ends the input with no U+000A after it is a terminator, like one before a U+000A, so it is not a lone carriage return and the line converts."
+    },
+    {
+      "id": "en-us-yaml-crlf-still-converts",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "title",
+        "slug",
+        "summary"
+      ],
+      "in": "title: a -- b\r\nslug: c -- d\r\n",
+      "out": "title: a—b\r\nslug: c—d\r\n",
+      "note": "modes.md §3.8.4: CRLF is two terminator characters and neither is lone, so step 9 does not apply to a Windows-line-ended document."
+    },
+    {
+      "id": "en-us-yaml-cr-line-is-not-blank",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "title",
+        "slug",
+        "summary"
+      ],
+      "in": "title: |\n  a -- b\n\r  \n  c -- d\n",
+      "out": "title: |\n  a—b\n\r  \n  c -- d\n",
+      "note": "modes.md §3.8.4 step 1 (spec 1.10.0): a line is blank only if it is empty or all U+0020, so `<CR>␣␣` is not blank. It has indent 0, which ends the block's value run, and `a -- b` converts while `c -- d` is outside the block. A port that trims U+000D before testing blankness keeps the line in the run and declines the whole block instead."
+    },
+    {
+      "id": "en-us-yaml-double-trailing-cr-untouched",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "title",
+        "slug",
+        "summary"
+      ],
+      "in": "title: a -- b\r\r",
+      "out": "title: a -- b\r\r",
+      "note": "modes.md §3.8.4: only the last U+000D at the end of the input is a terminator, so the first is a lone one and the line yields no spans. A port that strips every trailing U+000D converts it."
+    },
+    {
+      "id": "en-us-yaml-cr-before-lf-after-cr-untouched",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "title",
+        "slug",
+        "summary"
+      ],
+      "in": "title: a -- b\r\r\n",
+      "out": "title: a -- b\r\r\n",
+      "note": "modes.md §3.8.4: a U+000D immediately before a U+000A is a terminator, and only that one; the U+000D before it is lone. The same line as the previous case with a final U+000A."
+    },
+    {
+      "id": "en-us-yaml-cr-then-hash-untouched",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "title",
+        "slug",
+        "summary"
+      ],
+      "in": "title: a\r#c -- d\n",
+      "out": "title: a\r#c -- d\n",
+      "note": "modes.md §3.8.4 step 9: YAML 1.2 reads `<CR>#c -- d` as a comment line, which step 2 declines when it starts a line. The LF-only model sees one line, so the decline has to be on the carriage return itself."
     }
   ]
 }

data/lib/polytypo/data/fixtures/es.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "es",
   "cases": [
     {

data/lib/polytypo/data/fixtures/fi.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "fi",
   "cases": [
     {

data/lib/polytypo/data/fixtures/fr-CA.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "fr-CA",
   "cases": [
     {

data/lib/polytypo/data/fixtures/fr.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "fr",
   "cases": [
     {
@@ -961,7 +961,7 @@
       "note": "modes.md §3.8.6 in a locale where the apostrophe is elision rather than possession: the same cell as en-us-yaml-plain-apostrophe, and §8 item 4's requirement that a construct be pinned in more than one locale rather than the same English output with the locale field changed."
     },
     {
-      "id": "fr-yaml-single-quoted-escape-untouched",
+      "id": "fr-yaml-single-quoted-escape-elision-stays",
       "rule": "apostrophe",
       "mode": "yaml",
       "keys": [
@@ -969,7 +969,29 @@
       ],
       "in": "description: 'l''idee et l''autre'\n",
       "out": "description: 'l''idee et l''autre'\n",
-      "note": "modes.md §3.8.6: two consecutive U+0027 in a single-quoted scalar yield no spans, in every locale — the source spells the content with more characters than it has, and that is a property of YAML, not of the language. This is the cell §7.13 measures at 1040 of 1858 values on a real corpus."
+      "note": "modes.md §3.8.6: in a locale where the apostrophe is elision, the unit is the elision, and it is outside every span. The scalar has nothing else to convert and comes back unchanged — the cost §7.13 measures, pinned so that it is a documented property and not a surprise."
+    },
+    {
+      "id": "fr-yaml-single-quoted-escape-split",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "description"
+      ],
+      "in": "description: 'l''idee -- et ''l''autre...'\n",
+      "out": "description: 'l''idee — et ''l''autre…'\n",
+      "note": "modes.md §3.8.6: a spaced dash and an ellipsis interior to a span convert, either side of units; the elisions stay straight."
+    },
+    {
+      "id": "fr-yaml-single-quoted-escape-growth-declined",
+      "rule": "dashes",
+      "mode": "yaml",
+      "keys": [
+        "description"
+      ],
+      "in": "description: 'a''--''b'\n",
+      "out": "description: 'a''--''b'\n",
+      "note": "modes.md §3.8.6 and §3.4: `fr` is a `-spaced` locale, so the dash would grow from 2 to 3 code points against a unit. The edge-growth rule discards it, as it does against `:` and `#`. The en-US case with the same input is applied, which is what makes this one discriminate."
     },
     {
       "id": "fr-yaml-single-quoted-no-escape",
@@ -1003,6 +1025,19 @@
       "in": "---\ntitle: \"a\rb\"\nslug: c \"d\"\n---\n\nBody \"q\" here.\n",
       "out": "---\ntitle: \"a\rb\"\nslug: c « d »\n---\n\nBody « q » here.\n",
       "note": "modes.md §3.7.4 per-line U+000D bail in a locale whose quotation marks carry their own spacing, so the declined line and the converted one are visibly different constructions rather than the same English output relabelled (§8 item 4)."
+    },
+    {
+      "id": "fr-yaml-lone-cr-costs-one-line",
+      "rule": "quotes",
+      "mode": "yaml",
+      "keys": [
+        "title",
+        "slug",
+        "summary"
+      ],
+      "in": "title: a \"b\"\nslug: c\rd \"e\"\n",
+      "out": "title: a « b »\nslug: c\rd \"e\"\n",
+      "note": "modes.md §3.8.4 step 9 in a locale whose quotation marks carry their own spacing: the line with the lone U+000D is declined and the line before it converts, so the cost is one line and not the document."
     }
   ]
 }

data/lib/polytypo/data/fixtures/it.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "it",
   "cases": [
     {

data/lib/polytypo/data/fixtures/locale-resolution.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "cases": [
     {
       "id": "exact-en-us",

data/lib/polytypo/data/fixtures/nl.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "nl",
   "cases": [
     {

data/lib/polytypo/data/fixtures/pl.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "pl",
   "cases": [
     {

data/lib/polytypo/data/fixtures/pt-BR.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "pt-BR",
   "cases": [
     {

data/lib/polytypo/data/fixtures/pt-PT.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "pt-PT",
   "cases": [
     {

data/lib/polytypo/data/fixtures/ru.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "ru",
   "cases": [
     {

data/lib/polytypo/data/fixtures/sv.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "sv",
   "cases": [
     {

data/lib/polytypo/data/fixtures/tr.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "tr",
   "cases": [
     {

data/lib/polytypo/data/fixtures/uk.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "locale": "uk",
   "cases": [
     {

data/lib/polytypo/data/locales/registry.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "$comment": "Locale resolution input. Algorithm is specified in spec/rules/locale-resolution.md and is identical in every runtime — never delegate it to a platform locale-negotiation library. \"spec\" here must track spec/VERSION exactly — it is not itself the global version source; scripts/validate-spec.mjs enforces the match.",
   "locales": [
     "en-US",

data/lib/polytypo/data/rules/modes.md

@@ -7,13 +7,15 @@ for all five runtimes and is parser-agnostic by construction: `parse5`, `nokogir
 `golang.org/x/net/html` and PHP's DOM disagree about almost everything this document does not
 forbid them from doing. `yaml` mode is parser-**free** rather than parser-agnostic, for the
 reason §3.8.1 measures.
-**Spec version:** 1.8.0 (0.1.0 for everything except §3.3's class-membership table rows for
+**Spec version:** 1.10.0 (0.1.0 for everything except §3.3's class-membership table rows for
 `nbsp` and `apostrophe`, split in 1.2.0, §3.8, added in 1.3.0, §3.3's note on `quotes`'
 span-boundary elision veto reading the marker as a trigger, added in 1.4.0 — which changes no
 row of the table it follows — §3.3's `CLOSEDELIM` entry, added in 1.5.0, and §3.7.4 with the
 amendments it carries to §3.1's definitions, §3.2's Model C, §3.5 step 3, §3.7.3's frontmatter
-bullet, §3.8.6's accepted-cost paragraph, §5, §6 and §7, added in 1.7.0, and §3.7.3a, added in
-1.8.0).
+bullet, §3.8.6's accepted-cost paragraph, §5, §6 and §7, added in 1.7.0, §3.7.3a, added in
+1.8.0, and §3.8.6's treatment of `''` in a single-quoted scalar, which replaces the bail it had
+until 1.9.0, with the matching amendments to §3.7.4.1, §3.8.7, §5, §7.13 and §8, and §3.8.4
+step 9 with §3.8.5's content-line condition, added in 1.10.0).
 
 ---
 
@@ -727,8 +729,11 @@ already recognises, and the option only changes what happens inside it:
   because the window includes it. So `title` / `slug` / a content-final U+000D converts `slug` in
   `yaml` mode and does not here. The wider decline is the direction this section takes everywhere
   else — a miss is invisible, a U+000D inside a span is not — and the cost is conversions lost in
-  documents written with line endings from the last century. Widening §3.8.4 instead would be a
-  change to `yaml` mode for every caller and wants its own measurement;
+  documents written with line endings from the last century. Since 1.10.0 `yaml` mode declines a
+  line containing a lone U+000D itself (§3.8.4 step 9), so the two paths agree on a stray U+000D
+  inside a line; they still part on the shape named just above. It does
+  so by declining rather than by widening §3.8.4's line model to YAML's or CommonMark's, which
+  would change what `yaml` mode converts for every caller;
 - **both delimiter lines stay outside every span**, as does every line terminator, so no edit
   can reach `---` itself and §3.7.3's setext-underline hazard is unreachable;
 - an **unterminated** block is not a block — §3.7.3 already yields no frontmatter construct
@@ -859,14 +864,13 @@ and it is the only way this corpus can show a conversion at all.
   name `"HTPBE?"` becomes `"HTPBE ?"`. The other two are improvements a caller might well want
   (`"Niamh O'Sullivan"` → `"Niamh O’Sullivan"`), which is the point: only the caller can tell
   those apart, and that is the same argument §3.8.2 makes.
-- **The inherited escape bail, measured.** Re-emitting each of the 247 convertible values in
-  one quoting style and running the scan over it: **single-quoted, 130 of 247 yield no spans**;
-  double-quoted, none do. The reason is §3.8.6's — an apostrophe inside a single-quoted scalar
-  is written `''`, which spells content with more characters than it has. Every listed scalar in
-  this corpus as written is double-quoted, so the bail never fires on it; a caller whose YAML
-  style is single quotes gets nothing on half of their prose, silently. That number is the
-  accepted cost of reusing §3.8's scan rather than a defect of this section, and §7.13 records it
-  where a reader will look for it.
+- **The inherited escape bail, measured — and removed in 1.9.0.** Re-emitting each of the 247
+  convertible values in one quoting style and running the scan over it gave **single-quoted, 130 of
+  247 yield no spans**; double-quoted, none. The reason was §3.8.6's bail on `''`: an apostrophe
+  inside a single-quoted scalar is written that way. Every listed scalar in this corpus as written
+  is double-quoted, so the bail never fired on it; a caller whose YAML style is single quotes got
+  nothing on half of their prose, silently. 1.9.0 replaced the bail with a split at each `''`
+  (§3.8.6), and §7.13 records what that did and did not recover.
 
 ### 3.8 Skip list — `yaml`
 
@@ -988,8 +992,10 @@ throw this mode adds is the option throw of §3.8.2, which is about the call, no
 
 The source is the code-point array of §3.1. A **line** is a maximal run containing no U+000A,
 **and a U+000D immediately before that U+000A is not part of the line** — it is a terminator like
-the U+000A itself, so it lies outside every span and comes back untouched. Line terminators are
-never inside a span, so no rule can create or destroy one. `indent` is the number of leading
+the U+000A itself, so it lies outside every span and comes back untouched. **The same holds for a
+U+000D that ends the input with no U+000A after it.** Line terminators are never inside a span,
+so no rule can create or destroy one — and since 1.10.0 that sentence is true of a lone U+000D as
+well, by step 9 below and §3.8.5, rather than only of the terminators this paragraph defines. `indent` is the number of leading
 U+0020 on the line and `i` the index of the first code point after them.
 
 The carriage-return clause is normative rather than obvious, and it is here because five runtimes
@@ -1006,7 +1012,10 @@ return **inside** its span and hands it to the rules as content.
 Lines are processed in order.
 
 1. The line yields no spans if it is empty or all U+0020, or if it contains U+0009 anywhere. A
-   tab makes indentation undecidable, which is what every step below depends on.
+   tab makes indentation undecidable, which is what every step below depends on. **A line is
+   blank only if it is empty or all U+0020**, here and wherever this section says blank (step 7,
+   §3.8.5): a line containing U+0009 or U+000D is never blank, and a port must not test blankness
+   with a trim that strips either.
 2. The line yields no spans if `cp[i]` is `#` (comment) or `%` (directive).
 3. The line yields no spans if it begins at `i` with `---` or with `...` and the next code point
    is U+0020 or the line ends there. The trailing-content form is declined as well as the bare
@@ -1041,7 +1050,14 @@ Lines are processed in order.
    and its value run yield no spans. This is the step that makes `run:`, `if:` and `image:`
    unreachable, and it is checked before the value's form so that an unlisted key costs nothing
    to decline.
-9. `cp[v]` selects the form:
+9. **If the line contains a U+000D, it yields no spans, and its value run (step 7) is still
+   consumed.** The U+000D here is one the line definition above did not take for a terminator — a
+   *lone* one, which YAML 1.2.2 §5.4 reads as a line break and this section's LF-only model does
+   not. It would otherwise land inside a span and be handed to the rules as content, which the
+   opening of this section forbids in terms. The value run is consumed rather than left to be
+   rescanned for the reason step 7 gives: a continuation line is not a mapping line, and
+   `summary: a<CR>b` followed by an indented `title: c` must not make `title` a key. Otherwise
+   `cp[v]` selects the form:
    - `&`, `*` or `!` — anchor, alias or tag — and `{` or `[` — a flow collection: no spans.
    - `#`: the value is absent and only a comment follows: no spans.
    - `|` or `>`: §3.8.5, block scalar.
@@ -1057,10 +1073,14 @@ spans. **The header is never inside a span.**
 
 The block's content is the value run of §3.8.4 step 7. Let `bi` be the indentation of its first
 non-blank line, or `indent + n` when the header carried an explicit indentation indicator `n`.
-**One definition, and three conditions that make the whole block yield no spans** — the block is
+**One definition, and four conditions that make the whole block yield no spans** — the block is
 consumed either way, never rescanned:
 
 - any line of the run contains U+0009;
+- any line of the run contains a U+000D (§3.8.4 step 9, 1.10.0) — a lone one, since a terminator's
+  is not part of its line. The whole block goes, as for a tab, rather than the one line: the
+  block's lines are one construct, and a carriage return in the middle of it is evidence that
+  the author's line model is not the one this section reads;
 - the run's first non-blank line is indented less than `bi`, which can only happen when an
   explicit indicator disagrees with the block as written;
 - any later non-blank line of the run is indented less than `bi`.
@@ -1091,16 +1111,43 @@ that line the value yields no spans, and its continuation lines are consumed by
 rather than rescanned. After the closing quote only U+0020 and an optional `#` comment may
 follow. The span is the content between the quotes, exclusive of both.
 
-Two bails make source characters and content characters the same thing, which §3.1's offset model
-requires:
+The offset model of §3.1 requires that source characters and content characters be the same thing,
+and two constructs break that:
 
 - a double-quoted scalar whose content contains U+005C yields no spans — `\n`, `\"` and `é`
   are content the source spells with more characters than it has;
-- a single-quoted scalar whose content contains two consecutive U+0027 yields no spans, for the
-  same reason.
-
-This is the treatment `html` gives a character reference (§3.6), reached from the same constraint
-rather than by analogy. **No colon test applies to a quoted scalar** — quoting neutralises the
+- in a single-quoted scalar, **two consecutive U+0027 are one apostrophe of content, spelled with
+  two code points.** Since 1.9.0 that pair is an **opaque skipped unit**, treated as §3.8.6 treats
+  `:` and `#` in a plain scalar: the scalar's range is split around it, and the unit lies outside
+  every span. Until 1.9.0 the whole scalar yielded no spans instead.
+
+Neither is a mapping from source to content. A backslash escape has dozens of forms and each would
+need its own decoding table in five runtimes; `''` is one fixed pair, and a skipped unit needs no
+table at all. That is the treatment `html` gives a character reference (§3.6), reached from the
+same constraint rather than by analogy.
+
+> **The single-quoted scan, normative.** Starting after the opening quote, advance `j` one code
+> point at a time. A U+0027 at `j` followed by another U+0027 is a unit: end the current segment
+> before `j`, begin the next at `j + 2`, and continue at `j + 2`. A U+0027 at `j` not followed by
+> another U+0027 is the closing quote. If the line ends first, the value yields no spans, as for
+> any unterminated quoted scalar. The segments, each non-empty one a span, are what remains once
+> the units are removed. The scan is left to right and each pair is claimed as it is met, so
+> `'''a'` is the unit followed by `a`, and `''''` is one unit and the closing quote.
+
+**A span may begin or end at a unit, and §3.4 decides what that allows — identically to `:` and
+`#`.** Nothing about `''` needs a rule of its own. A replacement that is longer than what it
+replaced is discarded at an extremity, so `fr`'s spaced dash does not grow against a unit
+(`'a''--''b'` is left alone in `fr` and `de-DE`), while a contraction is applied (`'a''--''b'` →
+`'a''—''b'` in `en-US`). The unit is a boundary marker (§3.2) between the two spans it separates:
+no rule edits a code point that lies outside every span, so the apostrophe it spells **stays
+U+0027**. That is the accepted cost of the split, and it is not small: in `'l''idee'` everything
+but the elision converts, and a value whose only convertible character was that apostrophe still
+comes back unchanged. §7.13 measures it.
+
+**Why not convert the apostrophe too.** Replacing `''` with U+2019 outright would convert it and
+the result still parses, since U+2019 is ordinary content in a single-quoted scalar. It is
+rejected for the reason §3.8.1 gives: that is decode-then-remap, the offset mapping is no longer
+the identity, and five runtimes would each invent it. **No colon test applies to a quoted scalar** — quoting neutralises the
 colon, and applying the plain-scalar test here is what made the first draft decline
 `title: "Chapter 1: the beginning"`.
 
@@ -1177,6 +1224,36 @@ clause reverted, the same run reports **480 structural changes** — `k: a:---a`
 measurement that passes both with and without the thing it is supposed to protect proves nothing,
 and two earlier versions of this sweep were in exactly that position.
 
+**The same sweep, extended for `''` (1.9.0).** The alphabet gained `''` as one token beside the
+lone U+0027, and this run used four single-quoted templates alone, because the apostrophe check below cannot hold on a plain scalar, where `apostrophe` legitimately converts `a'a`. They are the payload alone, followed by a
+comment, between fixed `x''` and `''y`, and as a value in a sequence of mappings. Payloads of
+length 1–4, 19 locales, the sources that are not valid YAML discarded before the run,
+**872 328 cases over those four templates, 65 033 of them changed by the transform: no parse failure, no structural change, no idempotency failure, and the number of U+0027 in the output
+equal to the number in the source in every case** — the last check is what a structure-and-type
+comparison cannot see, since a `''` turned into `’’` leaves the document's shape intact. It
+discriminates, but narrowly, and that should be said: with the unit's width off by one (the
+second U+0027 left inside the span) the same run reports failures — `k: '''a'` becoming
+`k: ''’a'`, in `cs` and others — whereas an implementation that does not split at all, and hands
+the pair to the rules as two ordinary characters, **also passes**, because `quotes`' same-kind
+adjacency veto leaves an adjacent pair alone. The split is therefore specified for the reason
+§3.8.1 gives, that it needs no mapping and puts the pair out of every rule's reach by construction,
+and not because the sweep shows the unsplit alternative to be unsafe at payload length four.
+
+**A targeted sweep for U+000D (1.10.0).** It is narrower than the sweeps above on purpose —
+its alphabet carries neither U+005C, `''` nor `---`, which §5 requires of the idempotency sweeps
+and which those sweeps still carry — because it checks one property, §3.8.4's, that they never
+tested: **no span contains a line terminator.** Over documents built from the
+alphabet `a`, U+0020, U+000D, U+000A, `:`, `#`, `'`, `"`, `-` and `|`, in seven templates — a
+plain, a double-quoted and a single-quoted scalar, a block scalar, an unterminated final line, a
+payload ahead of a key, and a payload on the continuation line of a CRLF value — at payload
+lengths 1–5, **777 770 documents**, the scan's spans were read out and tested directly. Before
+1.10.0, **95 505 of them (12.3%) had a span containing a U+000D**; `title: a<CR>b -- c` was
+converted with the carriage return inside the span, and `"aaaa<CR>"` was a quoted scalar whose
+span ended in one. After, **none**. This check is discriminating by construction, since it
+failed 95 505 times on the engine it was written against. It reads spans rather than output
+because no rule emits or deletes a U+000D, so the output never showed it — the guarantee was
+false and nothing downstream noticed.
+
 **Three things it could not find**, each worth naming because each cost something.
 
 - It did not find the workflow damage, because a shell script inside a block scalar is a string
@@ -1372,7 +1449,13 @@ tests none of this document. In `yaml` the sweep alphabet must include `:`, `#`,
 as one token**, since those are the characters whose adjacency the argument above turns on and a
 three-dash run is not reachable from single dashes at a bounded payload length; and it must
 include U+0022 and U+005C, without which the sweep cannot reach a quoted scalar's delimiters at
-all. Since 1.7.0 the `markdown` run must also carry a template with a **frontmatter block and
+all. Since 1.10.0 it must also include **U+000D**, and a `yaml` run must test the property §3.8.4
+states — that no span contains a line terminator — directly on the spans, because a rule that
+neither emits nor deletes a U+000D leaves the output unchanged whether or not one is inside a
+span. Since 1.9.0 it must also include **`''` as one token** next to a lone U+0027, and its
+single-quoted templates must carry the pair at the start, in the middle and at the end of the
+payload, because a unit's behaviour at a span extremity is a different path from its behaviour
+between two letters. Since 1.7.0 the `markdown` run must also carry a template with a **frontmatter block and
 `frontmatterKeys` naming a key in it** — a document with two text units (§3.1) tests a composition
 the single-unit templates cannot reach. §3.8.7 records what each run did and did not establish — including that a sweep comparing
 structure and types is blind to a string whose content is damaged, and that an alphabet without
@@ -1589,7 +1672,11 @@ rule-local.
     - **a flow collection** — `tags: [one, two]` and `{a: b}` — even on one line;
     - **a quoted scalar containing an escape** — `"say \"hi\""` — where the source spells the
       content with more characters than it has;
-    - **a block scalar whose indentation is ambiguous**, per §3.8.5's three bail conditions.
+    - **a block scalar whose indentation is ambiguous**, per §3.8.5's bail conditions;
+    - **a line carrying a lone U+000D** (1.10.0), and a block scalar with one on any line. Measured
+      on the author's 367-file content directory: no file contains one. It is the price of keeping §3.8.4's line model
+      LF-only while not letting a carriage return into a span, and it falls on documents
+      written with Classic Mac line endings.
 
 12. **`keys` matches a bare name at any depth, and nothing narrower.** `description` is
     processable wherever it occurs, which is what makes the option portable — no paths, no
@@ -1627,20 +1714,32 @@ rule-local.
       line, so that a stray one inside a single quoted value costs that value and not the block.
       The block is still skipped, so nothing machine-read is typeset either way. A lone-U+000D
       document is one line to §3.8.4 and therefore loses the whole block, which is the price of
-      not widening that section here.
-    - **§3.8.6's single-quoted bail costs more here than anywhere it has been measured before.**
-      Of the 1858 corpus values a locale would convert across eight locales, **1040 yield no spans
-      when written as a single-quoted scalar** — 130 of 247 in `en-GB` alone, the figure 1.7.0
-      shipped with, and `scripts/miss-census.mjs` in this repository is what reproduces either on
-      demand. An apostrophe inside such a scalar is spelled `''`, and an apostrophe
-      is exactly what `apostrophe` and `quotes` convert, so the bail falls hardest on the values
-      the option exists for. Double-quoted, none bail. The corpus as authored is entirely
-      double-quoted, so its own author never meets this; a caller whose YAML style is single
-      quotes gets nothing on half their prose, and gets it silently. The repair is not this
-      section's to make: §3.8.6 is ratified and measured, and the obvious extension — treating
-      `''` as an opaque two-code-point unit, exactly as §3.8.6 already treats `:` and `#` inside a
-      plain scalar — changes `yaml` mode for every caller and needs its own measurement and its
-      own sign-off.
+      not widening that section here. Since 1.10.0 `yaml` mode makes the same decline for itself
+      (§3.8.4 step 9), per line and consuming the value run, so the two paths no longer differ on
+      a stray U+000D inside a line. One shape of the same character is still missed in both, and
+      is recorded rather than fixed: a U+000D at column 0 of a line inside an *unlisted* key's
+      block scalar ends that value run under §3.8.4's LF-only model, so a listed-looking line
+      after it is scanned as a mapping line of its own, which YAML would read as block content.
+      Closing it needs the line model widened, which this release declines to do.
+    - **§3.8.6's treatment of `''` in a single-quoted scalar: the cost, before and after 1.9.0.**
+      Until 1.9.0 a single-quoted scalar containing `''` yielded no spans. An apostrophe inside
+      such a scalar is spelled `''`, and an apostrophe is exactly what `apostrophe` and `quotes`
+      convert, so the bail fell hardest on the values the option exists for. On the author's own
+      207-file corpus, de-typeset and re-emitted as single-quoted, in eight locales: **1104 of
+      1913 values a locale would convert came back unchanged**. Since 1.9.0 the scalar is split at
+      each `''` (§3.8.6) and **672 of 1913** do — 35.1% against 57.7%. The remainder is not one
+      cost but two. **602 of the 672 have nothing else to convert**: the apostrophe is the only
+      thing a locale would have changed, it lies outside every span by construction, and it stays
+      straight. **The other 70 also change when their `'` are already curly**, so something besides
+      an apostrophe is lost — in the samples read, a phrase written `'like this'`, which a locale
+      would pair as a quotation, and whose two marks are units here. Not all 70 were read.
+      Double-quoted and plain scalars are unaffected, and the corpus as authored is entirely
+      double-quoted, so its own author never met either figure.
+      The "before" figure is what `scripts/miss-census.mjs` prints with a 1.8.x engine
+      installed; the "after" figure is the same run with the 1.9.0 engine, and the 602/70 split
+      is a separate run of that comparison on values with every `'` already curly, which the
+      script does not perform. The corpus has grown since 1.7.0 measured it (187 files, 1040 of
+      1858), which is why the denominator differs from that figure.
 
 ## 8. Fixture coverage strategy (non-normative)
 
@@ -1677,7 +1776,7 @@ changes.
    blocks, and the JSX-vs-skipped-element case distinction (`tests/modes/markdown.test.ts`).
 3a. **YAML span selection** — the three scalar forms, the `keys` option (a listed key, an
    unlisted one, a quoted one, the same name at two depths), block-scalar indentation and
-   chomping, the `:`/`#` split of §3.8.6, step 7's consumption of an inline value's continuation
+   chomping, the `:`/`#` split of §3.8.6, the `''` split of §3.8.6 (since 1.9.0), a lone U+000D (since 1.10.0), step 7's consumption of an inline value's continuation
    lines, and every bail of §3.8.4. `yaml` carries a heavier fixture burden than the other two
    for a reason §8's opening argument does not cover: its span selection is **specified rather
    than delegated** (§3.8.1), so a fixture is the only thing standing between five hand-written

data/lib/polytypo/data/rules/order.json

@@ -1,5 +1,5 @@
 {
-  "spec": "1.8.1",
+  "spec": "1.10.0",
   "$comment": "Single source of truth for pipeline order. Rules run in ascending `order`. Disabling a rule removes it from the sequence and never reorders the rest. Rule ids are public API (see docs/ARCHITECTURE.md section 5). \"spec\" here must track spec/VERSION exactly — it is not itself the global version source; scripts/validate-spec.mjs enforces the match.",
   "rules": [
     {

data/lib/polytypo/modes/yaml.rb

@@ -75,6 +75,13 @@ module Polytypo
       end
       private_class_method :tab?
 
+      # A U+000D the line definition did not take for a terminator, i.e. one that is not part of
+      # a CRLF and does not end the input. It would otherwise land inside a span.
+      def self.lone_cr?(chars, line)
+        (line.start...line.end).any? { |i| chars[i] == CR }
+      end
+      private_class_method :lone_cr?
+
       # 3.8.4 step 3. "---" and "..." at the head of a line, bare or introducing a node: the
       # trailing-content form is declined too, so "--- key: value" never yields a key of
       # "--- key".
@@ -176,7 +183,10 @@ module Polytypo
         # costs nothing to decline: this is what makes "run:", "if:" and "image:" unreachable.
         return nxt unless keys.include?(chars[key_start...key_end].join)
 
-        # step 9 -- the scalar form.
+        # step 9 -- a lone U+000D declines the line, and its value run is still consumed.
+        return nxt if lone_cr?(chars, line)
+
+        # step 10 -- the scalar form.
         value = chars[v]
         return nxt if NON_SCALAR_VALUE_CHARS.include?(value)
 
@@ -217,17 +227,18 @@ module Polytypo
         h += 1 while h < line.end && chars[h] == SPACE
         return if h < line.end && chars[h] != "#"
 
-        # One definition of the run, and three conditions that make the whole block yield no
+        # One definition of the run, and four conditions that make the whole block yield no
         # spans.
         content = []
         content_indent = -1
         run_lines.each do |nxt|
           next if blank?(chars, nxt) # blank lines belong to the block and yield no span
 
-          # A tab makes indentation undecidable; an explicit indicator that disagrees with the
-          # block as written, or a later line dedented inside it, is ambiguous rather than
-          # guessable. All three make the WHOLE block yield no spans -- bail rather than choose.
-          break content_indent = -1 if tab?(chars, nxt)
+          # A tab makes indentation undecidable, and a lone U+000D would land inside a span; an
+          # explicit indicator that disagrees with the block as written, or a later line dedented
+          # inside it, is ambiguous rather than guessable. All of them make the WHOLE block yield
+          # no spans -- bail rather than choose.
+          break content_indent = -1 if tab?(chars, nxt) || lone_cr?(chars, nxt)
 
           next_indent = first_non_space(chars, nxt) - nxt.start
           if content_indent.negative?
@@ -246,21 +257,30 @@ module Polytypo
       end
       private_class_method :block_scalar
 
-      # 3.8.6. The span is the content between the quotes. Both bails exist so that source
-      # characters and content characters are the same thing, which the offset model of 3.1
-      # requires -- the same constraint that makes an HTML character reference an opaque unit in
-      # 3.6. No colon test applies here: quoting neutralises the colon, and applying the
-      # plain-scalar test would decline `title: "Chapter 1: the beginning"`.
+      # 3.8.6. The span is the content between the quotes. A backslash in a double-quoted scalar
+      # bails, so that source characters and content characters are the same thing, which the
+      # offset model of 3.1 requires. In a single-quoted scalar each doubled apostrophe is an
+      # opaque two-code-point unit that lies outside every span, the same treatment an HTML
+      # character reference gets in 3.6; the scalar's range is split around it. No colon test
+      # applies here: quoting neutralises the colon, and applying the plain-scalar test would
+      # decline `title: "Chapter 1: the beginning"`.
       def self.quoted_scalar(chars, line, v, spans)
         quote = chars[v]
+        starts = []
         close = -1
+        segment = v + 1
         j = v + 1
         while j < line.end
           c = chars[j]
           return if quote == '"' && c == "\\"
-          return if quote == "'" && c == "'" && j + 1 < line.end && chars[j + 1] == "'"
 
           if c == quote
+            if quote == "'" && j + 1 < line.end && chars[j + 1] == "'"
+              starts << [segment, j] if j > segment
+              segment = j + 2
+              j += 2
+              next
+            end
             close = j
             break
           end
@@ -272,7 +292,8 @@ module Polytypo
         after += 1 while after < line.end && chars[after] == SPACE
         return if after < line.end && chars[after] != "#"
 
-        spans << Spans::Span.new(v + 1, close) if close > v + 1
+        starts << [segment, close] if close > segment
+        starts.each { |from, to| spans << Spans::Span.new(from, to) }
       end
       private_class_method :quoted_scalar
 

data/lib/polytypo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polytypo
-  VERSION = "1.8.1"
+  VERSION = "1.10.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polytypo
 version: !ruby/object:Gem::Version
-  version: 1.8.1
+  version: 1.10.0
 platform: ruby
 authors:
 - Iurii Rogulia