polytypo 1.17.3 → 1.17.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/lib/polytypo/data/VERSION +1 -1
  3. data/lib/polytypo/data/fixtures/bg.json +1 -1
  4. data/lib/polytypo/data/fixtures/cs.json +1 -1
  5. data/lib/polytypo/data/fixtures/de-CH.json +1 -1
  6. data/lib/polytypo/data/fixtures/de-DE.json +1 -1
  7. data/lib/polytypo/data/fixtures/el.json +1 -1
  8. data/lib/polytypo/data/fixtures/en-GB.json +288 -1
  9. data/lib/polytypo/data/fixtures/en-US.json +82 -1
  10. data/lib/polytypo/data/fixtures/es.json +1 -1
  11. data/lib/polytypo/data/fixtures/fi.json +1 -1
  12. data/lib/polytypo/data/fixtures/fr-CA.json +1 -1
  13. data/lib/polytypo/data/fixtures/fr.json +1 -1
  14. data/lib/polytypo/data/fixtures/hr.json +1 -1
  15. data/lib/polytypo/data/fixtures/hu.json +1 -1
  16. data/lib/polytypo/data/fixtures/it.json +1 -1
  17. data/lib/polytypo/data/fixtures/locale-resolution.json +1 -1
  18. data/lib/polytypo/data/fixtures/lt.json +1 -1
  19. data/lib/polytypo/data/fixtures/nl.json +1 -1
  20. data/lib/polytypo/data/fixtures/pl.json +1 -1
  21. data/lib/polytypo/data/fixtures/pt-BR.json +1 -1
  22. data/lib/polytypo/data/fixtures/pt-PT.json +1 -1
  23. data/lib/polytypo/data/fixtures/ro.json +1 -1
  24. data/lib/polytypo/data/fixtures/ru.json +1 -1
  25. data/lib/polytypo/data/fixtures/sk.json +1 -1
  26. data/lib/polytypo/data/fixtures/sl.json +1 -1
  27. data/lib/polytypo/data/fixtures/sr.json +1 -1
  28. data/lib/polytypo/data/fixtures/sv.json +1 -1
  29. data/lib/polytypo/data/fixtures/tr.json +1 -1
  30. data/lib/polytypo/data/fixtures/uk.json +1 -1
  31. data/lib/polytypo/data/locales/registry.json +1 -1
  32. data/lib/polytypo/data/rules/modes.md +116 -6
  33. data/lib/polytypo/data/rules/order.json +1 -1
  34. data/lib/polytypo/modes/html.rb +0 -1
  35. data/lib/polytypo/modes/markdown.rb +67 -12
  36. data/lib/polytypo/modes/runner.rb +2 -3
  37. data/lib/polytypo/modes/spans.rb +110 -5
  38. data/lib/polytypo/version.rb +1 -1
  39. metadata +1 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: edc406d95c832de832c565083d70026c88c9c81fbe5155c028546d4dd62b0bc7
4
- data.tar.gz: 7a20dab89c59c83926fb46f2d1a068683b41c18a8e9c9366abf57b6e24bf3dd9
3
+ metadata.gz: 1ada78393268412e145c286f3a52ca2b1529637472b413e2ad96533763ca4feb
4
+ data.tar.gz: ca2eab4d74188ca6f57e71a573722e8b51454621e2d648fcd4df6417d4a2701b
5
5
  SHA512:
6
- metadata.gz: 3a80f23b401fac0852442eddee666e318fc81658c1eab39c0112536b9310051088232100c1d7ee1ee8483282c47c9841c8c0034ebfa80550f12fb4c662dd1f34
7
- data.tar.gz: 1eee25a1cff3f33f8a92b7fdd20c8def024658f840a4c5ffafeea6a85a08f2943c878f8101c614c29f5944ae28b80e712037d91cf8cf58261c505e2bbcfa7c6e
6
+ metadata.gz: 72da361c3230aa58f7c06cb595685d53e6a949858eae318ae207fc5819924c598d99246393afdc917c3abe1431a6cbe88d548133e758668ef3d23ad382bfc6f2
7
+ data.tar.gz: b7b2f9646e6f326010604a6097409d34e3c80b3ffabf4a28582946832c4705da343437bd538188363b32b97393889f6ed204d06de47cf315623e80765e61830e
@@ -1 +1 @@
1
- 1.17.3
1
+ 1.17.5
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "bg",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "cs",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "de-CH",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "de-DE",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "el",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "en-GB",
4
4
  "cases": [
5
5
  {
@@ -1495,6 +1495,230 @@
1495
1495
  "out": "see booking._ , emails.booking_ here, and a, b.\n",
1496
1496
  "note": "modes.md §3.4: the delimiter-neighbour rule holds in both dialects."
1497
1497
  },
1498
+ {
1499
+ "id": "en-gb-markdown-commonmark-backslash-space-deletion",
1500
+ "rule": "spaces",
1501
+ "mode": "markdown",
1502
+ "dialect": "commonmark",
1503
+ "in": "foo\\ , bar and a , b.\n",
1504
+ "out": "foo\\ , bar and a, b.\n",
1505
+ "note": "modes.md §3.4 escape-and-reference rule: deleting the space between a literal backslash and the comma would make the backslash escape the comma, and the text would render without the backslash; that edit is discarded. The same deletion beside ordinary text (a , b) is applied."
1506
+ },
1507
+ {
1508
+ "id": "en-gb-markdown-commonmark-reference-shape-space-deletion",
1509
+ "rule": "spaces",
1510
+ "mode": "markdown",
1511
+ "dialect": "commonmark",
1512
+ "in": "AT&amp ;T and &#82 ;x and a ; b.\n",
1513
+ "out": "AT&amp ;T and &#82 ;x and a; b.\n",
1514
+ "note": "modes.md §3.4 escape-and-reference rule: deleting the space before the semicolon would complete a character reference (& and R) that the input does not have, and the text would render as AT&T and R; those edits are discarded. A semicolon after ordinary text (a ; b) is still tightened."
1515
+ },
1516
+ {
1517
+ "id": "en-gb-markdown-commonmark-space-after-ampersand-deleted-elsewhere",
1518
+ "rule": "spaces",
1519
+ "mode": "markdown",
1520
+ "dialect": "commonmark",
1521
+ "in": "AT & T , x & y .\n",
1522
+ "out": "AT & T, x & y.\n",
1523
+ "note": "modes.md §3.4 escape-and-reference rule: an ampersand followed by a space is no reference shape, and an escaped or well-formed reference is outside every span; the spaces before the comma and the full stop are deleted as usual."
1524
+ },
1525
+ {
1526
+ "id": "en-gb-markdown-commonmark-hex-reference-shape-space-deletion",
1527
+ "rule": "spaces",
1528
+ "mode": "markdown",
1529
+ "dialect": "commonmark",
1530
+ "in": "&#x52 ;x and a ; b.\n",
1531
+ "out": "&#x52 ;x and a; b.\n",
1532
+ "note": "modes.md §3.4 escape-and-reference rule, item 2: the x of a hexadecimal reference counts as a letter, so deleting the space before the semicolon would complete a reference (R, which renders as R); that edit is discarded."
1533
+ },
1534
+ {
1535
+ "id": "en-gb-markdown-commonmark-parsed-reference-outside-span",
1536
+ "rule": "spaces",
1537
+ "mode": "markdown",
1538
+ "dialect": "commonmark",
1539
+ "in": "x & , y and a , b.\n",
1540
+ "out": "x & , y and a, b.\n",
1541
+ "note": "modes.md §3.7.3: a character reference the parser recognises lies outside every span, and a span edge is the end of the text for a rule that deletes, so the space after it is kept; the space in a , b is deleted."
1542
+ },
1543
+ {
1544
+ "id": "en-gb-markdown-commonmark-inline-html-reference-shape",
1545
+ "rule": "spaces",
1546
+ "mode": "markdown",
1547
+ "dialect": "commonmark",
1548
+ "in": "<span>AT&amp ;T</span> and a , b.\n",
1549
+ "out": "<span>AT&amp ;T</span> and a, b.\n",
1550
+ "note": "modes.md §3.4: the escape-and-reference rule applies to every span of a Markdown document, including the text of inline HTML."
1551
+ },
1552
+ {
1553
+ "id": "en-gb-markdown-commonmark-escaped-punctuation-space-kept",
1554
+ "rule": "spaces",
1555
+ "mode": "markdown",
1556
+ "dialect": "commonmark",
1557
+ "in": "x \\* , y and a , b.\n",
1558
+ "out": "x \\* , y and a, b.\n",
1559
+ "note": "modes.md §3.7.3: a backslash escape lies outside every span, so the space after \\* is at a span edge and is kept; the space in a , b is deleted."
1560
+ },
1561
+ {
1562
+ "id": "en-gb-markdown-commonmark-escaped-quote-not-converted",
1563
+ "rule": "quotes",
1564
+ "mode": "markdown",
1565
+ "dialect": "commonmark",
1566
+ "in": "\"x \\\" y\"\n",
1567
+ "out": "‘x \\\" y’\n",
1568
+ "note": "modes.md §3.7.3: the escaped quotation mark lies outside every span and stays as typed; the outer pair is converted around it."
1569
+ },
1570
+ {
1571
+ "id": "en-gb-markdown-commonmark-escaped-apostrophe-kept",
1572
+ "rule": "apostrophe",
1573
+ "mode": "markdown",
1574
+ "dialect": "commonmark",
1575
+ "in": "it\\'s fine\n",
1576
+ "out": "it\\'s fine\n",
1577
+ "note": "modes.md §3.7.3: an escaped apostrophe is outside every span and stays U+0027; converting it would leave a backslash before U+2019."
1578
+ },
1579
+ {
1580
+ "id": "en-gb-markdown-commonmark-escaped-hyphens-kept",
1581
+ "rule": "dashes",
1582
+ "mode": "markdown",
1583
+ "dialect": "commonmark",
1584
+ "in": "x\\--y\n",
1585
+ "out": "x\\--y\n",
1586
+ "note": "modes.md §3.7.3: the escaped hyphen is outside every span, so the two hyphens are not a dash."
1587
+ },
1588
+ {
1589
+ "id": "en-gb-markdown-commonmark-unrecognised-reference-space-deleted",
1590
+ "rule": "spaces",
1591
+ "mode": "markdown",
1592
+ "dialect": "commonmark",
1593
+ "in": "x &foo; , y.\n\nx &Amp; , y.\n\nx &#12345678; , y.\n\nx &#x1234567; , y.\n",
1594
+ "out": "x &foo;, y.\n\nx &Amp;, y.\n\nx &#12345678;, y.\n\nx &#x1234567;, y.\n",
1595
+ "note": "modes.md §3.7.3: a name outside the HTML5 list (case-sensitive) and a numeric reference with more than 7 decimal or 6 hexadecimal digits are no character reference, so they stay in their span and the space after them is deleted."
1596
+ },
1597
+ {
1598
+ "id": "en-gb-markdown-commonmark-recognised-reference-space-kept",
1599
+ "rule": "spaces",
1600
+ "mode": "markdown",
1601
+ "dialect": "commonmark",
1602
+ "in": "x &AMP; , y.\n\nx &#0; , y.\n\nx &#x110000; , y.\n\nx &Abreve; , y.\n",
1603
+ "out": "x &AMP; , y.\n\nx &#0; , y.\n\nx &#x110000; , y.\n\nx &Abreve; , y.\n",
1604
+ "note": "modes.md §3.7.3: &AMP; and &Abreve; are in the HTML5 list, and &#0; and &#x110000; are within the digit limits, so each is a reference the parser recognises; it lies outside every span and the space after it is kept."
1605
+ },
1606
+ {
1607
+ "id": "en-gb-markdown-mdx-backslash-space-deletion",
1608
+ "rule": "spaces",
1609
+ "mode": "markdown",
1610
+ "dialect": "mdx",
1611
+ "in": "foo\\ , bar and a , b.\n",
1612
+ "out": "foo\\ , bar and a, b.\n",
1613
+ "note": "modes.md §3.4 escape-and-reference rule: deleting the space between a literal backslash and the comma would make the backslash escape the comma, and the text would render without the backslash; that edit is discarded. The same deletion beside ordinary text (a , b) is applied."
1614
+ },
1615
+ {
1616
+ "id": "en-gb-markdown-mdx-reference-shape-space-deletion",
1617
+ "rule": "spaces",
1618
+ "mode": "markdown",
1619
+ "dialect": "mdx",
1620
+ "in": "AT&amp ;T and &#82 ;x and a ; b.\n",
1621
+ "out": "AT&amp ;T and &#82 ;x and a; b.\n",
1622
+ "note": "modes.md §3.4 escape-and-reference rule: deleting the space before the semicolon would complete a character reference (&amp; and &#82;) that the input does not have, and the text would render as AT&T and R; those edits are discarded. A semicolon after ordinary text (a ; b) is still tightened."
1623
+ },
1624
+ {
1625
+ "id": "en-gb-markdown-mdx-space-after-ampersand-deleted-elsewhere",
1626
+ "rule": "spaces",
1627
+ "mode": "markdown",
1628
+ "dialect": "mdx",
1629
+ "in": "AT & T , x &amp; y .\n",
1630
+ "out": "AT & T, x &amp; y.\n",
1631
+ "note": "modes.md §3.4 escape-and-reference rule: an ampersand followed by a space is no reference shape, and an escaped or well-formed reference is outside every span; the spaces before the comma and the full stop are deleted as usual."
1632
+ },
1633
+ {
1634
+ "id": "en-gb-markdown-mdx-hex-reference-shape-space-deletion",
1635
+ "rule": "spaces",
1636
+ "mode": "markdown",
1637
+ "dialect": "mdx",
1638
+ "in": "&#x52 ;x and a ; b.\n",
1639
+ "out": "&#x52 ;x and a; b.\n",
1640
+ "note": "modes.md §3.4 escape-and-reference rule, item 2: the x of a hexadecimal reference counts as a letter, so deleting the space before the semicolon would complete a reference (&#x52;, which renders as R); that edit is discarded."
1641
+ },
1642
+ {
1643
+ "id": "en-gb-markdown-mdx-parsed-reference-outside-span",
1644
+ "rule": "spaces",
1645
+ "mode": "markdown",
1646
+ "dialect": "mdx",
1647
+ "in": "x &amp; , y and a , b.\n",
1648
+ "out": "x &amp; , y and a, b.\n",
1649
+ "note": "modes.md §3.7.3: a character reference the parser recognises lies outside every span, and a span edge is the end of the text for a rule that deletes, so the space after it is kept; the space in a , b is deleted."
1650
+ },
1651
+ {
1652
+ "id": "en-gb-markdown-mdx-inline-html-reference-shape",
1653
+ "rule": "spaces",
1654
+ "mode": "markdown",
1655
+ "dialect": "mdx",
1656
+ "in": "<span>AT&amp ;T</span> and a , b.\n",
1657
+ "out": "<span>AT&amp ;T</span> and a, b.\n",
1658
+ "note": "modes.md §3.4: the escape-and-reference rule applies to every span of a Markdown document, including the text of inline HTML."
1659
+ },
1660
+ {
1661
+ "id": "en-gb-markdown-mdx-escaped-punctuation-space-kept",
1662
+ "rule": "spaces",
1663
+ "mode": "markdown",
1664
+ "dialect": "mdx",
1665
+ "in": "x \\* , y and a , b.\n",
1666
+ "out": "x \\* , y and a, b.\n",
1667
+ "note": "modes.md §3.7.3: a backslash escape lies outside every span, so the space after \\* is at a span edge and is kept; the space in a , b is deleted."
1668
+ },
1669
+ {
1670
+ "id": "en-gb-markdown-mdx-escaped-quote-not-converted",
1671
+ "rule": "quotes",
1672
+ "mode": "markdown",
1673
+ "dialect": "mdx",
1674
+ "in": "\"x \\\" y\"\n",
1675
+ "out": "‘x \\\" y’\n",
1676
+ "note": "modes.md §3.7.3: the escaped quotation mark lies outside every span and stays as typed; the outer pair is converted around it."
1677
+ },
1678
+ {
1679
+ "id": "en-gb-markdown-mdx-escaped-apostrophe-kept",
1680
+ "rule": "apostrophe",
1681
+ "mode": "markdown",
1682
+ "dialect": "mdx",
1683
+ "in": "it\\'s fine\n",
1684
+ "out": "it\\'s fine\n",
1685
+ "note": "modes.md §3.7.3: an escaped apostrophe is outside every span and stays U+0027; converting it would leave a backslash before U+2019."
1686
+ },
1687
+ {
1688
+ "id": "en-gb-markdown-mdx-escaped-hyphens-kept",
1689
+ "rule": "dashes",
1690
+ "mode": "markdown",
1691
+ "dialect": "mdx",
1692
+ "in": "x\\--y\n",
1693
+ "out": "x\\--y\n",
1694
+ "note": "modes.md §3.7.3: the escaped hyphen is outside every span, so the two hyphens are not a dash."
1695
+ },
1696
+ {
1697
+ "id": "en-gb-markdown-mdx-unrecognised-reference-space-deleted",
1698
+ "rule": "spaces",
1699
+ "mode": "markdown",
1700
+ "dialect": "mdx",
1701
+ "in": "x &foo; , y.\n\nx &Amp; , y.\n\nx &#12345678; , y.\n\nx &#x1234567; , y.\n",
1702
+ "out": "x &foo;, y.\n\nx &Amp;, y.\n\nx &#12345678;, y.\n\nx &#x1234567;, y.\n",
1703
+ "note": "modes.md §3.7.3: a name outside the HTML5 list (case-sensitive) and a numeric reference with more than 7 decimal or 6 hexadecimal digits are no character reference, so they stay in their span and the space after them is deleted."
1704
+ },
1705
+ {
1706
+ "id": "en-gb-markdown-mdx-recognised-reference-space-kept",
1707
+ "rule": "spaces",
1708
+ "mode": "markdown",
1709
+ "dialect": "mdx",
1710
+ "in": "x &AMP; , y.\n\nx &#0; , y.\n\nx &#x110000; , y.\n\nx &Abreve; , y.\n",
1711
+ "out": "x &AMP; , y.\n\nx &#0; , y.\n\nx &#x110000; , y.\n\nx &Abreve; , y.\n",
1712
+ "note": "modes.md §3.7.3: &AMP; and &Abreve; are in the HTML5 list, and &#0; and &#x110000; are within the digit limits, so each is a reference the parser recognises; it lies outside every span and the space after it is kept."
1713
+ },
1714
+ {
1715
+ "id": "en-gb-html-reference-shape-space-deletion",
1716
+ "rule": "spaces",
1717
+ "mode": "html",
1718
+ "in": "<p>a &hellip ;b and c ; d.</p>\n",
1719
+ "out": "<p>a &hellip ;b and c; d.</p>\n",
1720
+ "note": "modes.md §3.4 escape-and-reference rule, item 2: &hellip followed by a space is no well-formed reference, so it stays in its span; deleting the space before the semicolon would complete one and the text would render as an ellipsis; that edit is discarded. The same deletion after ordinary text (c ; d) is applied."
1721
+ },
1498
1722
  {
1499
1723
  "id": "en-gb-markdown-commonmark-autolink-literal-closing-quote",
1500
1724
  "rule": "quotes",
@@ -1647,6 +1871,69 @@
1647
1871
  "in": "Note[^a's] and it's.\n\n[^a's]: it's text\n",
1648
1872
  "out": "Note[^a's] and it’s.\n\n[^a's]: it’s text\n",
1649
1873
  "note": "modes.md §3.7.3: footnote labels are skipped in both dialects."
1874
+ },
1875
+ {
1876
+ "id": "en-gb-markdown-commonmark-bracket-task-marker-space-after-open",
1877
+ "rule": "spaces",
1878
+ "mode": "markdown",
1879
+ "dialect": "commonmark",
1880
+ "in": "- [ x] item and a , b.\n",
1881
+ "out": "- [ x] item and a, b.\n",
1882
+ "note": "modes.md §3.4 bracket-shape rule: deleting the space after the opening bracket would make a task-list marker for a parser with GFM extensions; that edit is discarded. The space in a , b is deleted as usual."
1883
+ },
1884
+ {
1885
+ "id": "en-gb-markdown-commonmark-bracket-task-marker-space-before-close",
1886
+ "rule": "spaces",
1887
+ "mode": "markdown",
1888
+ "dialect": "commonmark",
1889
+ "in": "- [x ] item and a , b.\n",
1890
+ "out": "- [x ] item and a, b.\n",
1891
+ "note": "modes.md §3.4 bracket-shape rule: the same marker from the other side: deleting the space before the closing bracket is discarded."
1892
+ },
1893
+ {
1894
+ "id": "en-gb-markdown-commonmark-bracket-task-marker-both-spaces",
1895
+ "rule": "spaces",
1896
+ "mode": "markdown",
1897
+ "dialect": "commonmark",
1898
+ "in": "- [ X ] item , y\n",
1899
+ "out": "- [ X ] item, y\n",
1900
+ "note": "modes.md §3.4 bracket-shape rule: the skip over spaces keeps both deletions from being evaluated as safe on their own; the two are discarded together, and the space before the comma is deleted."
1901
+ },
1902
+ {
1903
+ "id": "en-gb-markdown-commonmark-bracket-footnote-space-after-open",
1904
+ "rule": "spaces",
1905
+ "mode": "markdown",
1906
+ "dialect": "commonmark",
1907
+ "in": "a [ ^1] b , c\n\n[^1]: n\n",
1908
+ "out": "a [ ^1] b, c\n\n[^1]: n\n",
1909
+ "note": "modes.md §3.4 bracket-shape rule: deleting the space after the opening bracket would make a footnote reference; that edit is discarded."
1910
+ },
1911
+ {
1912
+ "id": "en-gb-markdown-commonmark-bracket-footnote-space-before-close",
1913
+ "rule": "spaces",
1914
+ "mode": "markdown",
1915
+ "dialect": "commonmark",
1916
+ "in": "a [^1 ] b , c\n\n[^1]: n\n",
1917
+ "out": "a [^1 ] b, c\n\n[^1]: n\n",
1918
+ "note": "modes.md §3.4 bracket-shape rule: deleting the space before the closing bracket would make a footnote reference; that edit is discarded."
1919
+ },
1920
+ {
1921
+ "id": "en-gb-markdown-commonmark-bracket-ordinary-text-still-tightened",
1922
+ "rule": "spaces",
1923
+ "mode": "markdown",
1924
+ "dialect": "commonmark",
1925
+ "in": "a [ word ] b and ( x ) c.\n",
1926
+ "out": "a [word] b and (x) c.\n",
1927
+ "note": "modes.md §3.4 bracket-shape rule: the rule tests a shape, not a bracket: a bracket around an ordinary word, and a parenthesis around a single letter, are tightened as before."
1928
+ },
1929
+ {
1930
+ "id": "en-gb-markdown-commonmark-bracket-task-marker-two-spaces",
1931
+ "rule": "spaces",
1932
+ "mode": "markdown",
1933
+ "dialect": "commonmark",
1934
+ "in": "- [ ] item and a , b.\n",
1935
+ "out": "- [ ] item and a, b.\n",
1936
+ "note": "modes.md §3.4 bracket-shape rule: collapsing two spaces between the brackets to one would make an unchecked task-list marker out of literal text; that edit is discarded."
1650
1937
  }
1651
1938
  ]
1652
1939
  }
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "en-US",
4
4
  "cases": [
5
5
  {
@@ -4258,6 +4258,24 @@
4258
4258
  "out": "x*© y* and a*±5 b*\n",
4259
4259
  "note": "modes.md §3.4 delimiter-neighbour rule, premise: a replacement of punctuation or a symbol by another symbol beside a delimiter keeps the delimiter's class under CommonMark 0.31, so (c) and +/- still convert there. A parser that classifies symbols differently would read these two lines differently, and this fixture is what makes it fail."
4260
4260
  },
4261
+ {
4262
+ "id": "en-us-markdown-commonmark-reference-shape-is-not-a-name-lookup",
4263
+ "rule": "spaces",
4264
+ "mode": "markdown",
4265
+ "dialect": "commonmark",
4266
+ "in": "AT&T ;x and a ; b.\n",
4267
+ "out": "AT&T ;x and a; b.\n",
4268
+ "note": "modes.md §3.4 escape-and-reference rule, accepted cost: the test is of shape, not of the names an HTML5 parser knows, so &T; is treated like &amp; and the space before the semicolon is kept although no reference would form. The same deletion after ordinary text is applied."
4269
+ },
4270
+ {
4271
+ "id": "en-us-markdown-mdx-reference-shape-is-not-a-name-lookup",
4272
+ "rule": "spaces",
4273
+ "mode": "markdown",
4274
+ "dialect": "mdx",
4275
+ "in": "AT&T ;x and a ; b.\n",
4276
+ "out": "AT&T ;x and a; b.\n",
4277
+ "note": "modes.md §3.4 escape-and-reference rule, accepted cost, in the MDX dialect: see the CommonMark case."
4278
+ },
4261
4279
  {
4262
4280
  "id": "en-us-markdown-commonmark-autolink-literal-terminator-collapse",
4263
4281
  "rule": "dashes",
@@ -4266,6 +4284,69 @@
4266
4284
  "in": "Read https://example.com/a -- fine and b -- c\n",
4267
4285
  "out": "Read https://example.com/a -- fine and b—c\n",
4268
4286
  "note": "modes.md §3.7.3 terminator guard: collapsing the spaced dash would delete the space that ends the URL, and the parser would read the dash and the next word as part of it; that edit is discarded. The same collapse away from a URL (b -- c) is applied."
4287
+ },
4288
+ {
4289
+ "id": "en-us-markdown-commonmark-bracket-task-marker-space-after-open",
4290
+ "rule": "spaces",
4291
+ "mode": "markdown",
4292
+ "dialect": "commonmark",
4293
+ "in": "- [ x] item and a , b.\n",
4294
+ "out": "- [ x] item and a, b.\n",
4295
+ "note": "modes.md §3.4 bracket-shape rule: deleting the space after the opening bracket would make a task-list marker for a parser with GFM extensions; that edit is discarded. The space in a , b is deleted as usual."
4296
+ },
4297
+ {
4298
+ "id": "en-us-markdown-commonmark-bracket-task-marker-space-before-close",
4299
+ "rule": "spaces",
4300
+ "mode": "markdown",
4301
+ "dialect": "commonmark",
4302
+ "in": "- [x ] item and a , b.\n",
4303
+ "out": "- [x ] item and a, b.\n",
4304
+ "note": "modes.md §3.4 bracket-shape rule: the same marker from the other side: deleting the space before the closing bracket is discarded."
4305
+ },
4306
+ {
4307
+ "id": "en-us-markdown-commonmark-bracket-task-marker-both-spaces",
4308
+ "rule": "spaces",
4309
+ "mode": "markdown",
4310
+ "dialect": "commonmark",
4311
+ "in": "- [ X ] item , y\n",
4312
+ "out": "- [ X ] item, y\n",
4313
+ "note": "modes.md §3.4 bracket-shape rule: the skip over spaces keeps both deletions from being evaluated as safe on their own; the two are discarded together, and the space before the comma is deleted."
4314
+ },
4315
+ {
4316
+ "id": "en-us-markdown-commonmark-bracket-footnote-space-after-open",
4317
+ "rule": "spaces",
4318
+ "mode": "markdown",
4319
+ "dialect": "commonmark",
4320
+ "in": "a [ ^1] b , c\n\n[^1]: n\n",
4321
+ "out": "a [ ^1] b, c\n\n[^1]: n\n",
4322
+ "note": "modes.md §3.4 bracket-shape rule: deleting the space after the opening bracket would make a footnote reference; that edit is discarded."
4323
+ },
4324
+ {
4325
+ "id": "en-us-markdown-commonmark-bracket-footnote-space-before-close",
4326
+ "rule": "spaces",
4327
+ "mode": "markdown",
4328
+ "dialect": "commonmark",
4329
+ "in": "a [^1 ] b , c\n\n[^1]: n\n",
4330
+ "out": "a [^1 ] b, c\n\n[^1]: n\n",
4331
+ "note": "modes.md §3.4 bracket-shape rule: deleting the space before the closing bracket would make a footnote reference; that edit is discarded."
4332
+ },
4333
+ {
4334
+ "id": "en-us-markdown-commonmark-bracket-ordinary-text-still-tightened",
4335
+ "rule": "spaces",
4336
+ "mode": "markdown",
4337
+ "dialect": "commonmark",
4338
+ "in": "a [ word ] b and ( x ) c.\n",
4339
+ "out": "a [word] b and (x) c.\n",
4340
+ "note": "modes.md §3.4 bracket-shape rule: the rule tests a shape, not a bracket: a bracket around an ordinary word, and a parenthesis around a single letter, are tightened as before."
4341
+ },
4342
+ {
4343
+ "id": "en-us-markdown-commonmark-bracket-task-marker-two-spaces",
4344
+ "rule": "spaces",
4345
+ "mode": "markdown",
4346
+ "dialect": "commonmark",
4347
+ "in": "- [ ] item and a , b.\n",
4348
+ "out": "- [ ] item and a, b.\n",
4349
+ "note": "modes.md §3.4 bracket-shape rule: collapsing two spaces between the brackets to one would make an unchecked task-list marker out of literal text; that edit is discarded."
4269
4350
  }
4270
4351
  ]
4271
4352
  }
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "es",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "fi",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "fr-CA",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "fr",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "hr",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "hu",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "it",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "cases": [
4
4
  {
5
5
  "id": "exact-en-us",
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "lt",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "nl",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "pl",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "pt-BR",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "pt-PT",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "ro",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "ru",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "sk",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "sl",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "sr",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "sv",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "tr",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "locale": "uk",
4
4
  "cases": [
5
5
  {
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "$comment": "Locale resolution input. Algorithm is specified in spec/rules/locale-resolution.md and is identical in every runtime — never delegate it to a platform locale-negotiation library. \"spec\" here must track spec/VERSION exactly — it is not itself the global version source; scripts/validate-spec.mjs enforces the match.",
4
4
  "locales": [
5
5
  "en-US",
@@ -19,7 +19,7 @@ step 9 with §3.8.5's content-line condition, added in 1.10.0, §3.8.4's carriag
19
19
  continuation line with the bails of §3.8.6 that use it, added in 1.11.0, and §3.8.4's node lines,
20
20
  declined keys, node markers and own-line region, with the changes to steps 1, 3, 4, 5, 6, 7 and 8
21
21
  that go with them, added in 1.11.0, and §3.9 with the amendments it carries to §3.1, §3.5 step 2,
22
- §4, §5, §7 item 7 and §8, added in 1.13.0, and §3.4's delimiter-neighbour rule with §3.7.3's autolink-literal tail, terminator guard and reference labels, and the amendments they carry to §3.4's deletion paragraph, §5 items 2 and 3, its fifth-part note and testing obligation, and §6, added in 1.17.2).
22
+ §4, §5, §7 item 7 and §8, added in 1.13.0, and §3.4's delimiter-neighbour rule with §3.7.3's autolink-literal tail, terminator guard and reference labels, and the amendments they carry to §3.4's deletion paragraph, §5 items 2 and 3, its fifth-part note and testing obligation, and §6, added in 1.17.2, and §3.4's escape-and-reference rule with the amendments it carries to §3.6, §3.7.3, §5 item 2's fifth-part note, §5 item 3 and the testing obligation, added in 1.17.4, and §3.4's bracket-shape rule, added in 1.17.5).
23
23
 
24
24
  ---
25
25
 
@@ -412,6 +412,96 @@ the rule tests whitespace alone. A parser that implements an older definition (p
412
412
  would classify them differently, and two fixtures (`en-us-markdown-commonmark-delimiter-neighbour-symbols-kept`)
413
413
  make such a port fail rather than drift. No rule emits a member of `DELIM`.
414
414
 
415
+ **The escape-and-reference rule (`markdown` and `html`, spec 1.17.4).** The delimiter-neighbour rule
416
+ guards a change in how a parser _classifies_ a token. A second class changes the _text_ a parser
417
+ produces while the parse tree keeps its shape, which a comparison of shapes cannot see. A U+005C
418
+ that stands in a span as literal text has no escape in it, because the character after it is a
419
+ space; a `&` followed by letters and a space is no character reference (§3.6 keeps a `&` that
420
+ begins no well-formed reference inside its span). Deleting the space joins the two to what comes
421
+ next:
422
+
423
+ | Mode | Input | Locale | Output before 1.17.4 | What changed |
424
+ | ---------- | ------------ | ------- | -------------------- | ----------------------------------------------------------------- |
425
+ | `markdown` | `foo\ , bar` | `en-GB` | `foo\, bar` | the backslash now escapes the comma; it renders as `foo, bar` |
426
+ | `markdown` | `AT&amp ;T` | `en-GB` | `AT&amp;T` | a character reference now exists; it renders as `AT&T` |
427
+ | `html` | `<p>a &hellip ;b</p>` | `en-GB` | `<p>a &hellip;b</p>` | a character reference now exists; it renders as `a …b` |
428
+
429
+ A backslash escape or a well-formed reference that the parser already read lies outside every span
430
+ (§3.6, §3.7.3), so only a **literal** U+005C or `&` inside a span can be affected, and no parity
431
+ of backslashes needs counting. The rule applies to every span of a document, those that come from
432
+ inline HTML or an HTML block in a Markdown document included. With `ASCIIPUNCT` = U+0021–U+002F, U+003A–U+0040, U+005B–U+0060 and
433
+ U+007B–U+007E, and `p`, `q`, `w`, `r`, `old` and `new` as in the delimiter-neighbour rule:
434
+
435
+ > **The edit is discarded if either of these holds, evaluated on the array the rule read. Item 1
436
+ > applies to `markdown` only, since a backslash is literal in HTML; item 2 applies to `markdown`
437
+ > and `html`.**
438
+ >
439
+ > 1. `cp[p−1]` is U+005C, `old = cp[p]`, `new = w[0]` when `r > 0` and otherwise `cp[q+1]`, and
440
+ > exactly one of `old` and `new` is in `ASCIIPUNCT`.
441
+ > 2. The edit is a deletion (`r = 0`) and, with `a = cp[p−1]` and `b = cp[q+1]` made adjacent, the
442
+ > array then holds a **reference shape** that contains both: U+0026, an optional U+0023, one or
443
+ > more ASCII letters or digits, and U+003B (`a` may be the U+0026 and `b` the U+0023). A shape is read as a left-to-right scan over those
444
+ > classes; it asks whether the _name_ is a known one no more than the delimiter-neighbour rule
445
+ > asks whether a flank is a valid one.
446
+
447
+ A marker belongs to no class, a position outside the array is none, and the test reads no other
448
+ code point and no parser state, so it is a function of the array alone and fires identically on
449
+ every run (§5 item 3). It applies to `markdown` and `html` and to no other mode: `text` has neither
450
+ escapes nor references, and `yaml` selects spans by a scan that reads none (§3.8).
451
+
452
+ **Its premise is that no rule emits an ASCII letter, a digit, U+0023, U+0026, U+003B or
453
+ U+005C** (`ellipsis` emits U+002E, which is none of these), so item 2 has nothing to read in a
454
+ replacement and tests deletions only, and, outside raw HTML, a literal U+005C is never followed by ASCII
455
+ punctuation in the input, since that pair would be an escape. Inside raw HTML a backslash is literal
456
+ and item 1 still applies to it: that is **accepted** conservatism (a quotation mark after a backslash
457
+ in an HTML block is left as typed), deterministic and identical in every runtime. `spaces` is the only rule that deletes. The rule is deliberately a test of _shape_, not a
458
+ lookup of the reference names an HTML5 parser knows, because a name table would be a data file
459
+ in five runtimes; it is therefore stricter than a parser: `AT&T ;x` (en-US) keeps its space,
460
+ though `&T;` is no reference. **That is the accepted cost**, together with a literal backslash
461
+ before a space and punctuation, which is rare in prose.
462
+
463
+ **The bracket-shape rule (`markdown` only, spec 1.17.5).** Three more shapes are made by joining
464
+ what a deleted space kept apart, and both dialects, which enable GFM (§3.7), read the joined form
465
+ as structure where the spaced form is literal text: a task-list marker, `[x]`, `[X]` or `[ ]` at the
466
+ start of a list item, and a footnote reference or definition, `[^label]`. Measured, with the
467
+ pre-1.17.5 behaviour:
468
+
469
+ | Mode | Input | Locale | Output before 1.17.5 | What changed |
470
+ | ---------- | ------------- | ------- | -------------------- | --------------------------------------------------------------- |
471
+ | `markdown` | `- [ x] item` | `en-GB` | `- [x] item` | the text of a literal bracket becomes a checked task-list item |
472
+ | `markdown` | `- [x ] item` | `en-GB` | `- [x] item` | the same item, from the other side |
473
+ | `markdown` | `- [ ] item` | `en-GB` | `- [ ] item` | two spaces become one, and the text becomes an unchecked item |
474
+ | `markdown` | `a [ ^1] b` | `en-GB` | `a [^1] b` | a footnote reference, if a definition `[^1]: …` exists |
475
+
476
+ Write `SKIP(i, +1)` for the smallest index greater than `i` whose code point is not U+0020, and
477
+ `SKIP(i, −1)` for the largest index smaller than `i` whose code point is not U+0020; a marker, or a
478
+ position outside the array, belongs to no class below. With `p`, `q` and `r` as before, and
479
+ `a = SKIP(p, −1)` and `b = SKIP(q, +1)` (so that `a` and `b` are the nearest code points that are
480
+ not U+0020 on either side of the deleted run):
481
+
482
+ > **A deletion (`r = 0`) is discarded if either of these holds, evaluated on the array the rule read.**
483
+ >
484
+ > 1. `cp[a]` is U+005B, and either `cp[b]` is U+005E or U+005D, or `cp[b]` is U+0078 or U+0058 and
485
+ > `cp[SKIP(b, +1)]` is U+005D.
486
+ > 2. `cp[b]` is U+005D, and either `cp[a]` is U+0078 or U+0058 and `cp[SKIP(a, −1)]` is U+005B, or
487
+ > the **label run** is at least two code points long, its leftmost code point is U+005E, and
488
+ > the nearest code point left of the run that is not U+0020 is U+005B. The label run is the
489
+ > longest run ending at `a` of code points that are none of U+0020, U+005B, U+005D, and not a
490
+ > marker.
491
+ >
492
+ > **An edit whose replacement is the single code point U+0020 is discarded** when `cp[a]` is U+005B
493
+ > and `cp[b]` is U+005D: `spaces` collapses a run of spaces to one rather than deleting it, and
494
+ > `[ ]` becoming `[ ]` is the unchecked marker.
495
+
496
+ The skip over U+0020 is what keeps `[ x ]` whole: the two deletions are evaluated on the same
497
+ array, and each sees the other space as absent. The rule reads no parser state and no dialect, so it
498
+ fires identically on every run (§5 item 3), and it tests deletions only because `spaces` is the
499
+ only rule that deletes. It is a test of _shape_, not of whether a footnote definition or a list item
500
+ exists. **The accepted costs:** `a [ x] b` in the middle of a sentence keeps its space although no
501
+ list item is there, as do `[ x ]`, `[ ^1]` and `[ ]` in prose; `a [ ^ ] b` loses its second space only,
502
+ because a label run needs two code points; and a footnote label that contains an inline element
503
+ (`[^*a*b ]`) is missed, because a marker ends the label run.
504
+
415
505
  ### 3.5 Applying the result
416
506
 
417
507
  1. Extract the span sequence from the source. Record each span's **source offsets**.
@@ -444,7 +534,8 @@ Skipped, exhaustively and exactly:
444
534
  unit. A text node containing one is split into spans around it, so the reference's spelling is
445
535
  preserved exactly, which is the only way `&nbsp;` does not become a literal U+00A0 and back.
446
536
  **A bare `&` that begins no well-formed reference stays inside its span.** No rule emits or
447
- deletes `&`, so it is in no danger; lifting it out would create a boundary that suppresses
537
+ deletes `&`, so it is in no danger itself (a space deleted beside it can complete a reference,
538
+ which §3.4's escape-and-reference rule, 1.17.4, closes); lifting it out would create a boundary that suppresses
448
539
  conversions on both sides of it for nothing — `Tom & Jerry's "book"` would lose the pairing of
449
540
  its quotation marks. Well-formedness, not the presence of an ampersand, is what makes a
450
541
  skipped region;
@@ -547,6 +638,17 @@ Skipped, exhaustively:
547
638
  - fenced code blocks, including the info string and the fences;
548
639
  - indented code blocks — **`commonmark` only**; MDX has none;
549
640
  - inline code spans, including the backticks;
641
+ - **every backslash escape and every character reference the parser recognises (spec 1.17.4)**, as
642
+ an opaque unit, exactly as `html` treats a well-formed reference (§3.6): the span is split
643
+ around it and the unit lies outside every span. A port that took spans from text-node positions
644
+ would put `\,` and `&amp;` inside a span and disagree with the others about every deletion
645
+ beside them (`x &amp; , y` keeps its space). **"Recognises" is CommonMark 0.31's definition
646
+ (§2.5, with GFM), and it differs from §3.6's:** `&` followed by a name from the HTML5 named
647
+ character reference list, **case-sensitively** (`&AMP;` and `&Abreve;` are references; `&Amp;`
648
+ and `&foo;` are not), or by `#` and 1 to 7 decimal digits, or by `#x` or `#X` and 1 to 6
649
+ hexadecimal digits, then `;`. A value out of Unicode's range (`&#0;`, `&#x110000;`) is still a
650
+ reference. `html` mode tests the _shape_ of a reference (§3.6), not its name, so `&foo;` is
651
+ opaque there and a span's content here;
550
652
  - autolinks `<https://…>` — **`commonmark` only**;
551
653
  - **link and image destinations and titles** — the `(…)` of `[text](url "title")`, and the
552
654
  definition line of a reference link. The **link text** is processable;
@@ -1769,7 +1871,9 @@ they do not carry it over for free either. Write `M` for the whole mode transfor
1769
1871
  same reason. **This list is not claimed to be complete.** A deletion
1770
1872
  can also create a backslash escape or a character reference from literal text (`foo\ ,`
1771
1873
  becomes `foo\,` and `AT&amp ;T` becomes `AT&amp;T`): the parse tree keeps its shape and its
1772
- text changes, which a shape comparison does not see. That class is not closed by 1.17.2.
1874
+ text changes, which a shape comparison does not see. That class was not closed by 1.17.2 and
1875
+ is closed by §3.4's escape-and-reference rule (spec 1.17.4). A deletion can also join
1876
+ `[` and `x` or `^` into a shape a GFM parser reads as structure, and §3.4's bracket-shape rule (spec 1.17.5) closes that.
1773
1877
  - **Deletion** is only ever of a U+0020, and structural whitespace borders a `BREAK` — a real
1774
1878
  one in `text` mode, a −2 marker in the others — where `spaces` refuses to act
1775
1879
  (`spaces.md` §3.2 step 4). A deletion therefore cannot turn `- item` into `-item`, collapse
@@ -1841,8 +1945,9 @@ they do not carry it over for free either. Write `M` for the whole mode transfor
1841
1945
  that would place code points at a span extremity which were not there before — an insertion at
1842
1946
  a boundary, or a replacement longer than what it replaced — is discarded rather than assigned
1843
1947
  to a side. The discard is a pure function of the array and its span table (since 1.17.2: of
1844
- `(p, q, r, s₀, s₁)`, of the code points beside the edit for §3.4's delimiter-neighbour rule, and
1845
- of whether the span begins at the end of a web address for §3.7.3's terminator guard), so it
1948
+ `(p, q, r, s₀, s₁)`, of the code points beside the edit for §3.4's delimiter-neighbour rule, of
1949
+ the same code points and the run of letters and digits around the join for §3.4's
1950
+ escape-and-reference rule (since 1.17.4), of the code points around the join for §3.4's bracket-shape rule (since 1.17.5), and of whether the span begins at the end of a web address for §3.7.3's terminator guard), so it
1846
1951
  happens identically on every run: a rule that was declined at an edge once is declined there always, and the output
1847
1952
  contains nothing for the next run to reconsider.
1848
1953
 
@@ -1879,7 +1984,12 @@ quotes are curly, because each reaches a different rule. The tree to compare is
1879
1984
  runtime has another parser, that parser's as well), not only the tokenizer's own: the tokenizer reads
1880
1985
  no link in `"www.example.com"`, mdast does, and that is the class §3.7.3's scan closes. Templates
1881
1986
  must include an address with `_`, `*` and `~` inside it after `"`, `'` and `(`, and an address
1882
- directly after a digit. **Two classes are carved out because they are accepted, not because
1987
+ directly after a digit. **Since 1.17.4 the comparison also covers the text values of the tree**, not only its shape: the
1988
+ defect behind the escape-and-reference rule left the shape unchanged. The alphabet must then include
1989
+ U+005C, `&`, `#`, `;`, `,` and a letter, and the templates a literal backslash and a literal
1990
+ `&name` each followed by a space and by each punctuation mark. **Since 1.17.5 the alphabet also includes** U+005B, U+005D, U+005E,
1991
+ `x` and `X`, and the templates a list item and a footnote definition each holding every
1992
+ combination of those with zero, one or two spaces inside the brackets. **Two classes are carved out because they are accepted, not because
1883
1993
  they are untested**: a text that matched no definition in the input and matches one in the output
1884
1994
  (§3.7.3's accepted case), and an e-mail address, whose end only the parser decides.
1885
1995
 
@@ -1,5 +1,5 @@
1
1
  {
2
- "spec": "1.17.3",
2
+ "spec": "1.17.5",
3
3
  "$comment": "Single source of truth for pipeline order. Rules run in ascending `order`. Disabling a rule removes it from the sequence and never reorders the rest. Rule ids are public API (see docs/ARCHITECTURE.md section 5). \"spec\" here must track spec/VERSION exactly — it is not itself the global version source; scripts/validate-spec.mjs enforces the match.",
4
4
  "rules": [
5
5
  {
@@ -91,7 +91,6 @@ module Polytypo
91
91
  end
92
92
  refs
93
93
  end
94
- private_class_method :find_entity_refs
95
94
 
96
95
  # Parses "<name ...>", "</name>" or "<name .../>" (as an array of chars) into
97
96
  # [name, closing, self_closing]. Comments, declarations and processing instructions
@@ -45,7 +45,7 @@ module Polytypo
45
45
  # breaks (no prose content). Frontmatter is NOT in this list and is not a node type here:
46
46
  # comrak's front_matter_delimiter extension is never enabled, because since spec 1.8.0 the
47
47
  # block's extent is modes.md 3.7.3a's scan and not a parser's opinion of it.
48
- SKIP_WHOLE_TYPES = %i[code code_block thematic_break].freeze
48
+ SKIP_WHOLE_TYPES = %i[code code_block thematic_break escaped].freeze
49
49
 
50
50
  # A leaf whose Segment/source_position directly delineates literal, processable prose --
51
51
  # comrak's AST is segment-based like goldmark's: structural bytes (emphasis delimiters,
@@ -287,7 +287,10 @@ module Polytypo
287
287
  parser_source = normalize_lone_cr(mask_frontmatter(source, block))
288
288
 
289
289
  spans = []
290
- state = { stack: [], frontmatter_end: block.nil? ? 0 : block.end, labels: [], url_ends: Set.new }
290
+ state = {
291
+ stack: [], frontmatter_end: block.nil? ? 0 : block.end, labels: [], references: [],
292
+ url_ends: Set.new
293
+ }
291
294
  ParseError.wrap do
292
295
  doc = Commonmarker.parse(
293
296
  parser_source,
@@ -295,7 +298,8 @@ module Polytypo
295
298
  )
296
299
  walk(doc, chars, line_starts, spans, state)
297
300
  end
298
- trim_url_tails(chars, Spans.normalize_spans(remove_ranges(spans, state[:labels])), state[:url_ends])
301
+ kept = remove_ranges(spans, state[:labels] + state[:references])
302
+ trim_url_tails(chars, Spans.normalize_spans(kept), state[:url_ends])
299
303
  end
300
304
 
301
305
  URL_PREFIXES = ["www.", "http://", "https://"].freeze
@@ -352,21 +356,41 @@ module Polytypo
352
356
  def self.remove_ranges(spans, ranges)
353
357
  return spans if ranges.empty?
354
358
 
359
+ merged = merge_ranges(ranges)
360
+ starts = merged.map(&:first)
355
361
  spans.flat_map do |span|
356
- ranges.reduce([span]) do |pieces, r|
357
- pieces.flat_map do |q|
358
- next [q] if r.end <= q.start || r.start >= q.end
359
-
360
- res = []
361
- res << Spans::Span.new(q.start, r.start) if q.start < r.start
362
- res << Spans::Span.new(r.end, q.end) if r.end < q.end
363
- res
362
+ # The first merged range that ends after the span starts is the one before the first
363
+ # start beyond it, or that one's successor; later ranges are visited in order.
364
+ index = [(starts.bsearch_index { |s| s > span.start } || starts.length) - 1, 0].max
365
+ cursor = span.start
366
+ pieces = []
367
+ while index < merged.length && merged[index].first < span.end
368
+ from, to = merged[index]
369
+ if to > cursor
370
+ pieces << Spans::Span.new(cursor, from) if from > cursor
371
+ cursor = to
364
372
  end
373
+ index += 1
365
374
  end
375
+ next [span] if cursor == span.start && pieces.empty?
376
+
377
+ pieces << Spans::Span.new(cursor, span.end) if cursor < span.end
378
+ pieces
366
379
  end
367
380
  end
368
381
  private_class_method :remove_ranges
369
382
 
383
+ def self.merge_ranges(ranges)
384
+ ranges.map { |r| [r.start, r.end] }.sort.each_with_object([]) do |(from, to), out|
385
+ if out.any? && from <= out[-1][1]
386
+ out[-1][1] = to if to > out[-1][1]
387
+ else
388
+ out << [from, to]
389
+ end
390
+ end
391
+ end
392
+ private_class_method :merge_ranges
393
+
370
394
  # modes.md 3.7.3 (spec 1.17.2): the text of a shortcut or collapsed reference is its label.
371
395
  # A node exists only when the parser formed it from a definition, so the kind is read off the
372
396
  # source after the closing bracket of the text: nothing (shortcut), "[]" (collapsed); a
@@ -426,6 +450,34 @@ module Polytypo
426
450
  end
427
451
  private_class_method :emit_span
428
452
 
453
+ # modes.md 3.7.3 (spec 1.17.4): a character reference the parser recognises is an opaque unit
454
+ # outside every span, as in html (3.6). comrak folds a reference into its text node, so the
455
+ # candidates are found by shape in the node's source and each is kept only when the parser
456
+ # does recognise it: a numeric one within CommonMark's digit limits, a named one when parsing
457
+ # it alone yields something other than its own spelling.
458
+ def self.note_references(chars, start_offset, end_offset, state)
459
+ slice = chars[start_offset...end_offset]
460
+ Html.find_entity_refs(slice).each do |from, to|
461
+ text = slice[from...to].join
462
+ next unless recognised_reference?(text)
463
+
464
+ state[:references] << Spans::Span.new(start_offset + from, start_offset + to)
465
+ end
466
+ end
467
+ private_class_method :note_references
468
+
469
+ def self.recognised_reference?(text)
470
+ case text
471
+ when /\A&#[xX](\h{1,6});\z/o, /\A&#(\d{1,7});\z/o
472
+ true
473
+ when /\A&#/
474
+ false
475
+ else
476
+ Commonmarker.parse(text).first_child.first_child.string_content != text
477
+ end
478
+ end
479
+ private_class_method :recognised_reference?
480
+
429
481
  def self.walk(node, chars, line_starts, spans, state)
430
482
  type = node.type
431
483
  return if SKIP_WHOLE_TYPES.include?(type)
@@ -466,7 +518,10 @@ module Polytypo
466
518
  sp = node.source_position
467
519
  start_offset = offset_of(line_starts, sp[:start_line], sp[:start_column])
468
520
  end_offset = offset_of(line_starts, sp[:end_line], sp[:end_column]) + 1
469
- emit_span(spans, start_offset, end_offset, state) if state[:stack].empty?
521
+ if state[:stack].empty?
522
+ emit_span(spans, start_offset, end_offset, state)
523
+ note_references(chars, start_offset, end_offset, state)
524
+ end
470
525
  return
471
526
  end
472
527
 
@@ -15,13 +15,12 @@ module Polytypo
15
15
  # modes.md 3.4 interposed.
16
16
  def self.run_rules_over_spans(cp, plan, locale_data, ctx, guarded = [])
17
17
  current = cp
18
- markdown = ctx.mode == "markdown"
19
18
  plan.each do |rule_id|
20
19
  fn = Engine::Registry.rule(rule_id)
21
20
  edits = fn.call(current, locale_data, ctx)
22
21
  filtered = Spans.filter_boundary_edits(
23
22
  current, edits, Spans.span_ranges_of(current),
24
- delimiter_neighbours: markdown, guarded: guarded
23
+ mode: ctx.mode, guarded: guarded
25
24
  )
26
25
  next if filtered.empty?
27
26
 
@@ -104,7 +103,7 @@ module Polytypo
104
103
  source_cp.length
105
104
  ) do |current, edits|
106
105
  Spans.filter_boundary_edits(current, edits, Spans.span_ranges_of(current),
107
- delimiter_neighbours: ctx.mode == "markdown", guarded: guarded)
106
+ mode: ctx.mode, guarded: guarded)
108
107
  end
109
108
  end
110
109
  end
@@ -138,6 +138,107 @@ module Polytypo
138
138
  end
139
139
  private_class_method :changes_delimiter_flank?
140
140
 
141
+ def self.ascii_punctuation?(c)
142
+ return false if c.nil?
143
+
144
+ (c >= 0x21 && c <= 0x2F) || (c >= 0x3A && c <= 0x40) || (c >= 0x5B && c <= 0x60) ||
145
+ (c >= 0x7B && c <= 0x7E)
146
+ end
147
+ private_class_method :ascii_punctuation?
148
+
149
+ def self.ascii_alphanumeric?(c)
150
+ return false if c.nil?
151
+
152
+ (c >= 0x30 && c <= 0x39) || (c >= 0x41 && c <= 0x5A) || (c >= 0x61 && c <= 0x7A)
153
+ end
154
+ private_class_method :ascii_alphanumeric?
155
+
156
+ # modes.md 3.4 escape-and-reference rule, item 1 (markdown only, spec 1.17.4): an edit is
157
+ # discarded when it follows a backslash and changes whether the first code point after the
158
+ # backslash is ASCII punctuation, which would create or destroy a backslash escape.
159
+ def self.creates_escape?(cp, p, q, replacement)
160
+ return false if (p - 1).negative? || cp[p - 1] != 0x5C
161
+
162
+ after = replacement.empty? ? cp[q + 1] : replacement[0]
163
+ ascii_punctuation?(cp[p]) != ascii_punctuation?(after)
164
+ end
165
+ private_class_method :creates_escape?
166
+
167
+ # modes.md 3.4 escape-and-reference rule, item 2 (markdown and html, spec 1.17.4): a deletion
168
+ # that makes "&", an optional "#", one or more ASCII letters or digits and ";" adjacent
169
+ # around the join. A marker is in no class and stops either scan.
170
+ def self.creates_reference?(cp, p, q, replacement)
171
+ return false if !replacement.empty? || (p - 1).negative? || q + 1 >= cp.length
172
+
173
+ left = p - 1
174
+ alnum = 0
175
+ while left >= 0 && ascii_alphanumeric?(cp[left])
176
+ left -= 1
177
+ alnum += 1
178
+ end
179
+ right = q + 1
180
+ right += 1 if left >= 0 && cp[left] == 0x26 && cp[right] == 0x23 && alnum.zero?
181
+ while right < cp.length && ascii_alphanumeric?(cp[right])
182
+ right += 1
183
+ alnum += 1
184
+ end
185
+ return false if cp[right] != 0x3B || alnum.zero?
186
+
187
+ left -= 1 if left >= 0 && cp[left] == 0x23
188
+ left >= 0 && cp[left] == 0x26
189
+ end
190
+ private_class_method :creates_reference?
191
+
192
+ # modes.md 3.4 bracket-shape rule (markdown only, spec 1.17.5): a deletion that joins a
193
+ # task-list marker or a footnote label. A marker or a position outside the array is in no
194
+ # class. Ruby indexes negatively, so every read goes through +code_point_at+.
195
+ def self.code_point_at(cp, i)
196
+ i.negative? ? nil : cp[i]
197
+ end
198
+ private_class_method :code_point_at
199
+
200
+ def self.skip_spaces(cp, i, step)
201
+ k = i
202
+ k += step while k >= 0 && k < cp.length && cp[k] == SPACE
203
+ k
204
+ end
205
+ private_class_method :skip_spaces
206
+
207
+ def self.x_letter?(c)
208
+ c == 0x78 || c == 0x58
209
+ end
210
+ private_class_method :x_letter?
211
+
212
+ def self.label_character?(c)
213
+ !c.nil? && c >= 0 && c != SPACE && c != 0x5B && c != 0x5D
214
+ end
215
+ private_class_method :label_character?
216
+
217
+ def self.creates_bracket_shape?(cp, p, q, replacement)
218
+ is_collapse = replacement.length == 1 && replacement[0] == SPACE
219
+ return false unless replacement.empty? || is_collapse
220
+
221
+ a = skip_spaces(cp, p - 1, -1)
222
+ b = skip_spaces(cp, q + 1, 1)
223
+ at_a = code_point_at(cp, a)
224
+ at_b = code_point_at(cp, b)
225
+ return at_a == 0x5B && at_b == 0x5D if is_collapse
226
+ return true if at_a == 0x5B && at_b == 0x5D
227
+ if at_a == 0x5B
228
+ return true if at_b == 0x5E
229
+
230
+ return x_letter?(at_b) && code_point_at(cp, skip_spaces(cp, b + 1, 1)) == 0x5D
231
+ end
232
+ return false if at_b != 0x5D
233
+ return true if x_letter?(at_a) && code_point_at(cp, skip_spaces(cp, a - 1, -1)) == 0x5B
234
+
235
+ run = a
236
+ run -= 1 while run >= 0 && label_character?(cp[run])
237
+ first = run + 1
238
+ a - first >= 1 && cp[first] == 0x5E && code_point_at(cp, skip_spaces(cp, run, -1)) == 0x5B
239
+ end
240
+ private_class_method :creates_bracket_shape?
241
+
141
242
  # modes.md 3.4, two safety nets, both pure functions of (p, q, r, s0, s1):
142
243
  #
143
244
  # 1. No edit may contain a marker -- one that does is a bug, discarded rather than
@@ -151,10 +252,10 @@ module Polytypo
151
252
  # point any rule emits whose meaning comes from its position rather than from itself
152
253
  # (modes.md 5 item 2).
153
254
  #
154
- # +delimiter_neighbours+ (markdown only) enables the delimiter-neighbour rule of modes.md 3.4,
155
- # and +guarded+ holds, per span, whether its first code point is the terminator of a bare web
156
- # address (modes.md 3.7.3); both spec 1.17.2.
157
- def self.filter_boundary_edits(cp, edits, ranges, delimiter_neighbours: false, guarded: [])
255
+ # +mode+ selects the neighbour rules of modes.md 3.4 -- delimiter-neighbour, the escape rule
256
+ # and the bracket-shape rule for markdown, the reference rule for markdown and html -- and +guarded+ holds, per span,
257
+ # whether its first code point is the terminator of a bare web address (modes.md 3.7.3).
258
+ def self.filter_boundary_edits(cp, edits, ranges, mode: nil, guarded: [])
158
259
  edits.select do |edit|
159
260
  next false if (edit.start...edit.end).any? { |i| Engine.marker?(cp[i]) }
160
261
 
@@ -162,7 +263,11 @@ module Polytypo
162
263
  q = edit.end - 1
163
264
  d = edit.end - edit.start
164
265
  r = edit.replacement.length
165
- next false if delimiter_neighbours && changes_delimiter_flank?(cp, p, q, edit.replacement)
266
+ markdown = mode == "markdown"
267
+ next false if markdown && changes_delimiter_flank?(cp, p, q, edit.replacement)
268
+ next false if markdown && creates_escape?(cp, p, q, edit.replacement)
269
+ next false if markdown && creates_bracket_shape?(cp, p, q, edit.replacement)
270
+ next false if (markdown || mode == "html") && creates_reference?(cp, p, q, edit.replacement)
166
271
 
167
272
  span = span_containing(ranges, p)
168
273
  if span && p == span.first && guarded[ranges.index(span)] == true &&
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Polytypo
4
- VERSION = "1.17.3"
4
+ VERSION = "1.17.5"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: polytypo
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.17.3
4
+ version: 1.17.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Iurii Rogulia