polytypo 1.17.1 → 1.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/polytypo/data/VERSION +1 -1
- data/lib/polytypo/data/fixtures/bg.json +1 -1
- data/lib/polytypo/data/fixtures/cs.json +1 -1
- data/lib/polytypo/data/fixtures/de-CH.json +1 -1
- data/lib/polytypo/data/fixtures/de-DE.json +1 -1
- data/lib/polytypo/data/fixtures/el.json +1 -1
- data/lib/polytypo/data/fixtures/en-GB.json +172 -1
- data/lib/polytypo/data/fixtures/en-US.json +28 -1
- data/lib/polytypo/data/fixtures/es.json +1 -1
- data/lib/polytypo/data/fixtures/fi.json +1 -1
- data/lib/polytypo/data/fixtures/fr-CA.json +1 -1
- data/lib/polytypo/data/fixtures/fr.json +37 -1
- data/lib/polytypo/data/fixtures/hr.json +1 -1
- data/lib/polytypo/data/fixtures/hu.json +1 -1
- data/lib/polytypo/data/fixtures/it.json +1 -1
- data/lib/polytypo/data/fixtures/locale-resolution.json +1 -1
- data/lib/polytypo/data/fixtures/lt.json +1 -1
- data/lib/polytypo/data/fixtures/nl.json +1 -1
- data/lib/polytypo/data/fixtures/pl.json +1 -1
- data/lib/polytypo/data/fixtures/pt-BR.json +1 -1
- data/lib/polytypo/data/fixtures/pt-PT.json +1 -1
- data/lib/polytypo/data/fixtures/ro.json +1 -1
- data/lib/polytypo/data/fixtures/ru.json +1 -1
- data/lib/polytypo/data/fixtures/sk.json +1 -1
- data/lib/polytypo/data/fixtures/sl.json +1 -1
- data/lib/polytypo/data/fixtures/sr.json +1 -1
- data/lib/polytypo/data/fixtures/sv.json +1 -1
- data/lib/polytypo/data/fixtures/tr.json +1 -1
- data/lib/polytypo/data/fixtures/uk.json +1 -1
- data/lib/polytypo/data/locales/registry.json +1 -1
- data/lib/polytypo/data/rules/modes.md +124 -9
- data/lib/polytypo/data/rules/order.json +1 -1
- data/lib/polytypo/modes/markdown.rb +109 -5
- data/lib/polytypo/modes/runner.rb +13 -4
- data/lib/polytypo/modes/spans.rb +45 -3
- data/lib/polytypo/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8f0403eeaab340a17f36228069025bf2c7c848227e19b636801f066f9d3e7f7d
|
|
4
|
+
data.tar.gz: f73c193143caa9fd284d8b766f4a1812d4ab8db768e36cf93289073922b707db
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c0fbb5f7bc230e5dccf03bd23f60b670816ab00f5ebca90fc0461f95bdbc8496e092ec65d886cc47f88231421af8c2affe76a4b490cda8ccd451cf3d7caff3c5
|
|
7
|
+
data.tar.gz: ab7b930e384d2d14777d2b44566987a86bbf1cc236e3457315e42c4a80658c54b1c8855fc682b569be43e40804d08082b9afea3c415942dee40f7009958268ad
|
data/lib/polytypo/data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
1.17.
|
|
1
|
+
1.17.2
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"spec": "1.17.
|
|
2
|
+
"spec": "1.17.2",
|
|
3
3
|
"locale": "en-GB",
|
|
4
4
|
"cases": [
|
|
5
5
|
{
|
|
@@ -1476,6 +1476,177 @@
|
|
|
1476
1476
|
}
|
|
1477
1477
|
],
|
|
1478
1478
|
"note": "modes.md §3.9.2: touching ranges are one protected run and give one marker."
|
|
1479
|
+
},
|
|
1480
|
+
{
|
|
1481
|
+
"id": "en-gb-markdown-commonmark-delimiter-space-deletion",
|
|
1482
|
+
"rule": "spaces",
|
|
1483
|
+
"mode": "markdown",
|
|
1484
|
+
"dialect": "commonmark",
|
|
1485
|
+
"in": "see booking._ , emails.booking_ here, and a , b.\n",
|
|
1486
|
+
"out": "see booking._ , emails.booking_ here, and a, b.\n",
|
|
1487
|
+
"note": "modes.md §3.4 delimiter-neighbour rule: deleting the space before the comma would turn the two underscores into emphasis delimiters that the input does not have; that edit is discarded. The same deletion beside ordinary text (a , b) is applied."
|
|
1488
|
+
},
|
|
1489
|
+
{
|
|
1490
|
+
"id": "en-gb-markdown-mdx-delimiter-space-deletion",
|
|
1491
|
+
"rule": "spaces",
|
|
1492
|
+
"mode": "markdown",
|
|
1493
|
+
"dialect": "mdx",
|
|
1494
|
+
"in": "see booking._ , emails.booking_ here, and a , b.\n",
|
|
1495
|
+
"out": "see booking._ , emails.booking_ here, and a, b.\n",
|
|
1496
|
+
"note": "modes.md §3.4: the delimiter-neighbour rule holds in both dialects."
|
|
1497
|
+
},
|
|
1498
|
+
{
|
|
1499
|
+
"id": "en-gb-markdown-commonmark-autolink-literal-closing-quote",
|
|
1500
|
+
"rule": "quotes",
|
|
1501
|
+
"mode": "markdown",
|
|
1502
|
+
"dialect": "commonmark",
|
|
1503
|
+
"in": "Visit \"https://example.com/a\" now, and \"this\" converts.\n",
|
|
1504
|
+
"out": "Visit \"https://example.com/a\" now, and ‘this’ converts.\n",
|
|
1505
|
+
"note": "modes.md §3.7.3 autolink-literal tail: the closing mark sits inside the tail, which is skipped, so the pair is left as typed rather than half-converted. A converted closing mark is not ASCII punctuation and would become part of the link destination."
|
|
1506
|
+
},
|
|
1507
|
+
{
|
|
1508
|
+
"id": "en-gb-markdown-mdx-autolink-literal-closing-quote",
|
|
1509
|
+
"rule": "quotes",
|
|
1510
|
+
"mode": "markdown",
|
|
1511
|
+
"dialect": "mdx",
|
|
1512
|
+
"in": "Visit \"https://example.com/a\" now, and \"this\" converts.\n",
|
|
1513
|
+
"out": "Visit \"https://example.com/a\" now, and ‘this’ converts.\n",
|
|
1514
|
+
"note": "modes.md §3.7.3: the autolink-literal tail is skipped in both dialects."
|
|
1515
|
+
},
|
|
1516
|
+
{
|
|
1517
|
+
"id": "en-gb-markdown-commonmark-autolink-literal-single-quotes",
|
|
1518
|
+
"rule": "quotes",
|
|
1519
|
+
"mode": "markdown",
|
|
1520
|
+
"dialect": "commonmark",
|
|
1521
|
+
"in": "See 'https://example.com/a' now and 'this' too.\n",
|
|
1522
|
+
"out": "See 'https://example.com/a' now and ‘this’ too.\n",
|
|
1523
|
+
"note": "modes.md §3.7.3 autolink-literal tail: the same for a single-quoted URL; the pair is left as typed and the pair around an ordinary word converts."
|
|
1524
|
+
},
|
|
1525
|
+
{
|
|
1526
|
+
"id": "en-gb-markdown-commonmark-autolink-literal-ellipsis",
|
|
1527
|
+
"rule": "ellipsis",
|
|
1528
|
+
"mode": "markdown",
|
|
1529
|
+
"dialect": "commonmark",
|
|
1530
|
+
"in": "Read https://example.com/a... then wait...\n",
|
|
1531
|
+
"out": "Read https://example.com/a... then wait…\n",
|
|
1532
|
+
"note": "modes.md §3.7.3 autolink-literal tail: dots directly after a bare URL belong to the tail and stay; the dots after a word convert."
|
|
1533
|
+
},
|
|
1534
|
+
{
|
|
1535
|
+
"id": "en-gb-markdown-commonmark-autolink-literal-after-punctuation",
|
|
1536
|
+
"rule": "quotes",
|
|
1537
|
+
"mode": "markdown",
|
|
1538
|
+
"dialect": "commonmark",
|
|
1539
|
+
"in": "He said \"www.example.com/a\" today and \"this\" too.\n",
|
|
1540
|
+
"out": "He said \"www.example.com/a\" today and ‘this’ too.\n",
|
|
1541
|
+
"note": "modes.md §3.7.3 autolink-literal tail: a www. address is found in the source wherever it starts, not only where a parser recognises a link, because parsers disagree about what may precede it. The pair around it is left as typed."
|
|
1542
|
+
},
|
|
1543
|
+
{
|
|
1544
|
+
"id": "en-gb-markdown-commonmark-autolink-literal-delimiter-inside",
|
|
1545
|
+
"rule": "quotes",
|
|
1546
|
+
"mode": "markdown",
|
|
1547
|
+
"dialect": "commonmark",
|
|
1548
|
+
"in": "He said \"www.example.com/a_b\" today and \"this\" too.\n",
|
|
1549
|
+
"out": "He said \"www.example.com/a_b\" today and ‘this’ too.\n",
|
|
1550
|
+
"note": "modes.md §3.7.3 autolink-literal tail: the scan runs over the merged span, so an underscore inside the address, which the parser reports as a separate token, does not hide the tail from it."
|
|
1551
|
+
},
|
|
1552
|
+
{
|
|
1553
|
+
"id": "en-gb-markdown-mdx-autolink-literal-delimiter-inside",
|
|
1554
|
+
"rule": "quotes",
|
|
1555
|
+
"mode": "markdown",
|
|
1556
|
+
"dialect": "mdx",
|
|
1557
|
+
"in": "He said \"www.example.com/a_b\" today and \"this\" too.\n",
|
|
1558
|
+
"out": "He said \"www.example.com/a_b\" today and ‘this’ too.\n",
|
|
1559
|
+
"note": "modes.md §3.7.3: the same in the mdx dialect."
|
|
1560
|
+
},
|
|
1561
|
+
{
|
|
1562
|
+
"id": "en-gb-markdown-commonmark-autolink-literal-after-digit",
|
|
1563
|
+
"rule": "ellipsis",
|
|
1564
|
+
"mode": "markdown",
|
|
1565
|
+
"dialect": "commonmark",
|
|
1566
|
+
"in": "See 1https://example.com/a... more and wait...\n",
|
|
1567
|
+
"out": "See 1https://example.com/a... more and wait…\n",
|
|
1568
|
+
"note": "modes.md §3.7.3 autolink-literal tail: the address is found wherever the scheme or www. appears, with no test on the preceding character, because a parser may read a link directly after a digit."
|
|
1569
|
+
},
|
|
1570
|
+
{
|
|
1571
|
+
"id": "en-gb-markdown-commonmark-autolink-literal-email-converts",
|
|
1572
|
+
"rule": "ellipsis",
|
|
1573
|
+
"mode": "markdown",
|
|
1574
|
+
"dialect": "commonmark",
|
|
1575
|
+
"in": "Mail foo@example.com... then wait...\n",
|
|
1576
|
+
"out": "Mail foo@example.com… then wait…\n",
|
|
1577
|
+
"note": "modes.md §3.7.3 autolink-literal tail: the tail rule covers http, https and www addresses only. An e-mail address ends where the parser says and the dots after it convert, as before."
|
|
1578
|
+
},
|
|
1579
|
+
{
|
|
1580
|
+
"id": "en-gb-markdown-commonmark-autolink-literal-terminator-kept",
|
|
1581
|
+
"rule": "dashes",
|
|
1582
|
+
"mode": "markdown",
|
|
1583
|
+
"dialect": "commonmark",
|
|
1584
|
+
"in": "Read https://example.com/a - fine and b - c\n",
|
|
1585
|
+
"out": "Read https://example.com/a – fine and b – c\n",
|
|
1586
|
+
"note": "modes.md §3.7.3 terminator guard: an edit that keeps U+0020 at the terminator is applied; a spaced hyphen after a bare URL still becomes an en dash when a word precedes the URL. The address alone at the start of the text is left as typed."
|
|
1587
|
+
},
|
|
1588
|
+
{
|
|
1589
|
+
"id": "en-gb-markdown-commonmark-shortcut-reference-label",
|
|
1590
|
+
"rule": "apostrophe",
|
|
1591
|
+
"mode": "markdown",
|
|
1592
|
+
"dialect": "commonmark",
|
|
1593
|
+
"in": "See [Smith's book] and [Smith's book][] and [it's][ref].\n\n[Smith's book]: https://example.com\n[ref]: https://example.com\n",
|
|
1594
|
+
"out": "See [Smith's book] and [Smith's book][] and [it’s][ref].\n\n[Smith's book]: https://example.com\n[ref]: https://example.com\n",
|
|
1595
|
+
"note": "modes.md §3.7.3 reference label: the text of a shortcut or collapsed reference link is also the label matched against its definition, so it is skipped; the text of a full reference link is free prose and converts."
|
|
1596
|
+
},
|
|
1597
|
+
{
|
|
1598
|
+
"id": "en-gb-markdown-mdx-shortcut-reference-label",
|
|
1599
|
+
"rule": "apostrophe",
|
|
1600
|
+
"mode": "markdown",
|
|
1601
|
+
"dialect": "mdx",
|
|
1602
|
+
"in": "See [Smith's book] and [Smith's book][] and [it's][ref].\n\n[Smith's book]: https://example.com\n[ref]: https://example.com\n",
|
|
1603
|
+
"out": "See [Smith's book] and [Smith's book][] and [it’s][ref].\n\n[Smith's book]: https://example.com\n[ref]: https://example.com\n",
|
|
1604
|
+
"note": "modes.md §3.7.3: the reference-label rule holds in both dialects."
|
|
1605
|
+
},
|
|
1606
|
+
{
|
|
1607
|
+
"id": "en-gb-markdown-commonmark-image-shortcut-reference-label",
|
|
1608
|
+
"rule": "apostrophe",
|
|
1609
|
+
"mode": "markdown",
|
|
1610
|
+
"dialect": "commonmark",
|
|
1611
|
+
"in": "See ![it's] and ![it's][] and ![it's][ref].\n\n[it's]: https://example.com/i.png\n[ref]: https://example.com/i.png\n",
|
|
1612
|
+
"out": "See ![it's] and ![it's][] and ![it’s][ref].\n\n[it's]: https://example.com/i.png\n[ref]: https://example.com/i.png\n",
|
|
1613
|
+
"note": "modes.md §3.7.3 reference label: the alt text of a shortcut or collapsed image reference is its label and is skipped; a full reference's alt text converts."
|
|
1614
|
+
},
|
|
1615
|
+
{
|
|
1616
|
+
"id": "en-gb-markdown-commonmark-bracketed-text-without-definition",
|
|
1617
|
+
"rule": "apostrophe",
|
|
1618
|
+
"mode": "markdown",
|
|
1619
|
+
"dialect": "commonmark",
|
|
1620
|
+
"in": "See [it's] and it's.\n",
|
|
1621
|
+
"out": "See [it’s] and it’s.\n",
|
|
1622
|
+
"note": "modes.md §3.7.3 reference label: square brackets that match no definition are not a link, so the text converts. A port that skips every bracketed run fails here."
|
|
1623
|
+
},
|
|
1624
|
+
{
|
|
1625
|
+
"id": "en-gb-markdown-commonmark-reference-label-defined-later-accepted-cost",
|
|
1626
|
+
"rule": "apostrophe",
|
|
1627
|
+
"mode": "markdown",
|
|
1628
|
+
"dialect": "commonmark",
|
|
1629
|
+
"in": "See [it's] and it's.\n\n[it’s]: https://example.com\n",
|
|
1630
|
+
"out": "See [it’s] and it’s.\n\n[it’s]: https://example.com\n",
|
|
1631
|
+
"note": "modes.md §3.7.3 reference label, accepted cost: text that matched no definition in the input converts to the spelling a definition uses and becomes a link. This direction cannot be closed without comparing definitions, and it creates a link rather than destroying one."
|
|
1632
|
+
},
|
|
1633
|
+
{
|
|
1634
|
+
"id": "en-gb-markdown-commonmark-footnote-labels",
|
|
1635
|
+
"rule": "apostrophe",
|
|
1636
|
+
"mode": "markdown",
|
|
1637
|
+
"dialect": "commonmark",
|
|
1638
|
+
"in": "Note[^a's] and it's.\n\n[^a's]: it's text\n",
|
|
1639
|
+
"out": "Note[^a's] and it’s.\n\n[^a's]: it’s text\n",
|
|
1640
|
+
"note": "modes.md §3.7.3: the label of a footnote call and of a footnote definition is skipped, so the two keep matching; the prose around and inside the footnote converts."
|
|
1641
|
+
},
|
|
1642
|
+
{
|
|
1643
|
+
"id": "en-gb-markdown-mdx-footnote-labels",
|
|
1644
|
+
"rule": "apostrophe",
|
|
1645
|
+
"mode": "markdown",
|
|
1646
|
+
"dialect": "mdx",
|
|
1647
|
+
"in": "Note[^a's] and it's.\n\n[^a's]: it's text\n",
|
|
1648
|
+
"out": "Note[^a's] and it’s.\n\n[^a's]: it’s text\n",
|
|
1649
|
+
"note": "modes.md §3.7.3: footnote labels are skipped in both dialects."
|
|
1479
1650
|
}
|
|
1480
1651
|
]
|
|
1481
1652
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"spec": "1.17.
|
|
2
|
+
"spec": "1.17.2",
|
|
3
3
|
"locale": "en-US",
|
|
4
4
|
"cases": [
|
|
5
5
|
{
|
|
@@ -4601,6 +4601,33 @@
|
|
|
4601
4601
|
],
|
|
4602
4602
|
"throws": "POLYTYPO_UNKNOWN_RULE",
|
|
4603
4603
|
"note": "modes.md §3.9.2, validation order: an unknown rule id is reported before a malformed protect."
|
|
4604
|
+
},
|
|
4605
|
+
{
|
|
4606
|
+
"id": "en-us-markdown-commonmark-delimiter-dash-collapse",
|
|
4607
|
+
"rule": "dashes",
|
|
4608
|
+
"mode": "markdown",
|
|
4609
|
+
"dialect": "commonmark",
|
|
4610
|
+
"in": "x _ -- y_ and a -- b\n",
|
|
4611
|
+
"out": "x _ -- y_ and a—b\n",
|
|
4612
|
+
"note": "modes.md §3.4 delimiter-neighbour rule: collapsing the spaced dash to a closed-up dash would remove the space beside the underscore and let it open emphasis; that edit is discarded. The same collapse beside ordinary text (a -- b) is applied."
|
|
4613
|
+
},
|
|
4614
|
+
{
|
|
4615
|
+
"id": "en-us-markdown-commonmark-delimiter-neighbour-symbols-kept",
|
|
4616
|
+
"rule": "symbols",
|
|
4617
|
+
"mode": "markdown",
|
|
4618
|
+
"dialect": "commonmark",
|
|
4619
|
+
"in": "x*(c) y* and a*+/-5 b*\n",
|
|
4620
|
+
"out": "x*© y* and a*±5 b*\n",
|
|
4621
|
+
"note": "modes.md §3.4 delimiter-neighbour rule, premise: a replacement of punctuation or a symbol by another symbol beside a delimiter keeps the delimiter's class under CommonMark 0.31, so (c) and +/- still convert there. A parser that classifies symbols differently would read these two lines differently, and this fixture is what makes it fail."
|
|
4622
|
+
},
|
|
4623
|
+
{
|
|
4624
|
+
"id": "en-us-markdown-commonmark-autolink-literal-terminator-collapse",
|
|
4625
|
+
"rule": "dashes",
|
|
4626
|
+
"mode": "markdown",
|
|
4627
|
+
"dialect": "commonmark",
|
|
4628
|
+
"in": "Read https://example.com/a -- fine and b -- c\n",
|
|
4629
|
+
"out": "Read https://example.com/a -- fine and b—c\n",
|
|
4630
|
+
"note": "modes.md §3.7.3 terminator guard: collapsing the spaced dash would delete the space that ends the URL, and the parser would read the dash and the next word as part of it; that edit is discarded. The same collapse away from a URL (b -- c) is applied."
|
|
4604
4631
|
}
|
|
4605
4632
|
]
|
|
4606
4633
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"spec": "1.17.
|
|
2
|
+
"spec": "1.17.2",
|
|
3
3
|
"locale": "fr",
|
|
4
4
|
"cases": [
|
|
5
5
|
{
|
|
@@ -1078,6 +1078,42 @@
|
|
|
1078
1078
|
}
|
|
1079
1079
|
],
|
|
1080
1080
|
"note": "modes.md §3.9.4, accepted cost: the narrow space would be inserted at a span edge next to the hole, which the edge-growth rule (§3.4) discards. Without protect it is inserted."
|
|
1081
|
+
},
|
|
1082
|
+
{
|
|
1083
|
+
"id": "fr-markdown-commonmark-delimiter-space-insertion",
|
|
1084
|
+
"rule": "nbsp",
|
|
1085
|
+
"mode": "markdown",
|
|
1086
|
+
"dialect": "commonmark",
|
|
1087
|
+
"in": "*a.**! et mot !\n",
|
|
1088
|
+
"out": "*a.**! et mot !\n",
|
|
1089
|
+
"note": "modes.md §3.4 delimiter-neighbour rule: a space inserted before the exclamation mark would make the double asterisk a closing delimiter that pairs with the first asterisk; that edit is discarded. The same insertion after a word (mot !) is applied."
|
|
1090
|
+
},
|
|
1091
|
+
{
|
|
1092
|
+
"id": "fr-markdown-commonmark-delimiter-tilde-insertion",
|
|
1093
|
+
"rule": "nbsp",
|
|
1094
|
+
"mode": "markdown",
|
|
1095
|
+
"dialect": "commonmark",
|
|
1096
|
+
"in": "Voir ~25 et « ~25 et «~25 enfin\n",
|
|
1097
|
+
"out": "Voir ~25 et « ~25 et «~25 enfin\n",
|
|
1098
|
+
"note": "modes.md §3.4 delimiter-neighbour rule, tilde as a delimiter: inserting a no-break space between the opening guillemet and a tilde is declined, because the tilde could then open strikethrough; the spaced form (« ~25) is already correct. An accepted cost: the rule tests adjacency to whitespace, not CommonMark's full flanking rules."
|
|
1099
|
+
},
|
|
1100
|
+
{
|
|
1101
|
+
"id": "fr-markdown-commonmark-autolink-literal-terminator-nbsp",
|
|
1102
|
+
"rule": "nbsp",
|
|
1103
|
+
"mode": "markdown",
|
|
1104
|
+
"dialect": "commonmark",
|
|
1105
|
+
"in": "Voir https://example.com/a : oui, et voilà : non\n",
|
|
1106
|
+
"out": "Voir https://example.com/a : oui, et voilà : non\n",
|
|
1107
|
+
"note": "modes.md §3.7.3 terminator guard, accepted cost: a no-break space is not U+0020 and GFM parsers differ on whether it ends a URL, so the space after the URL is left as typed; the one after a word converts."
|
|
1108
|
+
},
|
|
1109
|
+
{
|
|
1110
|
+
"id": "fr-markdown-commonmark-autolink-literal-email-terminator-converts",
|
|
1111
|
+
"rule": "nbsp",
|
|
1112
|
+
"mode": "markdown",
|
|
1113
|
+
"dialect": "commonmark",
|
|
1114
|
+
"in": "Écrire à foo@example.com ; oui\n",
|
|
1115
|
+
"out": "Écrire à foo@example.com ; oui\n",
|
|
1116
|
+
"note": "modes.md §3.7.3 terminator guard: it applies to http, https and www addresses only, so the space after an e-mail address converts as before."
|
|
1081
1117
|
}
|
|
1082
1118
|
]
|
|
1083
1119
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"spec": "1.17.
|
|
2
|
+
"spec": "1.17.2",
|
|
3
3
|
"$comment": "Locale resolution input. Algorithm is specified in spec/rules/locale-resolution.md and is identical in every runtime — never delegate it to a platform locale-negotiation library. \"spec\" here must track spec/VERSION exactly — it is not itself the global version source; scripts/validate-spec.mjs enforces the match.",
|
|
4
4
|
"locales": [
|
|
5
5
|
"en-US",
|
|
@@ -19,7 +19,7 @@ step 9 with §3.8.5's content-line condition, added in 1.10.0, §3.8.4's carriag
|
|
|
19
19
|
continuation line with the bails of §3.8.6 that use it, added in 1.11.0, and §3.8.4's node lines,
|
|
20
20
|
declined keys, node markers and own-line region, with the changes to steps 1, 3, 4, 5, 6, 7 and 8
|
|
21
21
|
that go with them, added in 1.11.0, and §3.9 with the amendments it carries to §3.1, §3.5 step 2,
|
|
22
|
-
§4, §5, §7 item 7 and §8, added in 1.13.0).
|
|
22
|
+
§4, §5, §7 item 7 and §8, added in 1.13.0, and §3.4's delimiter-neighbour rule with §3.7.3's autolink-literal tail, terminator guard and reference labels, and the amendments they carry to §3.4's deletion paragraph, §5 items 2 and 3, its fifth-part note and testing obligation, and §6, added in 1.17.2).
|
|
23
23
|
|
|
24
24
|
---
|
|
25
25
|
|
|
@@ -356,9 +356,13 @@ Together they distinguish exactly the cases that matter:
|
|
|
356
356
|
| U+0020 → U+00A0 (`nbsp`) | yes | 1 → 1 | **applied** — U+00A0 is not U+0020 |
|
|
357
357
|
| insertion of U+202F (`nbsp` N1/N2) | yes | 0 → 1 | **discarded**, as before |
|
|
358
358
|
|
|
359
|
-
**Deletion at an edge is not restricted**, and does not need to be. A rule
|
|
360
|
-
U+0020 (`spaces`), and a deletion cannot bring into existence a structural token
|
|
361
|
-
already there — it can only remove a character from inside a span.
|
|
359
|
+
**Deletion at an edge is not restricted by the edge-growth rule**, and does not need to be. A rule
|
|
360
|
+
can only delete U+0020 (`spaces`), and a deletion cannot bring into existence a structural token
|
|
361
|
+
that was not already there — it can only remove a character from inside a span. **That sentence
|
|
362
|
+
was true of every token the parser had already recognised, and false of one the parser had
|
|
363
|
+
declined to recognise.** An unpaired `_`, `*` or `~` is literal text inside a span, and whether it
|
|
364
|
+
is a delimiter depends on what stands beside it; deleting a space is enough to change that, and so
|
|
365
|
+
is inserting or replacing one. The delimiter-neighbour rule below closes it. The whitespace that _is_
|
|
362
366
|
structural in Markdown — a line prefix, a list marker's indentation, the two spaces of a hard
|
|
363
367
|
break — is protected by the −2 line marker (§3.2): it sits beside a `BREAK`, and `spaces`
|
|
364
368
|
refuses to touch a run bordering one, in every mode.
|
|
@@ -371,6 +375,43 @@ fixed point, but it no longer demonstrates anything: `dashes` §3.2 step 2a decl
|
|
|
371
375
|
containing an authored U+2013 or U+2014, so that input is now refused a step earlier and never
|
|
372
376
|
reaches the edge-growth rule. `--` is the live carrier.)
|
|
373
377
|
|
|
378
|
+
**The delimiter-neighbour rule (`markdown` only, spec 1.17.2).** In `markdown` mode the three
|
|
379
|
+
code points `DELIM` = U+002A, U+005F, U+007E are literal content wherever the parser found no
|
|
380
|
+
delimiter pair, and CommonMark decides whether a run of them can open or close emphasis from the
|
|
381
|
+
code points on either side of it: whitespace or not. Measured, with the pre-1.17.2 behaviour:
|
|
382
|
+
|
|
383
|
+
| Mode | Input | Locale | Output before 1.17.2 | What changed |
|
|
384
|
+
| ---------- | ------------------------------------ | ------- | ----------------------------- | ---------------------------------------------------------------------------------- |
|
|
385
|
+
| `markdown` | `see booking._ , emails.booking_ here` | `en-GB` | `see booking._, emails.booking_ here` | `spaces` deleted the space; the two underscores now pair as emphasis |
|
|
386
|
+
| `markdown` | `x _ -- y_` | `en-US` | `x _—y_` | `dashes` collapsed `␣--␣`, removing the space beside the underscore; same result |
|
|
387
|
+
| `markdown` | `*a.**!` | `fr` | `*a.**` U+202F `!` | `nbsp` inserted a narrow no-break space before the `!`; the double asterisk now closes and pairs |
|
|
388
|
+
|
|
389
|
+
Write `W` for the set of Unicode whitespace in the CommonMark sense (U+0009, U+000A, U+000C,
|
|
390
|
+
U+000D, U+0020, U+00A0, U+1680, U+2000–U+200A, U+202F, U+205F, U+3000). For an edit replacing
|
|
391
|
+
`cp[p … q]` with `w` of length `r` (an insertion has `q = p − 1`), evaluated on the array the rule
|
|
392
|
+
read:
|
|
393
|
+
|
|
394
|
+
- if `cp[p−1]` is in `DELIM`, let `old = cp[p]` and `new = w[0]` when `r > 0`, otherwise `cp[q+1]`;
|
|
395
|
+
- if `cp[q+1]` is in `DELIM`, let `old = cp[q]` and `new = w[r−1]` when `r > 0`, otherwise `cp[p−1]`.
|
|
396
|
+
|
|
397
|
+
> **The edit is discarded if, on either side, exactly one of `old` and `new` is in `W`.** A marker
|
|
398
|
+
> belongs to neither class, a position outside the array is neither, and the test reads no other
|
|
399
|
+
> code point and no parser state.
|
|
400
|
+
|
|
401
|
+
It applies to `markdown` and to no other mode: `text` has no emphasis, and `html` has no
|
|
402
|
+
delimiters in that sense. It is a function of the array alone, so it fires identically on every
|
|
403
|
+
run (§5 item 3). It is deliberately a test of **adjacency to whitespace**, not a reimplementation
|
|
404
|
+
of CommonMark's flanking rules, and it is therefore stricter than flanking requires: in `fr`,
|
|
405
|
+
`«~25` (U+00AB U+007E …) keeps its tight form although the tilde would flank the same way after a
|
|
406
|
+
no-break space. That is accepted: a delimiter beside a space that a rule would change is rare in
|
|
407
|
+
prose.
|
|
408
|
+
|
|
409
|
+
**Its premise is that no rule swaps punctuation for non-punctuation beside a delimiter.** CommonMark
|
|
410
|
+
0.31 counts symbols as punctuation, so `(c)` → `©` and `+/-` → `±` keep a delimiter's class, and
|
|
411
|
+
the rule tests whitespace alone. A parser that implements an older definition (punctuation only)
|
|
412
|
+
would classify them differently, and two fixtures (`en-us-markdown-commonmark-delimiter-neighbour-symbols-kept`)
|
|
413
|
+
make such a port fail rather than drift. No rule emits a member of `DELIM`.
|
|
414
|
+
|
|
374
415
|
### 3.5 Applying the result
|
|
375
416
|
|
|
376
417
|
1. Extract the span sequence from the source. Record each span's **source offsets**.
|
|
@@ -515,6 +556,44 @@ Skipped, exhaustively:
|
|
|
515
556
|
unless tables are recognised at all, and §6's `[P: html, markdown]` marking on `dashes`' URL
|
|
516
557
|
bullet holds **only** because GFM autolink literals make a bare `https://…` a link — in pure
|
|
517
558
|
CommonMark a bare URL is ordinary text and would be typeset;
|
|
559
|
+
- **the tail of a bare web address (spec 1.17.2).** Wherever the source text of a **maximal span**
|
|
560
|
+
contains `http://`, `https://` or `www.` (ASCII letters in either case, **with no test on the
|
|
561
|
+
character before it**: a parser may read a link directly after a digit), everything from there up to — and **not** including — the first U+0020,
|
|
562
|
+
U+0009, U+000A, U+000D or `<`, or the end of the span, is skipped. The same holds from the
|
|
563
|
+
position where a parser-recognised GFM autolink literal that begins with one of those prefixes
|
|
564
|
+
ends, up to the same point.
|
|
565
|
+
This is a scan of the source and not a question put to a parser, because parsers disagree about
|
|
566
|
+
where such an address may start (`"www.example.com"`: GFM's own tokenizer reads no link there,
|
|
567
|
+
mdast's autolink transform reads one) and where it ends. The rules would otherwise change what
|
|
568
|
+
ends it: a closing `"` that becomes U+201D, an apostrophe that becomes U+2019, or `...` that
|
|
569
|
+
becomes U+2026 is not ASCII punctuation, and the parser reading the _output_ takes it into the
|
|
570
|
+
link destination. That is damage to the link, not to the typography. A quotation mark that cannot
|
|
571
|
+
convert leaves its partner unpaired, so the pair is left as typed rather than half-converted.
|
|
572
|
+
**An e-mail address is not covered**: it ends where the parser says, and no rule changes that.
|
|
573
|
+
A **maximal span** is the run of processable text with no source between its pieces: the pieces a
|
|
574
|
+
parser reports separately (an unpaired `_` or `*` is one) are **merged before this scan and before
|
|
575
|
+
§3.4's delimiter-neighbour rule reads its neighbours**. §7 item 5's 'should' is a 'must' in
|
|
576
|
+
`markdown`, since both depend on it;
|
|
577
|
+
- **the terminator of a bare web address (spec 1.17.2).** A span that begins at the U+0020 or
|
|
578
|
+
U+0009 that ends such an address discards every edit that starts at its first code point, unless
|
|
579
|
+
the replacement begins with U+0020. Collapsing `␣--␣` to a closed-up dash would delete the space
|
|
580
|
+
and let the address run on into the dash and the next word, and a no-break space is not
|
|
581
|
+
guaranteed to end an address in every GFM parser. Edits that keep a U+0020 there are applied:
|
|
582
|
+
`Read https://example.com/a - b` still becomes `Read https://example.com/a – b` in `en-GB`;
|
|
583
|
+
- **the label of a reference (spec 1.17.2).** In `[Smith's book]` and `[Smith's book][]` the link
|
|
584
|
+
text is also the label matched against `[Smith's book]: …`, and that label is skipped (above).
|
|
585
|
+
Converting the text makes it match nothing and the link disappears, so the text of a **shortcut**
|
|
586
|
+
or **collapsed** reference is skipped as well, and so is the alt text of such an image. A reference
|
|
587
|
+
is a bracketed run that the parser formed from a definition: **shortcut** when its source ends at
|
|
588
|
+
the `]` of the text, **collapsed** when `[]` follows it directly, **full** when a label in `[…]`
|
|
589
|
+
follows. Labels match as CommonMark §4.7 matches them, with no locale-dependent case mapping.
|
|
590
|
+
Square brackets that match no definition are not a reference and their text converts; the text of a full
|
|
591
|
+
reference, `[text][ref]`, is prose and converts. The label of a **footnote call** `[^x]` and of a
|
|
592
|
+
**footnote definition** `[^x]:` is skipped for the same reason; the 1.17.1 text above already said
|
|
593
|
+
so for the definition, and the JavaScript engine did not do it (measured on 1.17.1). **Accepted:** text that matched no
|
|
594
|
+
definition in the input and matches one after conversion (`[it's]` with `[it’s]: …` elsewhere)
|
|
595
|
+
becomes a link; closing that direction needs the definitions compared, and it creates a link
|
|
596
|
+
rather than destroying one;
|
|
518
597
|
- HTML blocks and inline raw HTML, which are handed to the `html` skip list of §3.6 rather than
|
|
519
598
|
processed as markdown. **Inline HTML arrives as isolated tags with Markdown between them**,
|
|
520
599
|
not as a tree, so §3.6's subtree rule must be implemented as a **stack of open skipped
|
|
@@ -1661,7 +1740,7 @@ they do not carry it over for free either. Write `M` for the whole mode transfor
|
|
|
1661
1740
|
indented code. `dashes` emits U+0020 in every `-spaced` locale. The claim was wrong when it
|
|
1662
1741
|
was written, and it is what hid the defect now repaired in §3.4.
|
|
1663
1742
|
|
|
1664
|
-
The honest discharge has four parts, and the third is **positional rather than alphabetic** —
|
|
1743
|
+
The honest discharge has four parts (a fifth, for what the parser decides, was added in 1.17.2 and follows them), and the third is **positional rather than alphabetic** —
|
|
1665
1744
|
the same move as CO-S in `pipeline-idempotency.md` §5.1a, applied to _where_ a character may
|
|
1666
1745
|
be placed rather than to _which_ characters exist:
|
|
1667
1746
|
|
|
@@ -1675,11 +1754,22 @@ they do not carry it over for free either. Write `M` for the whole mode transfor
|
|
|
1675
1754
|
- **U+0020 is made safe by position, not by inertness.** The edge-growth rule (§3.4) means a
|
|
1676
1755
|
rule can introduce a code point only at a position **strictly interior** to a span. An
|
|
1677
1756
|
interior position has span content on both sides, so an emitted U+0020 can never become
|
|
1678
|
-
adjacent to a delimiter —
|
|
1757
|
+
adjacent to a **recognised** delimiter — those are structural tokens, outside every span — and can
|
|
1679
1758
|
never begin a line, because a line terminator is always a −2 marker and a line start is
|
|
1680
1759
|
therefore a span edge or outside spans entirely. **The characters that decide Markdown
|
|
1681
1760
|
structure and the positions a rule may write to are disjoint sets.** That is what makes the
|
|
1682
1761
|
partition stable; no property of U+0020 itself is relied upon.
|
|
1762
|
+
- **A delimiter the parser did not recognise is the exception, and it is closed by §3.4's
|
|
1763
|
+
delimiter-neighbour rule (spec 1.17.2).** The sentence above holds for tokens outside every
|
|
1764
|
+
span. An unpaired `*`, `_` or `~` is _inside_ a span as literal text, and a rule that changes
|
|
1765
|
+
whether whitespace stands beside it can make it a delimiter: this was a false statement until
|
|
1766
|
+
1.17.2, and §3.4's delimiter-neighbour rule is the repair. The same is true of the end of a bare URL, which only
|
|
1767
|
+
the parser decides, and of a reference label, which only a definition elsewhere decides — §3.7.3
|
|
1768
|
+
skips the tail of an autolink literal and the text of a shortcut or collapsed reference for the
|
|
1769
|
+
same reason. **This list is not claimed to be complete.** A deletion
|
|
1770
|
+
can also create a backslash escape or a character reference from literal text (`foo\ ,`
|
|
1771
|
+
becomes `foo\,` and `AT& ;T` becomes `AT&T`): the parse tree keeps its shape and its
|
|
1772
|
+
text changes, which a shape comparison does not see. That class is not closed by 1.17.2.
|
|
1683
1773
|
- **Deletion** is only ever of a U+0020, and structural whitespace borders a `BREAK` — a real
|
|
1684
1774
|
one in `text` mode, a −2 marker in the others — where `spaces` refuses to act
|
|
1685
1775
|
(`spaces.md` §3.2 step 4). A deletion therefore cannot turn `- item` into `-item`, collapse
|
|
@@ -1720,7 +1810,9 @@ they do not carry it over for free either. Write `M` for the whole mode transfor
|
|
|
1720
1810
|
|
|
1721
1811
|
> **The fifth part, normative and binding on future modes: the predicate that decides whether
|
|
1722
1812
|
> a span exists must not read anything a rule can change.** In `html` and `markdown` the
|
|
1723
|
-
> question
|
|
1813
|
+
> question does not arise for the span _predicate_, because span selection reads only the
|
|
1814
|
+
> document's structure. It arose for the span's _neighbourhood_, which a parser also reads: see
|
|
1815
|
+
> the delimiter-neighbour rule and §3.7.3's autolink tail. `yaml`'s
|
|
1724
1816
|
> first draft decided a plain scalar's processability from **its own text** — it had to contain
|
|
1725
1817
|
> a space and a letter — and `spaces` can delete that space. One line was enough to falsify the
|
|
1726
1818
|
> obligation: `ref: ${{ steps.pin.outputs.sha }}` yields one span, comes back as
|
|
@@ -1748,8 +1840,10 @@ they do not carry it over for free either. Write `M` for the whole mode transfor
|
|
|
1748
1840
|
3. **Redistribution is deterministic.** Guaranteed by §3.4: no edit spans a marker, and any edit
|
|
1749
1841
|
that would place code points at a span extremity which were not there before — an insertion at
|
|
1750
1842
|
a boundary, or a replacement longer than what it replaced — is discarded rather than assigned
|
|
1751
|
-
to a side. The discard is a pure function of
|
|
1752
|
-
|
|
1843
|
+
to a side. The discard is a pure function of the array and its span table (since 1.17.2: of
|
|
1844
|
+
`(p, q, r, s₀, s₁)`, of the code points beside the edit for §3.4's delimiter-neighbour rule, and
|
|
1845
|
+
of whether the span begins at the end of a web address for §3.7.3's terminator guard), so it
|
|
1846
|
+
happens identically on every run: a rule that was declined at an edge once is declined there always, and the output
|
|
1753
1847
|
contains nothing for the next run to reconsider.
|
|
1754
1848
|
|
|
1755
1849
|
Given 1–3, `M(M(x)) = M(x)`.
|
|
@@ -1774,6 +1868,21 @@ the single-unit templates cannot reach. §3.8.7 records what each run did and di
|
|
|
1774
1868
|
structure and types is blind to a string whose content is damaged, and that an alphabet without
|
|
1775
1869
|
`---` reported clean twice while §3.4's `r = d` hole was open.
|
|
1776
1870
|
|
|
1871
|
+
Since 1.17.2 the `markdown` run must compare, between the input and the output, **the parse
|
|
1872
|
+
tree including link destinations and reference identifiers**, not only the span partition: the
|
|
1873
|
+
defect behind the delimiter-neighbour rule changed the tree while the partition looked the same,
|
|
1874
|
+
and a sweep that compares spans alone cannot see it. Its alphabet must include `*`, `_`, `~`, a
|
|
1875
|
+
space, U+002D, U+0022, U+0027, `.`, `!` and `:`, and its templates must include a **bare URL
|
|
1876
|
+
followed by each of those**, and a reference link with its definition. Run it in locales whose
|
|
1877
|
+
`dashes` collapses a spaced dash (`en-US`), whose `nbsp` inserts before `!` (`fr`), and whose
|
|
1878
|
+
quotes are curly, because each reaches a different rule. The tree to compare is **mdast's, as `remark-parse` with `remark-gfm` produces it** (and, where a
|
|
1879
|
+
runtime has another parser, that parser's as well), not only the tokenizer's own: the tokenizer reads
|
|
1880
|
+
no link in `"www.example.com"`, mdast does, and that is the class §3.7.3's scan closes. Templates
|
|
1881
|
+
must include an address with `_`, `*` and `~` inside it after `"`, `'` and `(`, and an address
|
|
1882
|
+
directly after a digit. **Two classes are carved out because they are accepted, not because
|
|
1883
|
+
they are untested**: a text that matched no definition in the input and matches one in the output
|
|
1884
|
+
(§3.7.3's accepted case), and an e-mail address, whose end only the parser decides.
|
|
1885
|
+
|
|
1777
1886
|
---
|
|
1778
1887
|
|
|
1779
1888
|
## 6. Which §4 claims change
|
|
@@ -1827,6 +1936,12 @@ means the claim holds in those modes only.
|
|
|
1827
1936
|
No rule's §4 becomes _false_ in a mode. Two become narrower ([P: html, markdown]); none becomes
|
|
1828
1937
|
rule-local.
|
|
1829
1938
|
|
|
1939
|
+
**Since 1.17.2 every rule's reach in `markdown` is narrower in three further places**, none of which
|
|
1940
|
+
makes a §4 statement false: beside an unpaired `*`, `_` or `~` where an edit would change whether
|
|
1941
|
+
whitespace stands next to it (§3.4); in the tail of a bare web address and at the space that ends it
|
|
1942
|
+
(§3.7.3); and in the text of a shortcut or collapsed reference and in footnote labels (§3.7.3). In
|
|
1943
|
+
each, the rule is given fewer chances to fire and the text is left as typed.
|
|
1944
|
+
|
|
1830
1945
|
---
|
|
1831
1946
|
|
|
1832
1947
|
## 7. Open questions
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"spec": "1.17.
|
|
2
|
+
"spec": "1.17.2",
|
|
3
3
|
"$comment": "Single source of truth for pipeline order. Rules run in ascending `order`. Disabling a rule removes it from the sequence and never reorders the rest. Rule ids are public API (see docs/ARCHITECTURE.md section 5). \"spec\" here must track spec/VERSION exactly — it is not itself the global version source; scripts/validate-spec.mjs enforces the match.",
|
|
4
4
|
"rules": [
|
|
5
5
|
{
|
|
@@ -287,12 +287,108 @@ module Polytypo
|
|
|
287
287
|
parser_source = normalize_lone_cr(mask_frontmatter(source, block))
|
|
288
288
|
|
|
289
289
|
spans = []
|
|
290
|
+
state = { stack: [], frontmatter_end: block.nil? ? 0 : block.end, labels: [], url_ends: Set.new }
|
|
290
291
|
ParseError.wrap do
|
|
291
|
-
doc = Commonmarker.parse(
|
|
292
|
-
|
|
292
|
+
doc = Commonmarker.parse(
|
|
293
|
+
parser_source,
|
|
294
|
+
options: { parse: { sourcepos_chars: true }, extension: { footnotes: true } }
|
|
295
|
+
)
|
|
296
|
+
walk(doc, chars, line_starts, spans, state)
|
|
297
|
+
end
|
|
298
|
+
trim_url_tails(chars, Spans.normalize_spans(remove_ranges(spans, state[:labels])), state[:url_ends])
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
URL_PREFIXES = ["www.", "http://", "https://"].freeze
|
|
302
|
+
|
|
303
|
+
# modes.md 3.7.3 (spec 1.17.2): does a bare web address begin at +i+ and fit before +limit+?
|
|
304
|
+
# ASCII case-insensitive, and no test on the character before it.
|
|
305
|
+
def self.starts_url?(chars, i, limit)
|
|
306
|
+
URL_PREFIXES.any? do |word|
|
|
307
|
+
next false if i + word.length > limit
|
|
308
|
+
|
|
309
|
+
word.each_char.with_index.all? do |ch, k|
|
|
310
|
+
c = chars[i + k]
|
|
311
|
+
(c.ord >= 0x41 && c.ord <= 0x5A ? c.downcase : c) == ch
|
|
312
|
+
end
|
|
313
|
+
end
|
|
314
|
+
end
|
|
315
|
+
private_class_method :starts_url?
|
|
316
|
+
|
|
317
|
+
URL_TERMINATORS = [SPACE, TAB, LF, CR, "<"].freeze
|
|
318
|
+
|
|
319
|
+
# modes.md 3.7.3 (spec 1.17.2): the tail of a bare web address, found by scanning the source
|
|
320
|
+
# rather than asking a parser, because parsers disagree about where one may start and end. A
|
|
321
|
+
# span that begins at the U+0020 or U+0009 ending such an address is flagged +url_tail+, which
|
|
322
|
+
# guards its first code point (modes.md 3.4, Spans.filter_boundary_edits).
|
|
323
|
+
def self.trim_url_tails(chars, spans, url_ends)
|
|
324
|
+
out = []
|
|
325
|
+
spans.each do |span|
|
|
326
|
+
from = span.start
|
|
327
|
+
i = span.start
|
|
328
|
+
guard = span.url_tail == true ||
|
|
329
|
+
(url_ends.include?(span.start) && [SPACE, TAB].include?(chars[span.start]))
|
|
330
|
+
while i < span.end
|
|
331
|
+
unless (i == span.start && url_ends.include?(i)) || starts_url?(chars, i, span.end)
|
|
332
|
+
i += 1
|
|
333
|
+
next
|
|
334
|
+
end
|
|
335
|
+
e = i
|
|
336
|
+
e += 1 while e < span.end && !URL_TERMINATORS.include?(chars[e])
|
|
337
|
+
if e == i
|
|
338
|
+
i += 1
|
|
339
|
+
next
|
|
340
|
+
end
|
|
341
|
+
out << Spans::Span.new(from, i, guard ? true : nil) if i > from
|
|
342
|
+
guard = e < span.end && [SPACE, TAB].include?(chars[e])
|
|
343
|
+
from = e
|
|
344
|
+
i = e
|
|
345
|
+
end
|
|
346
|
+
out << Spans::Span.new(from, span.end, guard ? true : nil) if from < span.end
|
|
293
347
|
end
|
|
294
|
-
|
|
348
|
+
out
|
|
349
|
+
end
|
|
350
|
+
private_class_method :trim_url_tails
|
|
351
|
+
|
|
352
|
+
def self.remove_ranges(spans, ranges)
|
|
353
|
+
return spans if ranges.empty?
|
|
354
|
+
|
|
355
|
+
spans.flat_map do |span|
|
|
356
|
+
ranges.reduce([span]) do |pieces, r|
|
|
357
|
+
pieces.flat_map do |q|
|
|
358
|
+
next [q] if r.end <= q.start || r.start >= q.end
|
|
359
|
+
|
|
360
|
+
res = []
|
|
361
|
+
res << Spans::Span.new(q.start, r.start) if q.start < r.start
|
|
362
|
+
res << Spans::Span.new(r.end, q.end) if r.end < q.end
|
|
363
|
+
res
|
|
364
|
+
end
|
|
365
|
+
end
|
|
366
|
+
end
|
|
367
|
+
end
|
|
368
|
+
private_class_method :remove_ranges
|
|
369
|
+
|
|
370
|
+
# modes.md 3.7.3 (spec 1.17.2): the text of a shortcut or collapsed reference is its label.
|
|
371
|
+
# A node exists only when the parser formed it from a definition, so the kind is read off the
|
|
372
|
+
# source after the closing bracket of the text: nothing (shortcut), "[]" (collapsed); a
|
|
373
|
+
# resource "(...)" or a non-empty "[label]" (full) leaves the text processable.
|
|
374
|
+
def self.reference_label_range(node, chars, line_starts)
|
|
375
|
+
sp = node.source_position
|
|
376
|
+
start_offset = offset_of(line_starts, sp[:start_line], sp[:start_column])
|
|
377
|
+
end_offset = offset_of(line_starts, sp[:end_line], sp[:end_column])
|
|
378
|
+
last = node.to_a.last
|
|
379
|
+
return nil if last.nil?
|
|
380
|
+
|
|
381
|
+
lsp = last.source_position
|
|
382
|
+
close = offset_of(line_starts, lsp[:end_line], lsp[:end_column]) + 1
|
|
383
|
+
return nil unless chars[close] == "]"
|
|
384
|
+
|
|
385
|
+
shortcut = end_offset == close
|
|
386
|
+
collapsed = end_offset == close + 2 && chars[close + 1] == "[" && chars[close + 2] == "]"
|
|
387
|
+
return nil unless shortcut || collapsed
|
|
388
|
+
|
|
389
|
+
Spans::Span.new(start_offset + (node.type == :image ? 2 : 1), close)
|
|
295
390
|
end
|
|
391
|
+
private_class_method :reference_label_range
|
|
296
392
|
|
|
297
393
|
# line_starts[i] is the 0-based code-point index at which comrak's 1-based line i begins.
|
|
298
394
|
# comrak counts CommonMark line endings, which are U+000A, U+000D, and U+000D U+000A --
|
|
@@ -347,12 +443,20 @@ module Polytypo
|
|
|
347
443
|
if type == :link || type == :image
|
|
348
444
|
sp = node.source_position
|
|
349
445
|
start_offset = offset_of(line_starts, sp[:start_line], sp[:start_column])
|
|
350
|
-
|
|
446
|
+
opener = type == :image ? start_offset + 1 : start_offset
|
|
447
|
+
unless chars[opener] == "[" && (type == :link || chars[start_offset] == "!")
|
|
351
448
|
# No leading "[" at the link's own source position: this is an autolink
|
|
352
449
|
# (<https://…> or a bare GFM autolink literal), not a real [text](url) link --
|
|
353
|
-
# skipped whole, per modes.md 3.7.3.
|
|
450
|
+
# skipped whole, per modes.md 3.7.3. The end of a literal that begins with a web
|
|
451
|
+
# prefix is where the tail scan resumes.
|
|
452
|
+
if type == :link && chars[start_offset] != "<" &&
|
|
453
|
+
starts_url?(chars, start_offset, chars.length)
|
|
454
|
+
state[:url_ends] << (offset_of(line_starts, sp[:end_line], sp[:end_column]) + 1)
|
|
455
|
+
end
|
|
354
456
|
return
|
|
355
457
|
end
|
|
458
|
+
label = reference_label_range(node, chars, line_starts)
|
|
459
|
+
state[:labels] << label unless label.nil?
|
|
356
460
|
# A real link/image: fall through to the generic child walk below, which emits the
|
|
357
461
|
# link text (or image alt text) as processable prose. The destination/title are never
|
|
358
462
|
# represented as child nodes at all, so there is nothing further to exclude.
|
|
@@ -13,12 +13,16 @@ module Polytypo
|
|
|
13
13
|
module Runner
|
|
14
14
|
# The same sequence as Engine::Pipeline.run_rules, with the two boundary filters of
|
|
15
15
|
# modes.md 3.4 interposed.
|
|
16
|
-
def self.run_rules_over_spans(cp, plan, locale_data, ctx)
|
|
16
|
+
def self.run_rules_over_spans(cp, plan, locale_data, ctx, guarded = [])
|
|
17
17
|
current = cp
|
|
18
|
+
markdown = ctx.mode == "markdown"
|
|
18
19
|
plan.each do |rule_id|
|
|
19
20
|
fn = Engine::Registry.rule(rule_id)
|
|
20
21
|
edits = fn.call(current, locale_data, ctx)
|
|
21
|
-
filtered = Spans.filter_boundary_edits(
|
|
22
|
+
filtered = Spans.filter_boundary_edits(
|
|
23
|
+
current, edits, Spans.span_ranges_of(current),
|
|
24
|
+
delimiter_neighbours: markdown, guarded: guarded
|
|
25
|
+
)
|
|
22
26
|
next if filtered.empty?
|
|
23
27
|
|
|
24
28
|
current = Engine::Edits.apply_edits(current, filtered, rule_id)
|
|
@@ -61,7 +65,8 @@ module Polytypo
|
|
|
61
65
|
return [] if normalized.empty?
|
|
62
66
|
|
|
63
67
|
concatenated = Spans.concatenate_spans(source_cp, normalized)
|
|
64
|
-
|
|
68
|
+
guarded = normalized.map { |s| s.url_tail == true }
|
|
69
|
+
transformed = run_rules_over_spans(concatenated, plan, locale_data, ctx, guarded)
|
|
65
70
|
pieces = Spans.split_on_marker(transformed, normalized.length)
|
|
66
71
|
normalized.each_with_index.map { |span, i| [span, pieces[i]] }
|
|
67
72
|
end
|
|
@@ -91,12 +96,16 @@ module Polytypo
|
|
|
91
96
|
normalized = Spans.normalize_spans(spans)
|
|
92
97
|
return [] if normalized.empty?
|
|
93
98
|
|
|
99
|
+
guarded = normalized.map { |s| s.url_tail == true }
|
|
94
100
|
Engine::Pipeline.run_rules_recording(
|
|
95
101
|
Spans.concatenate_spans(source_cp, normalized),
|
|
96
102
|
plan, locale_data, ctx,
|
|
97
103
|
Spans.origin_of_spans(normalized),
|
|
98
104
|
source_cp.length
|
|
99
|
-
)
|
|
105
|
+
) do |current, edits|
|
|
106
|
+
Spans.filter_boundary_edits(current, edits, Spans.span_ranges_of(current),
|
|
107
|
+
delimiter_neighbours: ctx.mode == "markdown", guarded: guarded)
|
|
108
|
+
end
|
|
100
109
|
end
|
|
101
110
|
end
|
|
102
111
|
end
|
data/lib/polytypo/modes/spans.rb
CHANGED
|
@@ -18,7 +18,10 @@ module Polytypo
|
|
|
18
18
|
# A processable span, identified by its offsets in the original source, addressed as
|
|
19
19
|
# code-point indices (ARCHITECTURE.md section 4.2) -- Ruby String indices are already
|
|
20
20
|
# code-point indices for a UTF-8 String, no separate byte/char distinction to manage here.
|
|
21
|
-
|
|
21
|
+
#
|
|
22
|
+
# +url_tail+ is true (or nil) on a span that begins at the U+0020 or U+0009 ending a bare web
|
|
23
|
+
# address, so the filter can guard its first code point (modes.md 3.7.3, spec 1.17.2).
|
|
24
|
+
Span = Struct.new(:start, :end, :url_tail)
|
|
22
25
|
|
|
23
26
|
# A span's extent in the concatenated code-point array: s0/s1 of modes.md 3.4. `:first`
|
|
24
27
|
# deliberately shadows Struct#first with the equivalent one-argument-less accessor -- this
|
|
@@ -50,7 +53,7 @@ module Polytypo
|
|
|
50
53
|
)
|
|
51
54
|
end
|
|
52
55
|
if span.start == last.end
|
|
53
|
-
out[-1] = Span.new(last.start, span.end)
|
|
56
|
+
out[-1] = Span.new(last.start, span.end, last.url_tail)
|
|
54
57
|
else
|
|
55
58
|
out << span
|
|
56
59
|
end
|
|
@@ -106,6 +109,35 @@ module Polytypo
|
|
|
106
109
|
end
|
|
107
110
|
private_class_method :span_containing
|
|
108
111
|
|
|
112
|
+
DELIMITERS = [0x2A, 0x5F, 0x7E].freeze
|
|
113
|
+
|
|
114
|
+
def self.markdown_whitespace?(c)
|
|
115
|
+
return false if c.nil?
|
|
116
|
+
|
|
117
|
+
[0x09, 0x0A, 0x0C, 0x0D, 0x20, 0xA0, 0x1680, 0x202F, 0x205F, 0x3000].include?(c) ||
|
|
118
|
+
(c >= 0x2000 && c <= 0x200A)
|
|
119
|
+
end
|
|
120
|
+
private_class_method :markdown_whitespace?
|
|
121
|
+
|
|
122
|
+
# modes.md 3.4 delimiter-neighbour rule (markdown only, spec 1.17.2): an edit is discarded
|
|
123
|
+
# when, beside U+002A, U+005F or U+007E, exactly one of the old and new neighbour is
|
|
124
|
+
# CommonMark whitespace. A marker belongs to neither class, nor does a position outside the
|
|
125
|
+
# array. An insertion has q = p - 1.
|
|
126
|
+
def self.changes_delimiter_flank?(cp, p, q, replacement)
|
|
127
|
+
r = replacement.length
|
|
128
|
+
before_edit = p.positive? ? cp[p - 1] : nil
|
|
129
|
+
if DELIMITERS.include?(before_edit)
|
|
130
|
+
after = r.positive? ? replacement[0] : cp[q + 1]
|
|
131
|
+
return true if markdown_whitespace?(cp[p]) != markdown_whitespace?(after)
|
|
132
|
+
end
|
|
133
|
+
if q + 1 < cp.length && DELIMITERS.include?(cp[q + 1])
|
|
134
|
+
after = r.positive? ? replacement[r - 1] : before_edit
|
|
135
|
+
return true if markdown_whitespace?(q.negative? ? nil : cp[q]) != markdown_whitespace?(after)
|
|
136
|
+
end
|
|
137
|
+
false
|
|
138
|
+
end
|
|
139
|
+
private_class_method :changes_delimiter_flank?
|
|
140
|
+
|
|
109
141
|
# modes.md 3.4, two safety nets, both pure functions of (p, q, r, s0, s1):
|
|
110
142
|
#
|
|
111
143
|
# 1. No edit may contain a marker -- one that does is a bug, discarded rather than
|
|
@@ -118,7 +150,11 @@ module Polytypo
|
|
|
118
150
|
# clause tests the CHARACTER, and only U+0020 needs testing -- it is the one code
|
|
119
151
|
# point any rule emits whose meaning comes from its position rather than from itself
|
|
120
152
|
# (modes.md 5 item 2).
|
|
121
|
-
|
|
153
|
+
#
|
|
154
|
+
# +delimiter_neighbours+ (markdown only) enables the delimiter-neighbour rule of modes.md 3.4,
|
|
155
|
+
# and +guarded+ holds, per span, whether its first code point is the terminator of a bare web
|
|
156
|
+
# address (modes.md 3.7.3); both spec 1.17.2.
|
|
157
|
+
def self.filter_boundary_edits(cp, edits, ranges, delimiter_neighbours: false, guarded: [])
|
|
122
158
|
edits.select do |edit|
|
|
123
159
|
next false if (edit.start...edit.end).any? { |i| Engine.marker?(cp[i]) }
|
|
124
160
|
|
|
@@ -126,7 +162,13 @@ module Polytypo
|
|
|
126
162
|
q = edit.end - 1
|
|
127
163
|
d = edit.end - edit.start
|
|
128
164
|
r = edit.replacement.length
|
|
165
|
+
next false if delimiter_neighbours && changes_delimiter_flank?(cp, p, q, edit.replacement)
|
|
166
|
+
|
|
129
167
|
span = span_containing(ranges, p)
|
|
168
|
+
if span && p == span.first && guarded[ranges.index(span)] == true &&
|
|
169
|
+
(r.zero? || edit.replacement[0] != SPACE)
|
|
170
|
+
next false
|
|
171
|
+
end
|
|
130
172
|
next true unless span && (p == span.first || q == span.last)
|
|
131
173
|
next false if r > d
|
|
132
174
|
next false if r.positive? && p == span.first && edit.replacement[0] == SPACE && cp[p] != SPACE
|
data/lib/polytypo/version.rb
CHANGED