@markup-carve/carve-grammars 0.1.6 → 0.1.7

package/highlightjs/carve.js

@@ -152,7 +152,7 @@
      * @returns {object} the mode's `begin`/`end` pair, so the closer used by
      *   the guard and the closer used by the mode cannot drift apart.
      */
-    const paired = (opener, closer) => {
+    const paired = (opener, closer, { escapeAware = false, flanked = false } = {}) => {
         // The guard used to be written `(?:[^\n]|\n(?!\s*\n))*?` - unbounded,
         // lazy, and free to cross newlines. Proving there is NO closer therefore
         // cost a whole paragraph from every position, and a document made of
@@ -183,11 +183,108 @@
         const literal = lead.length === 2 ? lead[1] : lead;
         const inClass = /[\\\]^-]/.test(literal) ? '\\' + literal : literal;
         const run = `(?:[^${inClass}\\n]|\\n(?!\\s*\\n)){0,4096}`;
-        const guard = `${run}(?:${lead}(?!${rest})${run}){0,32}${closer.source}`;
-        return {
+        /*
+         * `escapeAware` opts a mode's GUARD into "an escaped delimiter is not a
+         * delimiter" (carve-grammars#385). Only the bare highlight asks for it,
+         * because it is the only closer here that a document can put a
+         * backslash in front of and mean it: `x =\= y` renders `x == y` with no
+         * mark, and this mode opened a run the engine does not.
+         *
+         * THE RUN CONSUMES AN ESCAPE AS A PAIR and refuses a bare backslash,
+         * rather than counting the backslash run in a lookbehind. Counting is
+         * what #385 shipped and what carve-grammars#390 replaces: a lookbehind
+         * that repeats a group has to be BOUNDED or it backtracks superlinearly
+         * (carve-grammars#380), and every bound is reachable - at `{0,32}` the
+         * guard stopped seeing the run at 66 backslashes and this mode refused a
+         * highlight the engine marks. A pair has no bound to reach.
+         *
+         * ONLY HALF THE FIX IS HERE. The guard proves a closer EXISTS; the span
+         * comes from `end`, which highlight.js applies on its own and which
+         * would still stop at an escaped `=`. What stops it is HIGHLIGHT listing
+         * ESCAPE in `contains` - that submode begins at the backslash, one
+         * column before the delimiter, so it wins by position and eats the pair.
+         * Neither half works alone: without this one the mode never opens,
+         * without the other it closes on the escaped delimiter.
+         *
+         * THE BOUND COUNTS ATOMS, and an escape pair is two characters, so the
+         * worst case is 8192 rather than 4096. Still a constant per position,
+         * which is all the bound is for; halving it would halve every ordinary
+         * body too.
+         *
+         * The default path is byte-identical to before, so the twelve other
+         * modes are untouched.
+         */
+        const escapedRun = `(?:\\\\[^\\n]|[^${inClass}\\n\\\\]|\\n(?!\\s*\\n)){0,4096}`;
+        const body = escapeAware ? escapedRun : run;
+        /*
+         * `flanked` says a closer may not FOLLOW WHITESPACE, which is the
+         * mirror of the `(?=\S)` every bare opener already carries
+         * (carve-grammars#392). Without it `a =b = d` scoped a highlight the
+         * engine renders literally, and so did the four other bare delimiters -
+         * one gap with five spellings. The nine BRACED modes do not ask for it
+         * and must not: the engine builds `a {*b *} d`, so a guard there would
+         * turn an over-scope into an under-scope.
+         *
+         * `end` IS LEFT ALONE, and that is forced by the engine twice over.
+         *
+         * A lookbehind ANYWHERE in a mode's `end` stops highlight.js closing
+         * the mode at all - `(?<=\S)=(?!\w)` and `=(?!\w)(?<=\S=)` both leave
+         * `x =b= y` coloured to end of line, with no error. And a CONSUMED
+         * flank, `[^\s\n]=(?!\w)`, closes correctly but starts one column
+         * earlier, which is exactly where the escape submode starts: it wins
+         * the tie and closes on the `\=` that carve-grammars#390 exists to
+         * refuse. Both measured on a two-line grammar before this was written.
+         *
+         * SO THE FALSE CLOSER IS EATEN INSTEAD OF REFUSED, the same shape #390
+         * uses for the escape. `falseCloser` begins at the WHITESPACE, one
+         * column before `end` could fire, so it wins by position and takes the
+         * delimiter as content; `end` then only ever sees a closer that stands
+         * against a non-space. Nothing about `end` changes, so this composes
+         * with the escape submode rather than fighting it.
+         *
+         * THE GUARD IS THE OTHER HALF, and it may consume, because it is inside
+         * a lookahead where nothing is really taken. Without it a run with no
+         * real closer would open, eat its false closers and colour to the end
+         * of its parent - worse than the over-scope this fixes. `notCloser`
+         * widens for the same reason: `a *b * c* d` is a bold run in the
+         * engine, and the guard has to be able to step over the middle star to
+         * see the real one.
+         */
+        /*
+         * THE FLANK CHARACTER SHARES THE BODY'S ALPHABET. A bare `[^\s\n]`
+         * would match a lone backslash, so on `x =\= y` the guard ends its body
+         * before the backslash, eats it as the flank, and proves a closer that
+         * `end` - correctly held off by the escape submode - then never fires
+         * on: the mode opens and colours to the end of its parent. An
+         * escape-aware flank is a PAIR or a non-backslash, exactly like the run.
+         */
+        // AN ESCAPED SPACE IS STILL A SPACE to the flanking rule: the engine
+        // renders `x =\\ = y` with no mark, so the pair may not stand as the
+        // non-space character the closer needs behind it.
+        const flankUnit = escapeAware ? '(?:\\\\[^\\s\\n]|[^\\s\\n\\\\])' : '[^\\s\\n]';
+        const endSource = flanked ? `${flankUnit}${closer.source}` : closer.source;
+        const notCloser = flanked
+            ? `(?:\\s${lead}|${lead}(?!${rest}))`
+            : `${lead}(?!${rest})`;
+        const guard = `${body}(?:${notCloser}${body}){0,32}${endSource}`;
+        const mode = {
             begin: new RegExp(`${opener.source}(?=${guard})`),
             end: closer,
         };
+        if (flanked) mode.contains = [{ begin: new RegExp(`\\s${closer.source}`) }];
+
+        return mode;
+    };
+
+    // Escaped characters: \* \[ etc
+    //
+    // DEFINED HERE, above the modes rather than beside HARD_BREAK where it used
+    // to sit, because HIGHLIGHT now `contains` it (carve-grammars#390) and a
+    // `const` cannot be read before its own declaration.
+    const ESCAPE = {
+        className: 'symbol',
+        begin: /\\[!"#$%&'()*+,.\/:;<=>?@\[\\\]^_`{|}~-]/,
+        relevance: 0,
     };
 
     // Forced intraword family (PART 9 S22). Content may contain the delimiter
@@ -277,14 +374,14 @@
     // (a/b, ://); the end is a closing slash not followed by word char/slash.
     const EMPHASIS = {
         className: 'emphasis',
-        ...paired(/(?<![\w:/])\/(?=\S)/, /\/(?![\w/])/),
+        ...paired(/(?<![\w:/])\/(?=\S)/, /\/(?![\w/])/, { flanked: true }),
         relevance: 0,
     };
 
     // Underline (Carve): _text_ - not in the middle of words
     const UNDERLINE = {
         className: 'emphasis',
-        ...paired(/(?<!\w)_(?!\s)/, /_(?!\w)/),
+        ...paired(/(?<!\w)_(?!\s)/, /_(?!\w)/, { flanked: true }),
         relevance: 0,
     };
 
@@ -309,17 +406,27 @@
      */
     const BOLD_ITALIC = {
         className: 'strong',
-        begin: /\/\*(?=\S)(?:[^*\n]|\*(?!\/)|\n(?!\s*\n)){1,4096}\*\//,
+        // The body is non-space at BOTH ends. It was guarded only at the
+        // opener, so `a /*b */ c` came back one combined run where the engine
+        // renders `<em>*b *</em>` - an italic run holding literal asterisks
+        // (carve-grammars#375). A fixed-width lookbehind, so the bounded
+        // repetition that keeps this rule linear is untouched.
+        begin: /\/\*(?=\S)(?:[^*\n]|\*(?!\/)|\n(?!\s*\n)){1,4096}(?<=\S)\*\//,
         relevance: 5,
     };
 
     // Strong: *text* - not in the middle of words, can contain emphasis.
     // Excludes *[ which is abbreviation-definition syntax.
+    const STRONG_PAIR = paired(/(?<!\w)\*(?![\s\[])/, /\*(?!\w)/, { flanked: true });
     const STRONG = {
         className: 'strong',
-        ...paired(/(?<!\w)\*(?![\s\[])/, /\*(?!\w)/),
+        ...STRONG_PAIR,
         relevance: 0,
-        contains: [EMPHASIS, UNDERLINE],
+        // SPREAD FIRST, then merge: `paired()` supplies a false-closer submode
+        // (carve-grammars#392) and a bare re-declaration would drop it, which
+        // is silent - the mode still opens, it just closes at the first
+        // delimiter standing after a space.
+        contains: [EMPHASIS, UNDERLINE, ...STRONG_PAIR.contains],
     };
 
     /*
@@ -351,9 +458,15 @@
      * opened and closed on the two `=` of a doubled run and scoped an empty
      * highlight over a line the engine renders literally.
      *
-     * THE CLOSER IS DELIBERATELY NOT GUARDED, and the asymmetry is the
-     * engine's: once a highlight is open the closer wins over the pattern, so
-     * `x =y z<= w` marks `y z<` and `x =y z=> w` marks `y z`.
+     * THE CLOSER IS DELIBERATELY NOT GUARDED AGAINST SMART TYPOGRAPHY, and the
+     * asymmetry is the engine's: once a highlight is open the closer wins over
+     * the pattern, so `x =y z<= w` marks `y z<` and `x =y z=> w` marks `y z`.
+     *
+     * AN ESCAPE IS A DIFFERENT QUESTION (carve-grammars#385). An escaped `=` is
+     * not a delimiter at all - `x =\= y` renders `x == y` with no mark - and
+     * this mode closed on one. `paired`'s third argument is what says so; it is
+     * the only mode here that asks for it, because it is the only closer a
+     * document can put a backslash in front of and mean it.
      *
      * ONE SHAPE IT COSTS: `<https://e.example>=hi=`, where the `>` closes an
      * autolink rather than opening a comparison. A fixed-width lookbehind
@@ -361,9 +474,16 @@
      * that trade - the ticket's own reasoning, since a false highlight claims
      * the document holds a construct it does not.
      */
+    const HIGHLIGHT_PAIR = paired(/(?<![=\w])=(?=\S)(?![>=])/, /=(?![=\w])/, { escapeAware: true, flanked: true });
     const HIGHLIGHT = {
         className: 'addition',
-        ...paired(/(?<![=\w])=(?=\S)(?![>=])/, /=(?![=\w])/),
+        ...HIGHLIGHT_PAIR,
+        // The other half of carve-grammars#390's escape fix - see `paired`.
+        // `end` would close on an escaped `=`; this submode begins one column
+        // earlier, at the backslash, and highlight.js resolves by position.
+        // The false-closer submode `paired()` supplies (carve-grammars#392) is
+        // kept alongside it: a bare `contains: [ESCAPE]` here would drop it.
+        contains: [ESCAPE, ...HIGHLIGHT_PAIR.contains],
         relevance: 3,
     };
 
@@ -375,16 +495,21 @@
     };
 
     // Delete: {-text-}
+    //
+    // THE BODY IS NOT EMPTY. `{--}` is a braced EN DASH, not an empty deletion -
+    // the engine renders `a {--} b` as `a \u2013 b` - and this mode read it as
+    // `{-` plus nothing plus `-}` (carve-grammars#378). One character is enough:
+    // `{- -}`, `{---}` and `{----}` are all deletions.
     const DELETE = {
         className: 'deletion',
-        ...paired(/\{-/, /-\}/),
+        ...paired(/\{-(?!-\})/, /-\}/),
         relevance: 5,
     };
 
     // Strikethrough (Carve): ~text~ (Djot uses ~ for subscript instead)
     const STRIKETHROUGH = {
         className: 'deletion',
-        ...paired(/(?<!\w)~(?=\S)/, /~(?!\w)/),
+        ...paired(/(?<!\w)~(?=\S)/, /~(?!\w)/, { flanked: true }),
         relevance: 2,
     };
 
@@ -642,7 +767,7 @@
     // dash alternatives: preceded by a non-space, or not followed by a word.
     const TYPOGRAPHY = {
         className: 'literal',
-        begin: /\.\.\.|<-->|<--|-->|<=>|<==|==>|<->|<-|->|(?:(?<=\S)(?:---|--)|(?:---|--)(?!\w))|!=|<=|>=|\+-|\(c\)|\(r\)|\(tm\)/,
+        begin: /\{--\}|\.\.\.|<-->|<--|-->|<=>|<==|==>|<->|<-|->|(?:(?<=\S)(?:---|--)|(?:---|--)(?!\w))|!=|<=|>=|\+-|\(c\)|\(r\)|\(tm\)/,
         relevance: 0,
     };
 
@@ -1467,13 +1592,6 @@
         relevance: 5,
     };
 
-    // Escaped characters: \* \[ etc
-    const ESCAPE = {
-        className: 'symbol',
-        begin: /\\[!"#$%&'()*+,.\/:;<=>?@\[\\\]^_`{|}~-]/,
-        relevance: 0,
-    };
-
     // Hard line break: \ at end of line
     const HARD_BREAK = {
         className: 'meta',

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@markup-carve/carve-grammars",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "description": "Grammars for the Carve markup language: Tiptap editor kit + serializer, plus Prism, highlight.js and TextMate syntax-highlighting grammars",
   "type": "module",
   "main": "tiptap/index.js",
   "scripts": {
-    "test": "npm run test:types && node tests/carve-editor-test.js && node tests/block-battery-test.js && node tests/smart-typography-test.js && node tests/highlight-opener-test.js && node tests/fenced-quote-opener-test.js && node tests/coverage-test.js && node tests/construct-ledger-test.js && node tests/opaque-payload-test.js && node tests/latest-syntax-test.js && node tests/schema-map-test.js && node tests/scans-are-bounded-test.js && node tests/braced-scan-equivalence-test.js && node tests/line-ambiguity-test.js && node tests/snapshot-test.js && node tests/roundtrip-test.js && node tests/node-producers-test.js && node tests/wire-fixtures-test.js && node tests/loss-report-test.js && node tests/authored-spelling-test.js && node tests/optional-corpus-test.js && node tests/mounted-roundtrip-test.js && node tests/serializer-test.js && node tests/tabs-roundtrip-test.js && node tests/panel-bar-test.js && node tests/parse-test.js && node tests/language-attribute-test.js && node tests/blockquote-caption-test.js && node tests/composite-figure-test.js && node tests/composite-figure-tiptap-test.js && node tests/grammar-test.js && node tests/shiki-test.js && node tests/alias-parity-test.js && node tests/engine-sweep-test.js && node tests/abbreviation-term-test.js && node tests/unclosed-delimiter-test.js && node tests/textmate-sweep-test.js && node tests/kroki-test.js && node tests/client-diagram-test.js && node tests/packaging-test.js && node tests/no-git-dependencies-test.js && node tests/engine-drift-test.js && node tests/surface-drift-test.js",
+    "test": "npm run test:types && node tests/carve-editor-test.js && node tests/block-battery-test.js && node tests/smart-typography-test.js && node tests/highlight-opener-test.js && node tests/highlight-escape-test.js && node tests/right-flank-test.js && node tests/bold-unclosed-test.js && node tests/textmate-harness-test.js && node tests/fenced-quote-opener-test.js && node tests/coverage-test.js && node tests/construct-ledger-test.js && node tests/opaque-payload-test.js && node tests/latest-syntax-test.js && node tests/schema-map-test.js && node tests/scans-are-bounded-test.js && node tests/braced-scan-equivalence-test.js && node tests/line-ambiguity-test.js && node tests/snapshot-test.js && node tests/roundtrip-test.js && node tests/node-producers-test.js && node tests/wire-fixtures-test.js && node tests/loss-report-test.js && node tests/authored-spelling-test.js && node tests/optional-corpus-test.js && node tests/mounted-roundtrip-test.js && node tests/serializer-test.js && node tests/task-state-roundtrip-test.js && node tests/tabs-roundtrip-test.js && node tests/visual-tabs-fuzz-test.js && node tests/panel-bar-test.js && node tests/parse-test.js && node tests/language-attribute-test.js && node tests/blockquote-caption-test.js && node tests/composite-figure-test.js && node tests/composite-figure-tiptap-test.js && node tests/grammar-test.js && node tests/shiki-test.js && node tests/alias-parity-test.js && node tests/engine-sweep-test.js && node tests/abbreviation-term-test.js && node tests/unclosed-delimiter-test.js && node tests/textmate-sweep-test.js && node tests/kroki-test.js && node tests/client-diagram-test.js && node tests/packaging-test.js && node tests/no-git-dependencies-test.js && node tests/engine-drift-test.js && node tests/surface-drift-test.js",
     "test:types": "tsc -p tsconfig.types.json && node tests/source-merge-test.js",
     "test:citations": "node tests/spec-citations-test.js",
     "test:coverage": "node tests/coverage-test.js",
@@ -83,7 +83,7 @@
     "./package.json": "./package.json"
   },
   "dependencies": {
-    "@markup-carve/carve": "^0.1.5",
+    "@markup-carve/carve": "^0.1.6",
     "node-diff3": "^3.2.1"
   },
   "peerDependencies": {

package/prism/carve.js

@@ -198,8 +198,56 @@
     // Shared inline emphasis/markup, referenced from block tokens that contain
     // running text (headings, list items, table cells, quotes).
     var inline = {
+        // BOTH NESTING ORDERS. The engine renders `/*both*/` and `*/both/*`
+        // identically - each is <strong><em>both</em></strong> - and only the
+        // canonical order had a branch, so the mirrored one fell through to
+        // 'bold' and came back a bold run holding two literal slashes
+        // (carve-grammars#375).
+        //
+        // The mirrored branch carries guards the canonical one must not have,
+        // and each is the engine's own asymmetry:
+        //
+        //   - `x/*b*/y` IS bold-italic and `x*/b/*y` is `x*<em>b</em>*y`,
+        //     because the mirrored opener leads with `*` and a `*` glued to a
+        //     word opens nothing.
+        //   - a `/` before the opener makes the pair ambiguous with a canonical
+        //     opener: `a /*/a/* b` is `a <em>*/a</em>* b`.
+        //   - the closer may not follow a `*`: `a */b*/* c` is
+        //     `a <strong>/b</strong>/* c`, while `a */*b/* c` IS combined - so
+        //     it is the character before the closer that decides.
+        //
+        // BOTH BODIES ARE TEMPERED, BY DIFFERENT AMOUNTS (carve-grammars#382),
+        // and the asymmetry was measured rather than chosen. The canonical body
+        // admits any `*` that does not start the closer. The MIRRORED body may
+        // not do the same: a `/` that does not start the closer is still
+        // ambiguous with a canonical opener, so its `/` is admitted only
+        // BETWEEN TWO WORD CHARACTERS, which is what lets `a */b/c/* d` read as
+        // the combined run the engine renders while `x *///* y` stays bold.
+        //
+        // The opener's `(?!\*[\s*])` is what pays for it. Over all 21,844
+        // documents `x */ body /* y` for every body of up to seven characters
+        // drawn from `* / space a`, judged by asking whether the engine renders
+        // the WHOLE body as one combined run:
+        //
+        //   strict (before)                    over 440   missed 603   wrong 1043
+        //   any non-closing slash (rejected)   over 5091  missed 161   wrong 5252
+        //   word-flanked slash alone           over 510   missed 461   wrong 971
+        //   word-flanked + opener guard (this) over 226   missed 461   wrong 687
+        //
+        // The temper alone RAISES over-colouring, and every shape it adds is
+        // one the opener guard refuses - `a */**a/a/* d` and `a */* a/a/* d`
+        // are bold runs in the engine. The guard costs nothing: `missed` is
+        // unchanged with and without it. Repeated over `* / space a .` to
+        // separate a word-character flank from a non-space one, the word-
+        // character spelling wins there too (311 wrong against 454).
+        //
+        // Both bodies are non-space at each end, and both may cross a line
+        // without crossing a blank one - corpus 208 is a combined run spanning
+        // two lines, and the highlight.js rule spells the same body. `a /*b */ c`
+        // is `<em>*b *</em>` and not a combined run; the old `[^*]+` body,
+        // guarded only at the opener, read it as one.
         'bold-italic': {
-            pattern: /\/\*(?=\S)[^*]+\*\//,
+            pattern: /\/\*(?=\S)(?:[^*\n]|\*(?!\/)|\n(?!\s*\n)){1,4096}(?<=\S)\*\/|(?<![\w*/])\*\/(?=\S)(?!\*[\s*])(?:[^/\n]|(?<=\w)\/(?=\w)|\n(?!\s*\n)){1,4096}(?<![\s*])\/\*(?!\w)/,
             alias: 'important',
         },
         // The "no leading/trailing space" rule is expressed without JS
@@ -285,13 +333,70 @@
             // the characters that changed a measurement when reverted, which
             // is why the three spellings are not identical.
             //
-            // The closer is deliberately unguarded - once a highlight is open
-            // the closer wins over the pattern in the engine too, so
-            // `x =y z<= w` marks `y z<`.
+            // AND AN ESCAPED FLANK IS NOT A FLANK (carve-grammars#380). A
+            // lookbehind sees a character, not whether it was escaped, so
+            // `a \!=b c= d` - where the `!` is literal and the `=` after it is
+            // an ordinary opener the engine marks - was refused, and
+            // `a \=b c= d`, whose `=` cannot open at all, was taken.
+            //
+            // BACKSLASHES NEST, so both halves count them rather than looking
+            // at one character: the opener needs an EVEN run behind it (`\\=`
+            // is a literal backslash and an ordinary opener) and the admission
+            // needs an ODD one in front of the flank (`\\!=` is a literal
+            // backslash and a comparison, which the engine leaves alone). A
+            // first draft looked one character back and got both of those
+            // backwards.
+            //
+            // THE PAIR REPETITION IS BOUNDED, like every other scan in this
+            // file. Written as an unbounded repetition inside a lookbehind it
+            // backtracks superlinearly over a long backslash run - measured at
+            // 7 / 30 / 91 / 346 ms for 4K / 8K / 16K / 32K backslashes, four
+            // times per doubling, and 4 / 3 / 8 / 7 ms bounded. 64 backslashes
+            // in front of a highlight opener is not a document; past the bound
+            // the run does not open, which is the safe direction.
+            //
+            // A LOOKBEHIND IS WHAT THIS RULE HAS TO USE. Moving the `escape`
+            // token in front of this one does not help: the rule is `greedy`,
+            // so Prism runs it against the ORIGINAL text from the current
+            // offset and the lookbehind still sees the flank an earlier token
+            // consumed. Measured - with `escape` moved ahead of the inline
+            // family, `a \!=b c= d` still scoped nothing. The other two
+            // grammars need none of this, because their escape rule really
+            // does consume the flank before the highlight rule is asked.
+            // THE CLOSER IS UNGUARDED AGAINST SMART TYPOGRAPHY, deliberately -
+            // once a highlight is open the closer wins over the pattern in the
+            // engine too, so `x =y z<= w` marks `y z<`. AN ESCAPE IS A
+            // DIFFERENT QUESTION (carve-grammars#385): an escaped `=` is not a
+            // delimiter at all, so `x =\= y` renders `x == y` with no mark and
+            // this rule closed on it. THE BODY CONSUMES AN ESCAPE AS A PAIR and
+            // refuses a bare backslash, which settles both directions at once -
+            // the escaped `=` is eaten before the closer can see it, and a body
+            // that could not hold one would stop there and never reach the real
+            // closer past it, so `a =b c\= d= e`, which the engine marks, would
+            // have gone from wrongly-coloured to uncoloured. The closer needs no
+            // guard of its own as a CONSEQUENCE, not a simplification: the body
+            // can only get past a backslash by taking the pair, so the position
+            // immediately before an escaped `=` is never a body end.
+            // #385 COUNTED THE RUN IN A LOOKBEHIND on the closer and its odd
+            // twin on the body, and carve-grammars#390 replaced both. Every
+            // bound is reachable: at `{0,32}` the guard stopped seeing the run
+            // at 66 backslashes and this rule refused a highlight the engine
+            // marks. A pair has no bound - and the same spelling is what the
+            // TextMate family had to use, since one TextMate engine refuses a
+            // variable-length lookbehind outright. The OPENER's two lookbehinds
+            // stay: they are the escaped-FLANK guard from carve-grammars#380,
+            // which is a different question and which this rule cannot avoid
+            // for the `greedy` reason above.
             // ONE SHAPE IT COSTS: `<https://e.example>=hi=`, where the `>`
             // closes an autolink rather than opening a comparison; a
             // fixed-width lookbehind cannot tell the two apart, and this takes
             // the under-colouring side of that trade.
+            // THE BOUND COUNTS ATOMS, and an escape pair is two characters, so
+            // the worst case is 8192 characters rather than 4096. That is still
+            // a constant per position, which is all the bound is for; measured,
+            // an all-escape body costs what a plain body of the same LENGTH
+            // costs. Halving it to keep the character count would halve every
+            // ordinary body too.
             // ONE TOKEN, TWO FORMS, so the bare `=x=` alternative shares a
             // line with the braced one. Its `[^=\n]+?` was never the defect
             // this rule was fixed for - the class already excludes its own
@@ -299,7 +404,7 @@
             // the last unbounded quantifier on a line that spells a braced
             // construct, and the derived family check below reads lines. Given
             // a bound, at the same 4096 the rest of the file uses.
-            pattern: /\{=(?=\S)[^=\n]{0,4096}(?:=(?!\})[^=\n]{0,4096}){0,32}=\}|(?<![\w=<>!])=(?=\S)(?!>)[^=\n]{1,4096}?(?<=\S)=(?![\w=])/,
+            pattern: /\{=(?=\S)[^=\n]{0,4096}(?:=(?!\})[^=\n]{0,4096}){0,32}=\}|(?:(?<=(?:^|[^\\])(?:\\\\){0,32})(?<![\w=<>!])|(?<=(?:^|[^\\])(?:\\\\){0,32}\\[<>!]))=(?=\S)(?!>)(?:\\.|[^=\n\\]){1,4096}?(?<=\S)=(?![\w=])/,
             alias: 'important',
         },
         // Braced-only: a bare `^` / `,` is literal text (no bare sup/sub).
@@ -1366,8 +1471,12 @@
             pattern: /\{\+[^+}]{0,4096}(?:\+(?!\})[^+}]{0,4096}){0,32}\+\}/,
             alias: 'inserted',
         },
+        // THE BODY IS NOT EMPTY. `{--}` is a braced EN DASH, not an empty
+        // deletion - the engine renders `a {--} b` as `a \u2013 b` - and this rule
+        // read it as `{-` plus nothing plus `-}` (carve-grammars#378). One
+        // character is enough: `{- -}`, `{---}` and `{----}` are all deletions.
         'deleted': {
-            pattern: /\{-[^\-}]{0,4096}(?:-(?!\})[^\-}]{0,4096}){0,32}-\}/,
+            pattern: /\{-(?!-\})[^\-}]{0,4096}(?:-(?!\})[^\-}]{0,4096}){0,32}-\}/,
             alias: 'deleted',
         },
         // The one `{~ ... ~}` rule the sweep did NOT flag - a substitution's two
@@ -1452,7 +1561,7 @@
         // around the dash alternatives: preceded by a non-space, or not
         // followed by a word character.
         'typography': {
-            pattern: /\.\.\.|<-->|<--|-->|<=>|<==|==>|<->|<-|->|(?:(?<=\S)(?:---|--)|(?:---|--)(?!\w))|!=|<=|>=|\+-|\(c\)|\(r\)|\(tm\)/,
+            pattern: /\{--\}|\.\.\.|<-->|<--|-->|<=>|<==|==>|<->|<-|->|(?:(?<=\S)(?:---|--)|(?:---|--)(?!\w))|!=|<=|>=|\+-|\(c\)|\(r\)|\(tm\)/,
             alias: 'constant',
         },
     };

package/textmate/carve.tmLanguage.json

@@ -1173,7 +1173,8 @@
       "name": "markup.inserted.carve"
     },
     "critic_del": {
-      "match": "\\{-[^}]*-\\}",
+      "comment": "THE BODY IS NOT EMPTY. `{--}` is a braced EN DASH, not an empty deletion - grammar.ebnf gives `braced_en_dash = \"{--}\"` and the clause is normative (markup-carve/carve#1447); carve 0.1.5 renders `a {--} b` as `a \u2013 b`. This rule read it as `{-` plus nothing plus `-}` and coloured it a deletion, which is a colour that says the document holds something it does not, and no sweep could see it because the run WAS coloured (carve-grammars#378). One character of body is enough: `{- -}` deletes a space, `{---}` deletes a hyphen and `{----}` deletes an en dash, all measured against the engine.",
+      "match": "\\{-(?!-\\})[^}]*-\\}",
       "name": "markup.deleted.carve"
     },
     "critic_sub": {
@@ -1300,22 +1301,41 @@
       }
     },
     "bold_italic": {
-      "match": "(/\\*)([^*]+)(\\*/)",
-      "captures": {
-        "1": {
-          "name": "punctuation.definition.bold-italic.carve"
-        },
-        "2": {
-          "name": "markup.bold.italic.carve"
+      "comment": "BOTH NESTING ORDERS, because the engine renders them identically: `some /*both*/ text` and `some */both/* text` are each `<strong><em>both</em></strong>` (carve 0.1.5). Only the canonical order had a rule, so the mirrored one fell through to #strong and came back a bold run containing two literal slashes - a reading that says the document holds something it does not, and one no sweep could see because the row sampled the order this rule got right (carve-grammars#375). THE MIRRORED OPENER NEEDS A FLANKING GUARD AND THE CANONICAL ONE DOES NOT, and that asymmetry is the engine's: `x/*b*/y` IS bold-italic and `x*/b/*y` is `x*<em>b</em>*y`, because the mirrored opener leads with `*` and a `*` glued to a word opens nothing. Same at the other end: `x/*b*/y` closes into a word and `a */b/*y c` does not. CONTENT IS NON-SPACE AT BOTH ENDS in both orders. `a /* b */ c` is `<em>* b *</em>` and `a /*b */ c` is `<em>*b *</em>` - italic runs holding literal asterisks, not combined runs - and this rule read both as one combined run until the two guards arrived with the mirrored order. WHAT NEITHER ORDER COVERS: a body holding the OTHER order's delimiter. The engine reads `a /*b*c*/ d` and `a */b/c/* d` as combined runs and both branches decline them, which is the under-colouring side of the trade on the mirrored side - carve-grammars#382. BOTH BODIES ARE TEMPERED, BY DIFFERENT AMOUNTS, and the asymmetry was measured rather than chosen (carve-grammars#382). The canonical body admits any `*` that does not start the closer, which is what lets `a /*b*c*/ d` read as the combined run the engine renders. The MIRRORED body may not do the same: a `/` that does not start the closer is still ambiguous with a canonical opener, so its `/` is admitted only BETWEEN TWO WORD CHARACTERS - `a */b/c/* d` is claimed and `x *///* y` stays bold. The opener's `(?!\\*[\\s*])` is what pays for it. Over all 21,844 documents `x */ body /* y` for every body of up to seven characters drawn from `* / space a`, judged by asking whether the engine renders the WHOLE body as one combined run: strict was 440 over-coloured and 603 missed; admitting any non-closing slash is 5,091 and 161; the word-flanked slash alone is 510 and 461; the word-flanked slash with the opener guard is 226 and 461. The temper alone RAISES over-colouring and every shape it adds is one the opener guard refuses - `a */**a/a/* d` and `a */* a/a/* d` are bold runs in the engine - while the guard itself costs nothing, since `missed` is unchanged with and without it. TWO MORE GUARDS THE MIRRORED BRANCH NEEDS AND THE CANONICAL ONE DOES NOT, both found by probing the shapes where one order's delimiters sit inside the other's. A `/` before the opener makes the pair ambiguous with a canonical opener - `a /*/a/* b` is `a <em>*/a</em>* b`, an italic run - so the left guard refuses it as well as a word character and a `*`. And the closer may not follow a `*`: `a */b*/* c` is `a <strong>/b</strong>/* c` and `a */*a*/* b` is `a <strong>/*a</strong>/* b`, both bold runs, while `a */*b/* c` IS combined - so it is the character before the closer that decides, not the one after the opener.",
+      "patterns": [
+        {
+          "match": "(/\\*)(?=\\S)((?:[^*\\n]|\\*(?!/)){1,4096})(?<=\\S)(\\*/)",
+          "captures": {
+          "1": {
+            "name": "punctuation.definition.bold-italic.carve"
+          },
+          "2": {
+            "name": "markup.bold.italic.carve"
+          },
+          "3": {
+            "name": "punctuation.definition.bold-italic.carve"
+          }
+        }
         },
-        "3": {
-          "name": "punctuation.definition.bold-italic.carve"
+        {
+          "match": "(?<![\\w*/])(\\*/)(?=\\S)(?!\\*[\\s*])((?:[^/\\n]|(?<=\\w)/(?=\\w))+)(?<![\\s*])(/\\*)(?!\\w)",
+          "captures": {
+          "1": {
+            "name": "punctuation.definition.bold-italic.carve"
+          },
+          "2": {
+            "name": "markup.bold.italic.carve"
+          },
+          "3": {
+            "name": "punctuation.definition.bold-italic.carve"
+          }
         }
-      }
+        }
+      ]
     },
     "strong": {
       "begin": "(?<![\\w*])(\\*)(?=\\S)",
-      "end": "(?<=\\S)(\\*)(?![\\w*])",
+      "end": "(?<=\\S)(\\*)(?![\\w*])|^[ \\t]*$",
       "beginCaptures": {
         "1": {
           "name": "punctuation.definition.bold.carve"
@@ -1326,7 +1346,7 @@
           "name": "punctuation.definition.bold.carve"
         }
       },
-      "contentName": "markup.bold.carve",
+      "comment": "A BLANK LINE ENDS THE RUN, because an opener with no closer must not colour the rest of the file (carve-grammars#393). This is the ONLY bare inline rule spelled begin/end rather than as one `match`, and it has to be: a TextMate `match` cannot cross a line break at all - measured on a two-rule grammar - while the engine does read `a *b` over `c* d` as one bold run, which this rule gets right and its four `match` siblings get wrong. So the two properties trade off, and the trade is what the blank line bounds: without it `a *b c` scoped the next paragraph and the heading after it, and now it stops at the paragraph. WHAT IT DOES NOT FIX: a genuinely unclosed opener still colours to the end of its own paragraph, because `begin` cannot see whether a closer exists on a later line and TextMate offers no way to ask. Pinned in tests/bold-unclosed-test.js rather than left silent. MEASURE THIS RULE WITH THE RAW TOKENIZER: Shiki reports the fix as having no effect at all (carve-grammars#395), so a test written against it would call a correct grammar broken.", "contentName": "markup.bold.carve",
       "patterns": [
         {
           "include": "#braced_comment"
@@ -1334,6 +1354,7 @@
       ]
     },
     "italic": {
+      "comment": "ONE `match` AND NOT `begin`/`end`, DELIBERATELY, and the trade was measured rather than assumed (carve-grammars#397). A TextMate `match` cannot cross a line break, so this rule does not see `a /b` over `c/ d` as one run the way the engine does, and it cannot hold a delimiter standing after a space either. `begin`/`end` buys both - and opens eagerly, because `begin` cannot see whether a closer exists on a later line, so an opener with no closer colours to the end of its paragraph the way `strong` still does. Converted and swept over the 1,545-document corpus, all four bare rules at once: documents holding a run this grammar MISSES fell from 13 to 9, and documents where it claims a run the document does NOT hold rose from 3 to 35. The whole corpus contains two runs that span a line break. A false run is the worse direction - carve-grammars#324 and #325 both turned on that - so the `match` spelling stays and the missed multi-line run is the price. Narrowing the openers first (`(?<![\\\\w:/])` for the URL, `(?!_)` for the dunder, `(?!>)` for the arrow) took the false count from 40 to 35 and no further.",
       "match": "(?<![\\w/])(/)(?=\\S)([^/\\n]+?)(?<=\\S)(/)(?![\\w/])",
       "captures": {
         "1": {
@@ -1348,6 +1369,7 @@
       }
     },
     "underline": {
+      "comment": "ONE `match` AND NOT `begin`/`end`, deliberately: converting all four bare rules took corpus documents carrying a FALSE run from 3 to 35 to recover 4 missed ones. The measurement and the reasoning are on `italic` (carve-grammars#397).",
       "match": "(?<![\\w_])(_)(?=\\S)([^_\\n]+?)(?<=\\S)(_)(?![\\w_])",
       "captures": {
         "1": {
@@ -1362,6 +1384,7 @@
       }
     },
     "strike": {
+      "comment": "ONE `match` AND NOT `begin`/`end`, deliberately: converting all four bare rules took corpus documents carrying a FALSE run from 3 to 35 to recover 4 missed ones. The measurement and the reasoning are on `italic` (carve-grammars#397).",
       "match": "(?<![\\w~])(~)(?=\\S)([^~\\n]+?)(?<=\\S)(~)(?![\\w~])",
       "captures": {
         "1": {
@@ -1376,7 +1399,7 @@
       }
     },
     "highlight": {
-      "comment": "A BARE `=` THAT BEGINS OR ENDS A SMART-TYPOGRAPHY PATTERN IS NOT A HIGHLIGHT OPENER (grammar.ebnf, Inline parsing precedence: \"a delimiter that begins a multi-char smart-typography pattern: `=>` is the arrow, never a highlight opener - the pattern is consumed first\"). Corpus 386's third paragraph is the shape it costs: `Not an arrow: key => value stays literal, and p <= q is a comparison.` renders with NO mark, and the opener `=` of `=>` reaching the closer `=` of `<=` scoped the whole 68-character sentence as a highlight (carve-grammars#325). That is worse than leaving a run uncoloured, because the output then claims the document holds a construct it does not - the same reason carve-grammars#324 gave for the arrows. THE GUARD IS ONLY WHAT THIS GRAMMAR MEASURABLY NEEDS. The opener may not be FOLLOWED by `>`, which is `=>` and the tail of `==>`; and it may not FOLLOW `!`, which is the `!=` comparison. It needs no guard against `<` or `>` behind it, and carried one until each character was reverted and measured: vscode-textmate resolves by POSITION, so on `a <=b c= d` the typography rule's `<=` starts one column before the highlight rule's `=` and takes it. `!=` is the comparison that rule does NOT carry (its alternation is arrows, dashes and `<=`/`>=`), which is why `!` is the one character here that changes a measurement. A leading `=` needs no guard either - the content class already refuses one. THE CLOSER IS DELIBERATELY NOT GUARDED, and the asymmetry is the engine's: once a highlight is OPEN the closer wins over the pattern, so `x =y z<= w` marks `y z<` and `x =y z=> w` marks `y z`. Measured against the pinned engine, both. ONE SHAPE IT COSTS: an autolink or a cross-reference glued straight to a highlight (`<https://e.example>=hi=`) really does mark, because there the `>` closes the link rather than opening a comparison - a flat token map cannot tell the two apart within a fixed-width lookbehind, and this takes the under-colouring side of that trade on the ticket's own reasoning.",
+      "comment": "A BARE `=` THAT BEGINS OR ENDS A SMART-TYPOGRAPHY PATTERN IS NOT A HIGHLIGHT OPENER (grammar.ebnf, Inline parsing precedence: \"a delimiter that begins a multi-char smart-typography pattern: `=>` is the arrow, never a highlight opener - the pattern is consumed first\"). Corpus 386's third paragraph is the shape it costs: `Not an arrow: key => value stays literal, and p <= q is a comparison.` renders with NO mark, and the opener `=` of `=>` reaching the closer `=` of `<=` scoped the whole 68-character sentence as a highlight (carve-grammars#325). That is worse than leaving a run uncoloured, because the output then claims the document holds a construct it does not - the same reason carve-grammars#324 gave for the arrows. THE GUARD IS ONLY WHAT THIS GRAMMAR MEASURABLY NEEDS. The opener may not be FOLLOWED by `>`, which is `=>` and the tail of `==>`. It needs no guard against `<`, `>` or `!` behind it, and carried one for each until it was reverted and measured: vscode-textmate resolves by POSITION, so on `a <=b c= d` the typography rule's `<=` starts one column before the highlight rule's `=` and takes it. `!` was the last of the three to go, and it went in carve-grammars#374 rather than with the other two: while the typography alternation carried no `!=`, nothing reached `a !=b c= d` before this rule and removing the guard scoped `b c` as a highlight. The alternation carries `!=` now, so the comparison starts one column early like the other two and the guard stops changing any answer. What holds the shape is the pair that measures it - the `!=` row in tests/lib/constructs.js and `a !=b c= d` in tests/highlight-opener-test.js - rather than a lookbehind nobody can see working. Removing it also stopped an under-colouring a lookbehind cannot avoid: `a \\!=b c= d` escapes the `!`, so the `=` after it is an ordinary opener and the engine marks `b c`; this grammar refused it while the guard was here. The Prism grammar still does, and still needs its own guard because Prism applies tokens IN ORDER rather than by position - carve-grammars#380. A leading `=` needs no guard either - the content class already refuses one. THE CLOSER IS DELIBERATELY NOT GUARDED AGAINST SMART TYPOGRAPHY, and the asymmetry is the engine's: once a highlight is OPEN the closer wins over the pattern, so `x =y z<= w` marks `y z<` and `x =y z=> w` marks `y z`. Measured against the pinned engine, both. AN ESCAPE IS A DIFFERENT QUESTION (carve-grammars#385): an escaped `=` is not a delimiter at all, so `x =\\= y` renders `x == y` with no mark and this rule closed on it. THE BODY CONSUMES AN ESCAPE AS A PAIR (`\\\\.`) and refuses a bare backslash, which settles both directions at once - the escaped `=` is eaten before the closer can see it, and a body that could not hold one would stop there and never reach the real closer past it, so `a =b c\\= d= e`, which the engine marks, would have gone from wrongly-coloured to uncoloured. The closer needs no guard of its own, and that is a consequence rather than a simplification: the body can only get past a backslash by taking the pair, so the position immediately before an escaped `=` is never a body end. #385 COUNTED THE BACKSLASH RUN IN A LOOKBEHIND INSTEAD, and carve-grammars#390 replaced it for two measured reasons. The IDE's TextMate engine refuses a variable-length lookbehind by not matching ANYTHING - no error, the rule simply goes inert and `x =b= y` loses its mark too (markup-carve/intellij-carve#117), which made this file unusable in a TextMate engine Carve ships a plugin for. And every bound is reachable: at `{0,32}` the guard stopped seeing the run at 66 backslashes and refused a highlight the engine marks. A pair has no bound. THREE SHAPES STAY WRONG, pinned in tests/highlight-escape-test.js - `x =<== y`, `x =!== y` and `x =a== y`, a closer followed by another `=`, which the engine marks and `(?![=\\w])` refuses; widening that guard means letting the body hold its own delimiter. ONE SHAPE IT COSTS: an autolink or a cross-reference glued straight to a highlight (`<https://e.example>=hi=`) really does mark, because there the `>` closes the link rather than opening a comparison - a flat token map cannot tell the two apart within a fixed-width lookbehind, and this takes the under-colouring side of that trade on the ticket's own reasoning. ONE `match` AND NOT `begin`/`end`, deliberately: converting all four bare rules took corpus documents carrying a FALSE run from 3 to 35 to recover 4 missed ones. The measurement and the reasoning are on `italic` (carve-grammars#397).",
       "patterns": [
         {
           "match": "(\\{=)(?=\\S)([^\\n]*?)(=\\})",
@@ -1393,7 +1416,7 @@
           }
         },
         {
-          "match": "(?<![=!\\w])(=)(?=\\S)(?!>)([^=\\n]+?)(?<=\\S)(=)(?![=\\w])",
+          "match": "(?<![=\\w])(=)(?=\\S)(?!>)((?:\\\\.|[^=\\n\\\\])+?)(?<=\\S)(=)(?![=\\w])",
           "captures": {
             "1": {
               "name": "punctuation.definition.highlight.carve"
@@ -1510,7 +1533,7 @@
       }
     },
     "smart_typography": {
-      "match": "(<-->|<--|-->|<=>|<==|==>|---|--|\\.\\.\\.|<->|<-|->|<=|>=)",
+      "match": "(\\{--\\}|<-->|<--|-->|<=>|<==|==>|---|--|\\.\\.\\.|<->|<-|->|<=|>=|!=|\\+-|\\(c\\)|\\(r\\)|\\(tm\\))",
       "name": "constant.other.smart_typography.carve"
     },
     "math_inline": {

package/tiptap/carve-kit.js

@@ -587,6 +587,19 @@ export const CarveKit = Extension.create({
                                 'data-checked': attributes.checked,
                             }),
                         },
+                        // The four non-space task states (`-` `_` `>` `?`)
+                        // the engine emits as `data-task-state`. Null for a
+                        // plain `[ ]`/`[x]` item, whose state `checked` holds.
+                        // Without this, a load/save cycle collapsed every such
+                        // item back to `[ ]` (markup-carve/carve-grammars#371).
+                        carveTaskState: {
+                            default: null,
+                            keepOnSplit: false,
+                            parseHTML: element => element.getAttribute('data-task-state') || null,
+                            renderHTML: attributes => (
+                                attributes.carveTaskState ? { 'data-task-state': attributes.carveTaskState } : {}
+                            ),
+                        },
                         // A task item takes a marker attribute the same way a
                         // plain item does: `-{.c} [ ] text`.
                         id: { default: null },

package/tiptap/carve-to-pm.js

@@ -452,7 +452,14 @@ function convertBlock(node, ctx) {
                     ...(typeof node.tight === 'boolean' ? { attrs: { carveTight: node.tight } } : {}),
                     content: (node.items || []).map((it) => ({
                         type: 'taskItem',
-                        attrs: { checked: !!it.checked, ...(convertAttrs(it.attrs) || {}) },
+                        attrs: {
+                            checked: !!it.checked,
+                            // The engine carries the four non-space task states
+                            // (`-` `_` `>` `?`) in `taskState`; absent for the
+                            // plain `[ ]`/`[x]`, whose state `checked` already holds.
+                            ...(it.taskState != null ? { carveTaskState: it.taskState } : {}),
+                            ...(convertAttrs(it.attrs) || {}),
+                        },
                         content: convertBlocks(it.children || [], ctx),
                     })),
                 };
@@ -689,7 +696,9 @@ function convertAdmonition(node, ctx) {
             type: 'carveTab',
             attrs: {
                 label: node.label ?? inlinePlainText(node.title || []),
-                selected: !!node.attrs?.keyValues?.selected,
+                // Bare attributes are represented with an empty-string value,
+                // so truthiness would turn `{selected}` back off.
+                selected: Object.hasOwn(node.attrs?.keyValues ?? {}, 'selected'),
             },
             content: convertBlocks(node.children || [], ctx),
         };

package/tiptap/schema-map.json

@@ -92,6 +92,7 @@
         "carveAttrOrder": "the order the run's slots were WRITTEN in, as the AST records it: `#id`, `.class`, and each key by name",
         "carveKeyValues": "authored marker key=values",
         "checked": "task item state, taskItem only",
+        "carveTaskState": "non-space task marker (`-` `_` `>` `?`), taskItem only; null for a plain `[ ]`/`[x]` item",
         "class": "authored marker classes",
         "id": "authored marker id"
       }

package/tiptap/serializer.js

@@ -293,7 +293,13 @@ export function serializeToCarve(doc) {
                         num++;
                     } else if (node.type === 'taskList') {
                         marker = '-';
-                        taskBox = '[' + (item.attrs?.checked ? 'x' : ' ') + '] ';
+                        // `checked` wins: toggling the stock TaskItem checkbox
+                        // updates only `checked` and leaves `carveTaskState`
+                        // intact, so a user who checks a `[-]` item means `[x]`.
+                        // An unchecked item keeps its non-space state, or a
+                        // plain space when it has none.
+                        const taskState = item.attrs?.checked ? 'x' : (item.attrs?.carveTaskState || ' ');
+                        taskBox = '[' + taskState + '] ';
                     } else {
                         marker = '-';
                     }
@@ -301,7 +307,7 @@ export function serializeToCarve(doc) {
                     // (`- {.c} item`) is a different document: the brace is then
                     // content, either literal text or a block-attribute line for
                     // what follows (corpus 90-list-item-attributes-7, 172).
-                    const markerAttrs = serializeAttributes(item.attrs, ['checked']);
+                    const markerAttrs = serializeAttributes(item.attrs, ['checked', 'carveTaskState']);
                     const prefix = marker + markerAttrs + ' ';
                     output += indent + prefix + taskBox;
                     // The content column is measured from the prefix ACTUALLY