@tabnas/markdown 0.6.1 → 0.7.1

package/src/markdown.ts

@@ -17,8 +17,10 @@
 // they are imported as such. Without `import type`, Node's type stripping
 // keeps them in the emitted import and demands runtime exports that do not
 // exist.
-import type { Tabnas, Plugin, Rule, Context } from '@tabnas/parser'
+import type { Tabnas, Plugin } from '@tabnas/parser'
 
+import { makeMdActions, makeMdLineMatcher } from './engine-block.ts'
+import { makeInlineTn, parseInlinesEngine } from './engine-inline.ts'
 import { parse } from './commonmark.ts'
 import { toAst } from './ast.ts'
 import { renderHTML } from './html.ts'
@@ -40,49 +42,16 @@ export type MarkdownOptions = {
 }
 
 // ---------------------------------------------------------------------------
-// Grammar source.
+// Grammar.
 //
-// The prose grammar is intentionally trivial: block structure is decided by
-// the line-oriented algorithm in `block.ts`, not by declarative alts, so the
-// `markdown` rule exists only to consume the token stream. The file is kept
-// because `ts/embed-grammar.js` embeds it verbatim into both runtimes; see
-// AGENTS.md. It is documentation of the entry rule, not a source of behaviour.
-
-// --- BEGIN EMBEDDED markdown-grammar.jsonic ---
-const grammarText = `
-# Markdown entry rule — CommonMark 0.31.2
-#
-# This grammar is deliberately trivial, and it is NOT where the parser
-# lives. Block structure is decided by the two-phase line algorithm of
-# spec Appendix A, implemented in block.ts / block.go, and inline
-# structure by the delimiter and bracket stacks in inline.ts / inline.go.
-# None of that is expressible as declarative alts, which is why none of
-# it is here.
-#
-# The markdown rule exists only to give the engine an entry point and to
-# consume the token stream so its trailing-content check passes. The bo
-# action reads the whole source via ctx.src() and hands it to the parser.
-# Everything the rule does is in those two lines below.
-#
-# Consequences worth knowing rather than discovering:
-#   - The railroad diagram generated from this file is empty: a bare
-#     track with no boxes on it. That is accurate, not a rendering fault.
-#   - Editing this file changes documentation, not behaviour. The one
-#     exception is the open/close alts, which really are the rule.
-#
-# The file is kept because ts/embed-grammar.js embeds it verbatim into
-# both runtimes as grammarText (see AGENTS.md). It may not contain
-# backticks: the Go copy is a raw string literal.
-{
-  rule: markdown: open: [
-    { s: '#ZZ' }
-  ]
-  rule: markdown: close: [
-    { s: '#ZZ' }
-  ]
-}
-`
-// --- END EMBEDDED markdown-grammar.jsonic ---
+// The grammar is LIVE and lives in code: the `markdown`/`line` rules below,
+// the `mdLine` matcher (engine-block.ts) and the inline instance's rule and
+// matcher set (engine-inline.ts). `debug.model()` serializes it and
+// `ts/tools/gen-railroad.mjs` draws ts/doc/grammar.{svg,txt} from a live
+// instance, so the documentation is derived from the rules the engine
+// actually parses with. The historical `markdown-grammar.jsonic` file and
+// its embed step are gone: they documented an inert single-rule bypass that
+// no longer exists (dx-report §47).
 
 // ---------------------------------------------------------------------------
 // Public parse API (engine-free)
@@ -114,44 +83,105 @@ export function parseTree(src: string, opts?: MarkdownOptions | ParserOptions):
 // ---------------------------------------------------------------------------
 // Plugin wiring
 //
-// The `markdown` rule's `bo` action reads the whole source via `ctx.src()` and
-// hands it to the parser; the alts then consume the token stream so the
-// engine's trailing-content check (lex must end at #ZZ) passes. `#AA` is the
-// ANY-token wildcard and `r:'markdown'` loops at the same depth, consuming one
-// token per iteration until only #ZZ remains.
+// The engine parse IS the parse: the `mdLine` custom matcher (engine-block.ts)
+// lexes one `#LB` token per physical line, `parse.prepare` seeds a fresh
+// `BlockParser` on `ctx.u.md`, the `line` rule feeds each matched line to the
+// shared `incorporateLine`, and the `markdown` rule's close action on `#ZZ`
+// finalizes, runs the shared inline phase, and projects the public AST into
+// the rule's node. No recognition or block decision happens here — the
+// engine owns tokenization, dispatch and state carriage; the algorithm stays
+// in the engine-free core (the anti-drift rule, dx-report §42), which is
+// what keeps this path byte-identical to `parseDocument` (asserted by the
+// differential gate and the engine-path conformance harness).
 
 const Markdown: Plugin = (tn: Tabnas, options?: MarkdownOptions) => {
   const opts = resolveOptions(options)
 
+  // Human descriptions for the block token, surfaced in railroad diagram
+  // legends (read off the live config by @tabnas/railroad).
+  tn.options({
+    config: {
+      modify: {
+        'md-tokendesc': (cfg: any) => {
+          cfg.tokenDesc = Object.assign(cfg.tokenDesc || {}, {
+            '#LB': 'one physical line (text, blank and table-arming bits in use)',
+          })
+        },
+      },
+    },
+  })
+
+  // The inline phase's own engine instance — nested parsing, one instance
+  // per plugin install, reused across every parse and every leaf block (see
+  // engine-inline.ts).
+  const inlineTn = makeInlineTn(opts)
+  const actions = makeMdActions(opts, (doc, refmap) =>
+    parseInlinesEngine(inlineTn, doc, refmap, opts),
+  )
+
   tn.options({ rule: { start: 'markdown' } })
 
-  // The parser reads `ctx.src()` directly rather than the token stream, so
-  // disable the lexers that would corrupt Markdown syntax before it gets
-  // there: backtick code spans lex as unterminated strings, `# heading` as a
-  // comment, and `1. list` as a number.
+  // Every built-in matcher is off — deliberate configuration, not defense.
+  // Markdown's block alphabet is *lines*, so the one registered matcher is
+  // the complete lexical description of this phase: `mdLine`, at an order
+  // ahead of every built-in, consuming each physical line whole. (The
+  // built-ins would misread the syntax anyway: backtick code spans lex as
+  // unterminated strings, `# heading` as a comment, `1. list` as a number.)
   tn.options({
+    space: { lex: false },
+    line: { lex: false },
     string: { lex: false },
     comment: { lex: false },
     number: { lex: false },
     value: { lex: false },
-    lex: { emptyResult: { type: 'document', children: [] } },
-  })
-
-  const refs: Record<string, any> = {
-    '@markdown-bo': (r: Rule, ctx: Context) => {
-      const src: string = ctx.src()
-      r.node = parseDocument(src, opts)
+    text: { lex: false },
+    lex: {
+      emptyResult: { type: 'document', children: [] },
+      match: { mdLine: { order: 1e5, make: makeMdLineMatcher } },
     },
-  }
+    parse: { prepare: { 'md-reset': actions.prepare } },
+  })
 
   tn.rule('markdown', (rs) => {
     return rs
       .clear()
-      .bo(refs['@markdown-bo'] as any)
-      .open([{ s: '#AA', r: 'markdown' }, {}])
-      .close([{ s: '#ZZ' }, {}])
+      .open([{ p: 'line' }])
+      .close([{ s: '#ZZ', a: actions.finish }, {}])
+  })
+
+  // One `#LB` consumed per iteration, tail-recursing via `r:` until only
+  // `#ZZ` remains; the empty alts are the fall-through that hands control
+  // back to `markdown`'s close.
+  //
+  // The first alt is the GFM extension seam: tagged `g: 'gfm'`, gated on
+  // the token's `tblArm` bit, arming the shared table probe for this line.
+  // It is injected exactly the way a downstream dialect would extend this
+  // grammar — an alt ahead of the base one, subtractable by its group tag —
+  // and `gfm: false` removes it below via `rule.exclude`, so the base
+  // dialect is the grammar minus the tagged alts, visibly (`debug.model()`).
+  tn.rule('line', (rs) => {
+    return rs
+      .clear()
+      .open([
+        {
+          s: '#LB',
+          c: (_r: any, ctx: any) => true === ctx.t0.use.tblArm,
+          r: 'line',
+          a: actions.lineGfm,
+          g: 'gfm',
+        },
+        { s: '#LB', r: 'line', a: actions.line },
+        {},
+      ])
+      .close([{}])
   })
 
+  // The GFM dialect is subtracted, not branched around: without the option,
+  // the tagged alts above simply do not exist in this instance's grammar.
+  if (true !== opts.gfm) {
+    tn.options({ rule: { exclude: 'gfm' } })
+  }
+
   // Not part of the tabnas contract, but useful for tests, docs and callers
   // that want HTML or the native tree without a second import.
   ;(tn as any).markdown = {
@@ -164,7 +194,7 @@ const Markdown: Plugin = (tn: Tabnas, options?: MarkdownOptions) => {
 
 Markdown.defaults = { gfm: true, breaks: false } as MarkdownOptions
 
-export { VERSION, Markdown, grammarText }
+export { VERSION, Markdown }
 
 export type {
   DocumentNode,
@@ -203,4 +233,4 @@ export { renderHTML } from './html.ts'
 // VERSION is this package's version. It MUST equal package.json "version":
 // the release orchestrator rewrites both, and the version test fails the
 // build if they drift. Mirrors `const VERSION` in go/markdown.go.
-const VERSION = '0.6.1'
+const VERSION = '0.7.1'