grammatik 0.1.0 → 0.1.1

package/src/codegen.js

@@ -2,7 +2,7 @@
 'use strict'
 
 import { FAIL, expect } from './state.js'
-import { buildDispatch, opOf, recognizeSeqIdiom } from './analyze.js'
+import { buildDispatch, opOf, recognizeSeqIdiom, recognizeAltPrefixIdiom } from './analyze.js'
 
 /**
  * A real source generator for grammars.
@@ -23,10 +23,23 @@ import { buildDispatch, opOf, recognizeSeqIdiom } from './analyze.js'
 
 /**
  * @param {{ planner: any, rules: Record<string, any>, firsts: Map<string, any>|null,
- *           memo: boolean, maxSteps: number }} config
+ *           memo: boolean, maxSteps: number, tokenIds?: Map<string, number>|null }} config
  */
 export function createCodegen (config) {
-  const { planner, rules, firsts, memo, maxSteps } = config
+  const { planner, rules, firsts, memo, maxSteps, tokenIds = null } = config
+
+  /**
+   * `token.id === <n>` is an integer compare instead of a string compare, and
+   * is what generated code prefers wherever a token's declared type matters.
+   * Falls back to `token.type === <name>` only for a hand-built token that
+   * bypassed this parser's own lexer (`parseTokens` accepts anything shaped
+   * like a Token) — every token the bundled lexer produces has an id.
+   */
+  const typeCheck = (tokenVar, name) => {
+    const id = tokenIds ? tokenIds.get(name) : undefined
+    if (id === undefined) return `${tokenVar}.type === ${JSON.stringify(name)}`
+    return `(${tokenVar}.id === ${id} || (${tokenVar}.id === undefined && ${tokenVar}.type === ${JSON.stringify(name)}))`
+  }
 
   /** Runtime values the generated code closes over. */
   const HELPERS = []
@@ -99,7 +112,7 @@ export function createCodegen (config) {
         out.push(
           `let ${result};`,
           `const ${token} = c.tokens[c.idx];`,
-          `if (${token} !== undefined && ${token}.type === ${JSON.stringify(name)}) { c.idx++; ${result} = ${token}; }`,
+          `if (${token} !== undefined && ${typeCheck(token, name)}) { c.idx++; ${result} = ${token}; }`,
           `else { E(c, ${JSON.stringify(name)}); ${result} = F; }`
         )
         return result
@@ -212,7 +225,7 @@ export function createCodegen (config) {
       case 'oneOf': {
         const names = Array.isArray(arg) ? arg : [arg]
         const token = nextId()
-        const test = names.map((n) => `${token}.type === ${JSON.stringify(n)}`).join(' || ')
+        const test = names.map((n) => typeCheck(token, n)).join(' || ')
         out.push(
           `let ${result};`,
           `const ${token} = c.tokens[c.idx];`,
@@ -520,29 +533,75 @@ export function createCodegen (config) {
     return result
   }
 
+  /**
+   * Emits an inline ordered-backtracking block over `list`, into a fresh
+   * variable. Used both for a whole `alt` with no dispatch table, and for the
+   * residual subset of branches a dispatch table cannot separate by one
+   * token.
+   */
+  function genOrderedBacktrack (list, out, bindingsVar) {
+    const result = nextId()
+    const label = 'L' + nextId()
+    const start = nextId()
+    out.push(`let ${result};`, `${label}: {`, `const ${start} = c.idx;`)
+    for (const branch of list) {
+      out.push('{')
+      const value = gen(branch, out, bindingsVar)
+      out.push(`if (${value} !== F) { ${result} = ${value}; break ${label}; }`, `c.idx = ${start};`, '}')
+    }
+    out.push(`${result} = F;`, '}')
+    return result
+  }
+
   function genAlt (branches, out, result, bindingsVar) {
     if (!Array.isArray(branches)) throw new Error("'alt' expects an array of branches")
 
+    const factored = recognizeAltPrefixIdiom(branches)
+    if (factored !== null) return gen(factored, out, bindingsVar)
+
     const table = firsts ? buildDispatch(branches, firsts) : null
     const label = 'L' + nextId()
 
     if (table !== null) {
       // A switch on the token type: V8 turns a dense string switch into a
       // hash lookup with inlined bodies, and every branch is inlined here.
+      // A table value that is an array is a residual group — tokens shared by
+      // more than one branch — resolved by ordered backtracking over just
+      // that subset instead of a single inlined branch.
+      //
+      // Switching on token.id (an integer) instead of token.type (a string)
+      // when every expected token resolves to one — a numeric switch is a jump
+      // table without even the string-hashing step a dense string switch still
+      // does. `I[token.type]` is the same fallback typeCheck uses: a hand-built
+      // token from parseTokens has no id, and I is a name -> id table built
+      // once from the same tokenIds this parser resolved every other id from.
       const expected = [...table.keys()]
+      const expectedIds = tokenIds ? expected.map((t) => tokenIds.get(t)) : null
+      const numeric = expectedIds !== null && expectedIds.every((id) => id !== undefined)
       const token = nextId()
       out.push(tickSrc, `let ${result};`, `${label}: {`, `const ${token} = c.tokens[c.idx];`)
       out.push(`if (${token} === undefined) { ${expected.map((t) => `E(c, ${JSON.stringify(t)});`).join(' ')} ${result} = F; break ${label}; }`)
-      out.push(`switch (${token}.type) {`)
+
+      let discriminant
+      if (numeric) {
+        discriminant = nextId()
+        out.push(`const ${discriminant} = ${token}.id !== undefined ? ${token}.id : I[${token}.type];`)
+        out.push(`switch (${discriminant}) {`)
+      } else {
+        discriminant = `${token}.type`
+        out.push(`switch (${discriminant}) {`)
+      }
 
       const byBranch = new Map()
-      for (const [tokenName, branch] of table) {
+      for (const tokenName of expected) {
+        const branch = table.get(tokenName)
         if (!byBranch.has(branch)) byBranch.set(branch, [])
         byBranch.get(branch).push(tokenName)
       }
+      const caseLabel = (t) => numeric ? tokenIds.get(t) : JSON.stringify(t)
       for (const [branch, tokenNames] of byBranch) {
-        out.push(tokenNames.map((t) => `case ${JSON.stringify(t)}:`).join(' ') + ' {')
-        const value = gen(branch, out, bindingsVar)
+        out.push(tokenNames.map((t) => `case ${caseLabel(t)}:`).join(' ') + ' {')
+        const value = Array.isArray(branch) ? genOrderedBacktrack(branch, out, bindingsVar) : gen(branch, out, bindingsVar)
         out.push(`${result} = ${value}; break ${label};`, '}')
       }
       out.push(
@@ -554,17 +613,13 @@ export function createCodegen (config) {
     }
 
     // Ordered backtracking, still fully inlined.
-    const start = nextId()
-    out.push(tickSrc, `let ${result};`, `${label}: {`, `const ${start} = c.idx;`)
-    for (const branch of branches) {
-      out.push('{')
-      const value = gen(branch, out, bindingsVar)
-      out.push(`if (${value} !== F) { ${result} = ${value}; break ${label}; }`, `c.idx = ${start};`, '}')
-    }
-    out.push(`${result} = F;`, '}')
-    return result
+    out.push(tickSrc)
+    return genOrderedBacktrack(branches, out, bindingsVar)
   }
 
+  /** Name -> id, for a generated switch's runtime fallback when token.id is missing. */
+  const idByName = tokenIds ? Object.fromEntries(tokenIds) : null
+
   /** Generates and installs the function for one rule. */
   function buildRule (name) {
     const out = []
@@ -576,8 +631,8 @@ export function createCodegen (config) {
     SOURCES[ruleIndex.get(name)] = body
     const source = `return ${body}`
     // eslint-disable-next-line no-new-func
-    const factory = new Function('F', 'E', 'P', 'T', 'R', 'H', source)
-    return factory(FAIL, expect, progressError, stepsError, RULES, HELPERS)
+    const factory = new Function('F', 'E', 'P', 'T', 'R', 'H', 'I', source)
+    return factory(FAIL, expect, progressError, stepsError, RULES, HELPERS, idByName)
   }
 
   function progressError (op, at) {

package/src/emit.js

@@ -139,9 +139,13 @@ export function emitModule (spec, options = {}) {
   // The planner is needed only so codegen can record what each helper is; the
   // closures it makes are thrown away.
   const firsts = ll1 ? analysis.firsts : null
+  // The generated file's own `createLexer(tokenDefs, ...)` call assigns the same
+  // ids, from the same `tokenDefs` array in the same order — see the matching
+  // comment in parser.js.
+  const tokenIds = new Map(lexer.tokenNames.map((name, id) => [name, id]))
   const engine = { build: () => { throw new Error('not compiled during emit') }, run: () => {} }
-  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval: false })
-  const codegen = createCodegen({ planner, rules, firsts, memo, maxSteps })
+  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval: false, tokenIds })
+  const codegen = createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds })
 
   codegen.compile(start)
 
@@ -335,6 +339,14 @@ const H = [
 ${helpers.map((source, index) => `  /* ${index} */ ${source}`).join(',\n')}
 ]
 
+/**
+ * Name -> id, the same numbering \`lexer\` assigns. A generated \`alt\`'s switch
+ * dispatches on a token's id when it has one; this is only read as a fallback,
+ * for a hand-built token from \`parseTokens\` that bypassed this file's own
+ * lexer and so has no id of its own.
+ */
+const I = ${literal(Object.fromEntries(tokenIds))}
+
 ${ruleSources}
 
 const RULES = ${literal(Object.fromEntries(reached.map(([name, index]) => [name, index])))}

package/src/lexer.js

@@ -24,11 +24,19 @@ import { firstCharSet } from './firstchars.js'
 /**
  * @typedef {object} Token
  * @property {string} type
+ * @property {number} [id]   The token definition's declaration index, present
+ *                           in `offset` position mode only — see the note on
+ *                           `positions` below. Not part of the public contract
+ *                           of a token — internal combinators use it as a
+ *                           faster stand-in for `type` (an integer compare and
+ *                           an array index instead of a string compare and a
+ *                           property lookup); `type` is what a grammar
+ *                           author's code should read.
  * @property {string} image
  * @property {number} start
  * @property {number} end
- * @property {number} line
- * @property {number} col
+ * @property {number} [line] Present in `full` position mode only.
+ * @property {number} [col]  Present in `full` position mode only.
  */
 
 export class LexError extends Error {
@@ -76,9 +84,13 @@ function modesOf (def) {
  * @param {TokenDef[]} defs
  * @param {{ positions?: 'full' | 'offset' }} [options]
  *   `full` (default) records line and column on every token. `offset` records
- *   only `start`/`end`, which skips a scan of the entire input for newlines —
- *   worth taking when errors do not need to name a line, or when positions are
- *   resolved lazily from the offset afterwards.
+ *   only `start`/`end` plus `id`, which skips a scan of the entire input for
+ *   newlines — worth taking when errors do not need to name a line, or when
+ *   positions are resolved lazily from the offset afterwards. The two modes
+ *   deliberately differ in which extra field they carry rather than both
+ *   carrying every field: a 7th own property on the token object, in either
+ *   mode, measured at a real ~30% tokenize regression (see the comment at the
+ *   push site) that a 6-field object does not pay.
  * @returns {{ tokenize: (text: string) => Token[], tokenNames: string[], positions: string, modeNames: string[] }}
  */
 export function createLexer (defs, options = {}) {
@@ -109,7 +121,7 @@ export function createLexer (defs, options = {}) {
   }
   for (const def of defs) for (const mode of modesOf(def)) idOf(mode)
 
-  const compiled = defs.map((def) => {
+  const compiled = defs.map((def, id) => {
     if (!def.name) throw new Error('createLexer: every token definition needs a name')
     if (!def.pattern && !def.literal) throw new Error(`createLexer: token '${def.name}' needs a pattern or a literal`)
     if (def.pattern && def.literal) throw new Error(`createLexer: token '${def.name}' has both a pattern and a literal; pick one`)
@@ -128,6 +140,11 @@ export function createLexer (defs, options = {}) {
 
     return {
       name: def.name,
+      // The declaration index. `createParser` derives the identical mapping
+      // from `tokenNames` independently (same array, same order), so a
+      // combinator that resolves a `consume` target's id at build time agrees
+      // with what a token actually carries without the two ever comparing notes.
+      id,
       skip: !!def.skip,
       literal: def.literal,
       longerAlt: def.longerAlt || null,
@@ -283,16 +300,21 @@ export function createLexer (defs, options = {}) {
           const image = winner.literal !== undefined
             ? winner.literal
             : (winner === def && matched !== null ? matched : text.slice(offset, end))
-          // One object shape either way, so the parser's property loads stay
-          // monomorphic regardless of the position mode.
-          tokens.push({
-            type: winner.name,
-            image,
-            start: offset,
-            end,
-            line: trackLines ? line : 0,
-            col: trackLines ? offset - lineStart + 1 : 0
-          })
+          // Two shapes, not one: a 7th own property on this literal — in
+          // either position mode — measured at a genuine ~30% tokenize
+          // regression in this exact function, reproducible and isolated to
+          // the field count alone (a constant 7th field regresses exactly as
+          // much as a computed one), but one that does not reduce to a
+          // minimal repro elsewhere. Each mode instead stays at 6 own
+          // properties: `id` only in offset mode, where line/col would have
+          // been dead weight (always 0) anyway; line/col only in full mode,
+          // where `id` is what gets dropped. `token.id === undefined` is
+          // already how consume/oneOf/alt's dispatch fall back to comparing
+          // `type` for a hand-built token from parseTokens, so a full-mode
+          // token needs no special casing there — it takes the same fallback.
+          tokens.push(trackLines
+            ? { type: winner.name, image, start: offset, end, line, col: offset - lineStart + 1 }
+            : { type: winner.name, id: winner.id, image, start: offset, end })
         }
 
         // A bounded scan over this token's own span, so the whole tokenize costs

package/src/parser.js

@@ -114,9 +114,16 @@ export function createParser (spec, options = {}) {
   const startRule = rules[start]
   const firsts = ll1 ? analysis.firsts : null
 
+  // Declaration index of each token name, the same numbering `createLexer`
+  // assigns to the tokens it produces (both derive it from the same array in
+  // the same order). `consume` compares this instead of the string `type`
+  // where it can — an integer compare instead of a string compare, on the
+  // hottest path in the parser.
+  const tokenIds = new Map(lexer.tokenNames.map((name, id) => [name, id]))
+
   // The planner specializes the grammar into closures; the combinators'
   // `compile` hooks reach it through the engine.
-  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval })
+  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval, tokenIds })
   engine.__jlGrammar = { planner }
 
   // Source generation emits one JS function per rule with every node inlined,
@@ -125,7 +132,7 @@ export function createParser (spec, options = {}) {
   // lands on the generated code rather than a method dispatch.
   // Code generation needs `new Function`, so it is off whenever eval is.
   const codegen = compile && unsafeEval
-    ? createCodegen({ planner, rules, firsts, memo, maxSteps })
+    ? createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds })
     : null
   const compiledStart = codegen && codegen.compile(start)
 

package/src/plan.js

@@ -2,7 +2,7 @@
 'use strict'
 
 import { FAIL, expect } from './state.js'
-import { buildDispatch, opOf, recognizeSeqIdiom } from './analyze.js'
+import { buildDispatch, opOf, recognizeSeqIdiom, recognizeAltPrefixIdiom } from './analyze.js'
 
 /**
  * The specializing planner.
@@ -76,10 +76,11 @@ function mayReturnContext (logic) {
 
 /**
  * @param {{ engine: any, rules: Record<string, any>, firsts: Map<string, any>|null,
- *           memo: boolean, maxSteps: number, unsafeEval?: boolean }} config
+ *           memo: boolean, maxSteps: number, unsafeEval?: boolean,
+ *           tokenIds?: Map<string, number>|null }} config
  */
 export function createPlanner (config) {
-  const { engine, rules, firsts, memo, maxSteps, unsafeEval = true } = config
+  const { engine, rules, firsts, memo, maxSteps, unsafeEval = true, tokenIds = null } = config
 
   /** @type {Map<string, { fn: ((state: any) => any) | null }>} */
   const slots = new Map()
@@ -227,9 +228,14 @@ export function createPlanner (config) {
   /** @returns {(state: any) => any} */
   const buildConsume = (arg) => {
       const name = typeof arg === 'string' ? arg : arg[0]
+      const id = tokenIds ? tokenIds.get(name) : undefined
+      // `token.id` is undefined only for a hand-built token that bypassed
+      // `createLexer` (parseTokens accepts any array shaped like a Token), in
+      // which case falling back to the string compare keeps that documented
+      // path working; every token this parser's own lexer produces has an id.
       return (state) => {
         const token = state.tokens[state.idx]
-        if (token !== undefined && token.type === name) {
+        if (token !== undefined && (token.id === id || (token.id === undefined && token.type === name))) {
           state.idx++
           return token
         }
@@ -348,10 +354,11 @@ export function createPlanner (config) {
       // Terser than an `alt` of `consume`s, and a single set lookup.
       const names = Array.isArray(arg) ? arg : [arg]
       const set = new Set(names)
+      const idSet = tokenIds ? new Set(names.map((n) => tokenIds.get(n))) : null
       const count = names.length
       return (state) => {
         const token = state.tokens[state.idx]
-        if (token !== undefined && set.has(token.type)) {
+        if (token !== undefined && (idSet ? (token.id !== undefined ? idSet.has(token.id) : set.has(token.type)) : set.has(token.type))) {
           state.idx++
           return token
         }
@@ -700,6 +707,7 @@ export function createPlanner (config) {
     const kinds = []
     const names = []     // binding name, for the labelled kinds
     const tokens = []    // token type, for the terminal kinds
+    const tokenIdList = [] // token.id equivalent of `tokens`, for the terminal kinds
     const fns = []
     const actionLogic = []   // raw logic of each action child, for the reuse check
     let hasAction = false
@@ -724,11 +732,13 @@ export function createPlanner (config) {
           kinds.push(LABELLED_TERMINAL)
           names.push(name)
           tokens.push(terminal)
+          tokenIdList.push(tokenIds ? tokenIds.get(terminal) : undefined)
           fns.push(null)
         } else {
           kinds.push(LABELLED)
           names.push(name)
           tokens.push(null)
+          tokenIdList.push(undefined)
           fns.push(plan(inner))
         }
         continue
@@ -738,6 +748,7 @@ export function createPlanner (config) {
         kinds.push(ACTION)
         names.push(null)
         tokens.push(null)
+        tokenIdList.push(undefined)
         fns.push(planAction(child.action))
         actionLogic.push(child.action)
         hasAction = true
@@ -749,11 +760,13 @@ export function createPlanner (config) {
         kinds.push(TERMINAL)
         names.push(null)
         tokens.push(terminal)
+        tokenIdList.push(tokenIds ? tokenIds.get(terminal) : undefined)
         fns.push(null)
       } else {
         kinds.push(PARSER)
         names.push(null)
         tokens.push(null)
+        tokenIdList.push(undefined)
         fns.push(plan(child))
       }
     }
@@ -803,7 +816,7 @@ export function createPlanner (config) {
 
         if (kind === TERMINAL || kind === LABELLED_TERMINAL) {
           const token = state.tokens[state.idx]
-          if (token !== undefined && token.type === tokens[i]) {
+          if (token !== undefined && (token.id === tokenIdList[i] || (token.id === undefined && token.type === tokens[i]))) {
             state.idx++
             value = token
           } else {
@@ -837,6 +850,12 @@ export function createPlanner (config) {
   function planAlt (branches) {
     if (!Array.isArray(branches)) throw new Error("'alt' expects an array of branches")
 
+    // A shared, possibly multi-token prefix across every branch is matched
+    // once here rather than once per branch that has it — see the comment on
+    // recognizeAltPrefixIdiom for the safety conditions.
+    const factored = recognizeAltPrefixIdiom(branches)
+    if (factored !== null) return planUncached(factored)
+
     const planned = branches.map(plan)
     const table = firsts ? buildDispatch(branches, firsts) : null
 
@@ -850,8 +869,39 @@ export function createPlanner (config) {
       // internalized string is an inline-cached property load instead. The null
       // prototype also means a token type of `toString` or `constructor` cannot
       // find an inherited value.
+      //
+      // byId is the same table again, indexed by token.id instead of
+      // token.type — an array index instead of a property load. Used whenever
+      // a token has an id; byToken stays as the fallback for a hand-built
+      // token from parseTokens, which has none.
       const byToken = Object.create(null)
-      for (const [token, branch] of table) byToken[token] = planned[branches.indexOf(branch)]
+      const byId = tokenIds ? [] : null
+      const groupClosures = new Map()
+      for (const [token, branch] of table) {
+        let closure = groupClosures.get(branch)
+        if (closure === undefined) {
+          closure = Array.isArray(branch)
+            ? (() => {
+                const group = branch.map((b) => planned[branches.indexOf(b)])
+                return (state) => {
+                  const start = state.idx
+                  for (let i = 0; i < group.length; i++) {
+                    const value = group[i](state)
+                    if (value !== FAIL) return value
+                    state.idx = start
+                  }
+                  return FAIL
+                }
+              })()
+            : planned[branches.indexOf(branch)]
+          groupClosures.set(branch, closure)
+        }
+        byToken[token] = closure
+        if (byId) {
+          const id = tokenIds.get(token)
+          if (id !== undefined) byId[id] = closure
+        }
+      }
       const expectedTokens = [...table.keys()]
       const expectedCount = expectedTokens.length
 
@@ -859,7 +909,7 @@ export function createPlanner (config) {
         tick(state)
         const token = state.tokens[state.idx]
         if (token !== undefined) {
-          const branch = byToken[token.type]
+          const branch = byId && token.id !== undefined ? byId[token.id] : byToken[token.type]
           if (branch !== undefined) return branch(state)
         }
         for (let i = 0; i < expectedCount; i++) expect(state, expectedTokens[i])