grammatik 0.1.0 → 0.1.2

package/src/parser.js

@@ -9,6 +9,7 @@ import { createValueMethods } from './values.js'
 import { analyze } from './analyze.js'
 import { createPlanner } from './plan.js'
 import { createCodegen } from './codegen.js'
+import { selectMemoRules } from './memoplan.js'
 import { FAIL, createState, buildParseError, ParseError } from './state.js'
 
 /**
@@ -23,6 +24,21 @@ import { FAIL, createState, buildParseError, ParseError } from './state.js'
  * @property {boolean} [memo=false]      Enable packrat memoization. Costs a Map
  *                                       write per rule invocation; pays for itself
  *                                       only on grammars that backtrack heavily.
+ * @property {boolean} [autoMemo=true]  Memoize just the rules that would otherwise be
+ *                                       re-parsed by several colliding branches of an
+ *                                       `alt` (see `selectMemoRules`). Unlike `memo` it
+ *                                       costs nothing on a grammar with no such rule,
+ *                                       and it is ignored when `memo` is on.
+ * @property {boolean} [inlineRules=true] In generated code, write single-caller operator
+ *                                       levels (`infixLeft`, `infixRight`, `prefix`, `postfix`)
+ *                                       into their caller instead of calling them. Costs code
+ *                                       size, which is why `emitModule` defaults it off.
+ * @property {boolean} [idioms=true]    Recognize common spellings (`seq(open, label, close, action)`,
+ *                                       a prefix plus one labelled result, a constant action, an
+ *                                       `alt` of bare `consume`s, a shared leading run of `alt`
+ *                                       branches) and run them as the
+ *                                       cheaper combinator they are equivalent to. Off runs the
+ *                                       grammar exactly as written; for debugging and benchmarking.
  * @property {boolean} [strict=true]     Require the whole token stream to be consumed.
  * @property {boolean} [validate=true]   Throw on grammar errors (left recursion, bad refs).
  * @property {number} [maxSteps=0]       Runaway guard; 0 disables.
@@ -75,6 +91,9 @@ export function createParser (spec, options = {}) {
   const { tokens: tokenDefs, rules, start } = spec
   const {
     memo = false,
+    autoMemo = true,
+    inlineRules = true,
+    idioms = true,
     strict = true,
     validate = true,
     maxSteps = 0,
@@ -114,9 +133,23 @@ export function createParser (spec, options = {}) {
   const startRule = rules[start]
   const firsts = ll1 ? analysis.firsts : null
 
+  // Declaration index of each token name, the same numbering `createLexer`
+  // assigns to the tokens it produces (both derive it from the same array in
+  // the same order). `consume` compares this instead of the string `type`
+  // where it can — an integer compare instead of a string compare, on the
+  // hottest path in the parser.
+  const tokenIds = new Map(lexer.tokenNames.map((name, id) => [name, id]))
+
   // The planner specializes the grammar into closures; the combinators'
   // `compile` hooks reach it through the engine.
-  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval })
+  // Which rules to remember per position. Global `memo` already remembers all of
+  // them, and `validate: false` can hand us a grammar the analysis did not accept.
+  const memoRules = autoMemo && !memo && analysis.errors.length === 0
+    ? selectMemoRules(rules, analysis.firsts, { ll1 })
+    : []
+  const memoSlots = new Map(memoRules.map((name, slot) => [name, slot]))
+
+  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval, tokenIds, memoSlots, idioms })
   engine.__jlGrammar = { planner }
 
   // Source generation emits one JS function per rule with every node inlined,
@@ -125,7 +158,7 @@ export function createParser (spec, options = {}) {
   // lands on the generated code rather than a method dispatch.
   // Code generation needs `new Function`, so it is off whenever eval is.
   const codegen = compile && unsafeEval
-    ? createCodegen({ planner, rules, firsts, memo, maxSteps })
+    ? createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds, memoSlots, inlineRules, idioms })
     : null
   const compiledStart = codegen && codegen.compile(start)
 
@@ -142,7 +175,7 @@ export function createParser (spec, options = {}) {
    * @param {string} [source] Original text, so `text` can slice exact spans.
    */
   function parseTokens (tokens, source, track = false) {
-    const state = createState(tokens, rules, { memo, maxSteps, track })
+    const state = createState(tokens, rules, { memo, maxSteps, track, memoSlots: memoRules.length })
     state.firsts = firsts
     state.source = source
     state.positions = lexer.positions
@@ -185,6 +218,8 @@ export function createParser (spec, options = {}) {
     planner,
     codegen,
     execution: mode,
+    /** Rules memoized by `autoMemo`, for introspection. */
+    memoRules,
     compiled: Boolean(compiledStart),
     /** Rules whose `alt`s resolved to O(1) LL(1) dispatch, for introspection. */
     get firsts () { return analysis.firsts }

package/src/plan.js

@@ -2,7 +2,7 @@
 'use strict'
 
 import { FAIL, expect } from './state.js'
-import { buildDispatch, opOf, recognizeSeqIdiom } from './analyze.js'
+import { buildDispatch, opOf, recognizeSeqIdiom, recognizeAltPrefixIdiom, recognizeAltConsumeIdiom, usesOuterScope } from './analyze.js'
 
 /**
  * The specializing planner.
@@ -31,27 +31,6 @@ function declareSync (fn) {
   return fn
 }
 
-/**
- * True when the logic reads from an outer scope (`{val: [[-1], ...]}` or
- * `{var: "../x"}`). The engine's compiler does not thread `above` into built
- * functions, so such actions must stay interpreted.
- */
-function usesOuterScope (logic) {
-  if (Array.isArray(logic)) return logic.some(usesOuterScope)
-  if (!logic || typeof logic !== 'object') return false
-
-  for (const key of Object.keys(logic)) {
-    const arg = logic[key]
-    if (key === 'val' && Array.isArray(arg) && Array.isArray(arg[0])) return true
-    if (key === 'var') {
-      if (typeof arg === 'string' && arg.includes('../')) return true
-      if (Array.isArray(arg) && typeof arg[0] === 'string' && arg[0].includes('../')) return true
-    }
-    if (usesOuterScope(arg)) return true
-  }
-  return false
-}
-
 /**
  * True when the logic could return the *context object itself* rather than a
  * value derived from it — `{"var": ""}` and `{"val": []}` both do.
@@ -76,10 +55,12 @@ function mayReturnContext (logic) {
 
 /**
  * @param {{ engine: any, rules: Record<string, any>, firsts: Map<string, any>|null,
- *           memo: boolean, maxSteps: number, unsafeEval?: boolean }} config
+ *           memo: boolean, maxSteps: number, unsafeEval?: boolean,
+ *           tokenIds?: Map<string, number>|null,
+ *           memoSlots?: Map<string, number>, idioms?: boolean }} config
  */
 export function createPlanner (config) {
-  const { engine, rules, firsts, memo, maxSteps, unsafeEval = true } = config
+  const { engine, rules, firsts, memo, maxSteps, unsafeEval = true, tokenIds = null, memoSlots = new Map(), idioms = true } = config
 
   /** @type {Map<string, { fn: ((state: any) => any) | null }>} */
   const slots = new Map()
@@ -227,9 +208,14 @@ export function createPlanner (config) {
   /** @returns {(state: any) => any} */
   const buildConsume = (arg) => {
       const name = typeof arg === 'string' ? arg : arg[0]
+      const id = tokenIds ? tokenIds.get(name) : undefined
+      // `token.id` is undefined only for a hand-built token that bypassed
+      // `createLexer` (parseTokens accepts any array shaped like a Token), in
+      // which case falling back to the string compare keeps that documented
+      // path working; every token this parser's own lexer produces has an id.
       return (state) => {
         const token = state.tokens[state.idx]
-        if (token !== undefined && token.type === name) {
+        if (token !== undefined && (token.id === id || (token.id === undefined && token.type === name))) {
           state.idx++
           return token
         }
@@ -297,6 +283,25 @@ export function createPlanner (config) {
       }
   }
 
+  /**
+   * `{"flag": p}` — `true` when `p` matched, `false` when it did not; never fails.
+   * `option` with the answer already turned into a boolean, for the common
+   * `label(negated, option(consume(Not)))` followed by `{"!==": [negated, null]}`.
+   *
+   * @returns {(state: any) => any}
+   */
+  const buildFlag = (arg) => {
+      const child = plan(arg)
+      return (state) => {
+        const start = state.idx
+        if (child(state) === FAIL) {
+          state.idx = start
+          return false
+        }
+        return true
+      }
+  }
+
   /** @returns {(state: any) => any} */
   const buildInfixLeft = (arg) => {
       // A whole precedence level in one closure.
@@ -348,10 +353,11 @@ export function createPlanner (config) {
       // Terser than an `alt` of `consume`s, and a single set lookup.
       const names = Array.isArray(arg) ? arg : [arg]
       const set = new Set(names)
+      const idSet = tokenIds ? new Set(names.map((n) => tokenIds.get(n))) : null
       const count = names.length
       return (state) => {
         const token = state.tokens[state.idx]
-        if (token !== undefined && set.has(token.type)) {
+        if (token !== undefined && (idSet ? (token.id !== undefined ? idSet.has(token.id) : set.has(token.type)) : set.has(token.type))) {
           state.idx++
           return token
         }
@@ -460,6 +466,35 @@ export function createPlanner (config) {
       }
   }
 
+  /**
+   * `{"pick": {parsers, index}}` and `{"yields": {parsers, value}}` run their parsers
+   * in order and give back one of the results or a constant. They are what a `seq`
+   * with a single labelled child and `action({val: label})`, or with a constant
+   * action, is rewritten to: neither needs the bindings object a `seq` allocates.
+   *
+   * @returns {(state: any) => any}
+   */
+  const buildSequence = (arg, pickIndex, constant) => {
+      const fns = arg.parsers.map(plan)
+      const count = fns.length
+      return (state) => {
+        tick(state)
+        const start = state.idx
+        let picked
+        for (let i = 0; i < count; i++) {
+          const value = fns[i](state)
+          if (value === FAIL) {
+            state.idx = start
+            return FAIL
+          }
+          if (i === pickIndex) picked = value
+        }
+        return pickIndex === -1 ? constant : picked
+      }
+  }
+  const buildPick = (arg) => buildSequence(arg, arg.index, undefined)
+  const buildYields = (arg) => buildSequence(arg, -1, arg.value)
+
   /** @returns {(state: any) => any} */
   const buildPrefix = (arg) => {
       // Any number of prefix operators, folded right onto the operand:
@@ -560,6 +595,34 @@ export function createPlanner (config) {
           return value
         }
       }
+      // A rule chosen by `selectMemoRules` remembers its result per position, so
+      // the branches of an `alt` that all start with it parse it once between
+      // them. Global `memo` already covers every rule, hence the order here.
+      const memoSlot = memoSlots.get(name)
+      if (memoSlot !== undefined) {
+        return (state) => {
+          tick(state)
+          const table = state.rm[memoSlot]
+          const at = state.idx
+          const end = table.ends[at]
+          if (end !== 0) {
+            state.idx = end - 1
+            return table.vals[at]
+          }
+          let depth = 0
+          if (state.track) {
+            depth = state.ruleDepth
+            state.ruleNames[depth] = name
+            state.ruleDepth = depth + 1
+          }
+          const value = slot.fn(state)
+          if (state.track) state.ruleDepth = depth
+          if (value === FAIL) state.idx = at
+          table.ends[at] = state.idx + 1
+          table.vals[at] = value
+          return value
+        }
+      }
       return (state) => {
         tick(state)
         // See above: rule-stack bookkeeping is failure-path only.
@@ -657,6 +720,9 @@ export function createPlanner (config) {
     infixRight: buildInfixRight,
     as: buildAs,
     between: buildBetween,
+    flag: buildFlag,
+    pick: buildPick,
+    yields: buildYields,
     prefix: buildPrefix,
     postfix: buildPostfix,
     manySep: (arg) => planSeparated(arg, false),
@@ -683,7 +749,7 @@ export function createPlanner (config) {
     // A `seq` written in one of the recognized idioms is planned as the cheaper
     // combinator instead, so grammars written with the general primitives get
     // the same treatment as ones written with the specific ones.
-    const idiom = recognizeSeqIdiom(children)
+    const idiom = idioms ? recognizeSeqIdiom(children) : null
     if (idiom !== null) return planUncached(idiom)
 
     // Step kinds. Terminals are inlined rather than dispatched to a closure:
@@ -700,6 +766,7 @@ export function createPlanner (config) {
     const kinds = []
     const names = []     // binding name, for the labelled kinds
     const tokens = []    // token type, for the terminal kinds
+    const tokenIdList = [] // token.id equivalent of `tokens`, for the terminal kinds
     const fns = []
     const actionLogic = []   // raw logic of each action child, for the reuse check
     let hasAction = false
@@ -724,11 +791,13 @@ export function createPlanner (config) {
           kinds.push(LABELLED_TERMINAL)
           names.push(name)
           tokens.push(terminal)
+          tokenIdList.push(tokenIds ? tokenIds.get(terminal) : undefined)
           fns.push(null)
         } else {
           kinds.push(LABELLED)
           names.push(name)
           tokens.push(null)
+          tokenIdList.push(undefined)
           fns.push(plan(inner))
         }
         continue
@@ -738,6 +807,7 @@ export function createPlanner (config) {
         kinds.push(ACTION)
         names.push(null)
         tokens.push(null)
+        tokenIdList.push(undefined)
         fns.push(planAction(child.action))
         actionLogic.push(child.action)
         hasAction = true
@@ -749,11 +819,13 @@ export function createPlanner (config) {
         kinds.push(TERMINAL)
         names.push(null)
         tokens.push(terminal)
+        tokenIdList.push(tokenIds ? tokenIds.get(terminal) : undefined)
         fns.push(null)
       } else {
         kinds.push(PARSER)
         names.push(null)
         tokens.push(null)
+        tokenIdList.push(undefined)
         fns.push(plan(child))
       }
     }
@@ -803,7 +875,7 @@ export function createPlanner (config) {
 
         if (kind === TERMINAL || kind === LABELLED_TERMINAL) {
           const token = state.tokens[state.idx]
-          if (token !== undefined && token.type === tokens[i]) {
+          if (token !== undefined && (token.id === tokenIdList[i] || (token.id === undefined && token.type === tokens[i]))) {
             state.idx++
             value = token
           } else {
@@ -837,6 +909,15 @@ export function createPlanner (config) {
   function planAlt (branches) {
     if (!Array.isArray(branches)) throw new Error("'alt' expects an array of branches")
 
+    // A shared, possibly multi-token prefix across every branch is matched
+    // once here rather than once per branch that has it — see the comment on
+    // recognizeAltPrefixIdiom for the safety conditions.
+    const oneOf = idioms ? recognizeAltConsumeIdiom(branches) : null
+    if (oneOf !== null) return planUncached(oneOf)
+
+    const factored = idioms ? recognizeAltPrefixIdiom(branches) : null
+    if (factored !== null) return planUncached(factored)
+
     const planned = branches.map(plan)
     const table = firsts ? buildDispatch(branches, firsts) : null
 
@@ -850,8 +931,39 @@ export function createPlanner (config) {
       // internalized string is an inline-cached property load instead. The null
       // prototype also means a token type of `toString` or `constructor` cannot
       // find an inherited value.
+      //
+      // byId is the same table again, indexed by token.id instead of
+      // token.type — an array index instead of a property load. Used whenever
+      // a token has an id; byToken stays as the fallback for a hand-built
+      // token from parseTokens, which has none.
       const byToken = Object.create(null)
-      for (const [token, branch] of table) byToken[token] = planned[branches.indexOf(branch)]
+      const byId = tokenIds ? [] : null
+      const groupClosures = new Map()
+      for (const [token, branch] of table) {
+        let closure = groupClosures.get(branch)
+        if (closure === undefined) {
+          closure = Array.isArray(branch)
+            ? (() => {
+                const group = branch.map((b) => planned[branches.indexOf(b)])
+                return (state) => {
+                  const start = state.idx
+                  for (let i = 0; i < group.length; i++) {
+                    const value = group[i](state)
+                    if (value !== FAIL) return value
+                    state.idx = start
+                  }
+                  return FAIL
+                }
+              })()
+            : planned[branches.indexOf(branch)]
+          groupClosures.set(branch, closure)
+        }
+        byToken[token] = closure
+        if (byId) {
+          const id = tokenIds.get(token)
+          if (id !== undefined) byId[id] = closure
+        }
+      }
       const expectedTokens = [...table.keys()]
       const expectedCount = expectedTokens.length
 
@@ -859,7 +971,7 @@ export function createPlanner (config) {
         tick(state)
         const token = state.tokens[state.idx]
         if (token !== undefined) {
-          const branch = byToken[token.type]
+          const branch = byId && token.id !== undefined ? byId[token.id] : byToken[token.type]
           if (branch !== undefined) return branch(state)
         }
         for (let i = 0; i < expectedCount; i++) expect(state, expectedTokens[i])

package/src/runtime.js

@@ -5,12 +5,14 @@
  * The surface a generated parser module needs at run time.
  *
  * `emitModule` writes files that import from `grammatik/runtime` rather than
- * from the main entry point, so a generated parser pulls in the lexer, the parse
- * state and error construction — but not the analyzer, the planner, or the code
- * generator, none of which it can ever use. The grammar was already analyzed
- * when the file was written.
+ * from the main entry point, so a generated parser pulls in the parse state and
+ * error construction — and not the analyzer, the planner or the code generator,
+ * none of which it can ever use. The grammar was already analyzed when the file
+ * was written, and so, by default, was its lexer (see `lexgen.js`): the file
+ * carries that as source and needs only `LexError` from here. `createLexer` is
+ * exported for files emitted with `lexer: 'runtime'`.
  */
 
-export { FAIL, expect, createState, buildParseError, ParseError } from './state.js'
-export { createLexer, LexError } from './lexer.js'
+export { FAIL, expect, createState, buildParseError, ParseError, LexError } from './state.js'
+export { createLexer } from './lexer.js'
 export { createValueMethods } from './values.js'