grammatik 0.1.1 → 0.1.2

package/src/parser.js

@@ -9,6 +9,7 @@ import { createValueMethods } from './values.js'
 import { analyze } from './analyze.js'
 import { createPlanner } from './plan.js'
 import { createCodegen } from './codegen.js'
+import { selectMemoRules } from './memoplan.js'
 import { FAIL, createState, buildParseError, ParseError } from './state.js'
 
 /**
@@ -23,6 +24,21 @@ import { FAIL, createState, buildParseError, ParseError } from './state.js'
  * @property {boolean} [memo=false]      Enable packrat memoization. Costs a Map
  *                                       write per rule invocation; pays for itself
  *                                       only on grammars that backtrack heavily.
+ * @property {boolean} [autoMemo=true]  Memoize just the rules that would otherwise be
+ *                                       re-parsed by several colliding branches of an
+ *                                       `alt` (see `selectMemoRules`). Unlike `memo` it
+ *                                       costs nothing on a grammar with no such rule,
+ *                                       and it is ignored when `memo` is on.
+ * @property {boolean} [inlineRules=true] In generated code, write single-caller operator
+ *                                       levels (`infixLeft`, `infixRight`, `prefix`, `postfix`)
+ *                                       into their caller instead of calling them. Costs code
+ *                                       size, which is why `emitModule` defaults it off.
+ * @property {boolean} [idioms=true]    Recognize common spellings (`seq(open, label, close, action)`,
+ *                                       a prefix plus one labelled result, a constant action, an
+ *                                       `alt` of bare `consume`s, a shared leading run of `alt`
+ *                                       branches) and run them as the
+ *                                       cheaper combinator they are equivalent to. Off runs the
+ *                                       grammar exactly as written; for debugging and benchmarking.
  * @property {boolean} [strict=true]     Require the whole token stream to be consumed.
  * @property {boolean} [validate=true]   Throw on grammar errors (left recursion, bad refs).
  * @property {number} [maxSteps=0]       Runaway guard; 0 disables.
@@ -75,6 +91,9 @@ export function createParser (spec, options = {}) {
   const { tokens: tokenDefs, rules, start } = spec
   const {
     memo = false,
+    autoMemo = true,
+    inlineRules = true,
+    idioms = true,
     strict = true,
     validate = true,
     maxSteps = 0,
@@ -123,7 +142,14 @@ export function createParser (spec, options = {}) {
 
   // The planner specializes the grammar into closures; the combinators'
   // `compile` hooks reach it through the engine.
-  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval, tokenIds })
+  // Which rules to remember per position. Global `memo` already remembers all of
+  // them, and `validate: false` can hand us a grammar the analysis did not accept.
+  const memoRules = autoMemo && !memo && analysis.errors.length === 0
+    ? selectMemoRules(rules, analysis.firsts, { ll1 })
+    : []
+  const memoSlots = new Map(memoRules.map((name, slot) => [name, slot]))
+
+  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval, tokenIds, memoSlots, idioms })
   engine.__jlGrammar = { planner }
 
   // Source generation emits one JS function per rule with every node inlined,
@@ -132,7 +158,7 @@ export function createParser (spec, options = {}) {
   // lands on the generated code rather than a method dispatch.
   // Code generation needs `new Function`, so it is off whenever eval is.
   const codegen = compile && unsafeEval
-    ? createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds })
+    ? createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds, memoSlots, inlineRules, idioms })
     : null
   const compiledStart = codegen && codegen.compile(start)
 
@@ -149,7 +175,7 @@ export function createParser (spec, options = {}) {
    * @param {string} [source] Original text, so `text` can slice exact spans.
    */
   function parseTokens (tokens, source, track = false) {
-    const state = createState(tokens, rules, { memo, maxSteps, track })
+    const state = createState(tokens, rules, { memo, maxSteps, track, memoSlots: memoRules.length })
     state.firsts = firsts
     state.source = source
     state.positions = lexer.positions
@@ -192,6 +218,8 @@ export function createParser (spec, options = {}) {
     planner,
     codegen,
     execution: mode,
+    /** Rules memoized by `autoMemo`, for introspection. */
+    memoRules,
     compiled: Boolean(compiledStart),
     /** Rules whose `alt`s resolved to O(1) LL(1) dispatch, for introspection. */
     get firsts () { return analysis.firsts }

package/src/plan.js

@@ -2,7 +2,7 @@
 'use strict'
 
 import { FAIL, expect } from './state.js'
-import { buildDispatch, opOf, recognizeSeqIdiom, recognizeAltPrefixIdiom } from './analyze.js'
+import { buildDispatch, opOf, recognizeSeqIdiom, recognizeAltPrefixIdiom, recognizeAltConsumeIdiom, usesOuterScope } from './analyze.js'
 
 /**
  * The specializing planner.
@@ -31,27 +31,6 @@ function declareSync (fn) {
   return fn
 }
 
-/**
- * True when the logic reads from an outer scope (`{val: [[-1], ...]}` or
- * `{var: "../x"}`). The engine's compiler does not thread `above` into built
- * functions, so such actions must stay interpreted.
- */
-function usesOuterScope (logic) {
-  if (Array.isArray(logic)) return logic.some(usesOuterScope)
-  if (!logic || typeof logic !== 'object') return false
-
-  for (const key of Object.keys(logic)) {
-    const arg = logic[key]
-    if (key === 'val' && Array.isArray(arg) && Array.isArray(arg[0])) return true
-    if (key === 'var') {
-      if (typeof arg === 'string' && arg.includes('../')) return true
-      if (Array.isArray(arg) && typeof arg[0] === 'string' && arg[0].includes('../')) return true
-    }
-    if (usesOuterScope(arg)) return true
-  }
-  return false
-}
-
 /**
  * True when the logic could return the *context object itself* rather than a
  * value derived from it — `{"var": ""}` and `{"val": []}` both do.
@@ -77,10 +56,11 @@ function mayReturnContext (logic) {
 /**
  * @param {{ engine: any, rules: Record<string, any>, firsts: Map<string, any>|null,
  *           memo: boolean, maxSteps: number, unsafeEval?: boolean,
- *           tokenIds?: Map<string, number>|null }} config
+ *           tokenIds?: Map<string, number>|null,
+ *           memoSlots?: Map<string, number>, idioms?: boolean }} config
  */
 export function createPlanner (config) {
-  const { engine, rules, firsts, memo, maxSteps, unsafeEval = true, tokenIds = null } = config
+  const { engine, rules, firsts, memo, maxSteps, unsafeEval = true, tokenIds = null, memoSlots = new Map(), idioms = true } = config
 
   /** @type {Map<string, { fn: ((state: any) => any) | null }>} */
   const slots = new Map()
@@ -303,6 +283,25 @@ export function createPlanner (config) {
       }
   }
 
+  /**
+   * `{"flag": p}` — `true` when `p` matched, `false` when it did not; never fails.
+   * `option` with the answer already turned into a boolean, for the common
+   * `label(negated, option(consume(Not)))` followed by `{"!==": [negated, null]}`.
+   *
+   * @returns {(state: any) => any}
+   */
+  const buildFlag = (arg) => {
+      const child = plan(arg)
+      return (state) => {
+        const start = state.idx
+        if (child(state) === FAIL) {
+          state.idx = start
+          return false
+        }
+        return true
+      }
+  }
+
   /** @returns {(state: any) => any} */
   const buildInfixLeft = (arg) => {
       // A whole precedence level in one closure.
@@ -467,6 +466,35 @@ export function createPlanner (config) {
       }
   }
 
+  /**
+   * `{"pick": {parsers, index}}` and `{"yields": {parsers, value}}` run their parsers
+   * in order and give back one of the results or a constant. They are what a `seq`
+   * with a single labelled child and `action({val: label})`, or with a constant
+   * action, is rewritten to: neither needs the bindings object a `seq` allocates.
+   *
+   * @returns {(state: any) => any}
+   */
+  const buildSequence = (arg, pickIndex, constant) => {
+      const fns = arg.parsers.map(plan)
+      const count = fns.length
+      return (state) => {
+        tick(state)
+        const start = state.idx
+        let picked
+        for (let i = 0; i < count; i++) {
+          const value = fns[i](state)
+          if (value === FAIL) {
+            state.idx = start
+            return FAIL
+          }
+          if (i === pickIndex) picked = value
+        }
+        return pickIndex === -1 ? constant : picked
+      }
+  }
+  const buildPick = (arg) => buildSequence(arg, arg.index, undefined)
+  const buildYields = (arg) => buildSequence(arg, -1, arg.value)
+
   /** @returns {(state: any) => any} */
   const buildPrefix = (arg) => {
       // Any number of prefix operators, folded right onto the operand:
@@ -567,6 +595,34 @@ export function createPlanner (config) {
           return value
         }
       }
+      // A rule chosen by `selectMemoRules` remembers its result per position, so
+      // the branches of an `alt` that all start with it parse it once between
+      // them. Global `memo` already covers every rule, hence the order here.
+      const memoSlot = memoSlots.get(name)
+      if (memoSlot !== undefined) {
+        return (state) => {
+          tick(state)
+          const table = state.rm[memoSlot]
+          const at = state.idx
+          const end = table.ends[at]
+          if (end !== 0) {
+            state.idx = end - 1
+            return table.vals[at]
+          }
+          let depth = 0
+          if (state.track) {
+            depth = state.ruleDepth
+            state.ruleNames[depth] = name
+            state.ruleDepth = depth + 1
+          }
+          const value = slot.fn(state)
+          if (state.track) state.ruleDepth = depth
+          if (value === FAIL) state.idx = at
+          table.ends[at] = state.idx + 1
+          table.vals[at] = value
+          return value
+        }
+      }
       return (state) => {
         tick(state)
         // See above: rule-stack bookkeeping is failure-path only.
@@ -664,6 +720,9 @@ export function createPlanner (config) {
     infixRight: buildInfixRight,
     as: buildAs,
     between: buildBetween,
+    flag: buildFlag,
+    pick: buildPick,
+    yields: buildYields,
     prefix: buildPrefix,
     postfix: buildPostfix,
     manySep: (arg) => planSeparated(arg, false),
@@ -690,7 +749,7 @@ export function createPlanner (config) {
     // A `seq` written in one of the recognized idioms is planned as the cheaper
     // combinator instead, so grammars written with the general primitives get
     // the same treatment as ones written with the specific ones.
-    const idiom = recognizeSeqIdiom(children)
+    const idiom = idioms ? recognizeSeqIdiom(children) : null
     if (idiom !== null) return planUncached(idiom)
 
     // Step kinds. Terminals are inlined rather than dispatched to a closure:
@@ -853,7 +912,10 @@ export function createPlanner (config) {
     // A shared, possibly multi-token prefix across every branch is matched
     // once here rather than once per branch that has it — see the comment on
     // recognizeAltPrefixIdiom for the safety conditions.
-    const factored = recognizeAltPrefixIdiom(branches)
+    const oneOf = idioms ? recognizeAltConsumeIdiom(branches) : null
+    if (oneOf !== null) return planUncached(oneOf)
+
+    const factored = idioms ? recognizeAltPrefixIdiom(branches) : null
     if (factored !== null) return planUncached(factored)
 
     const planned = branches.map(plan)

package/src/runtime.js

@@ -5,12 +5,14 @@
  * The surface a generated parser module needs at run time.
  *
  * `emitModule` writes files that import from `grammatik/runtime` rather than
- * from the main entry point, so a generated parser pulls in the lexer, the parse
- * state and error construction — but not the analyzer, the planner, or the code
- * generator, none of which it can ever use. The grammar was already analyzed
- * when the file was written.
+ * from the main entry point, so a generated parser pulls in the parse state and
+ * error construction — and not the analyzer, the planner or the code generator,
+ * none of which it can ever use. The grammar was already analyzed when the file
+ * was written, and so, by default, was its lexer (see `lexgen.js`): the file
+ * carries that as source and needs only `LexError` from here. `createLexer` is
+ * exported for files emitted with `lexer: 'runtime'`.
  */
 
-export { FAIL, expect, createState, buildParseError, ParseError } from './state.js'
-export { createLexer, LexError } from './lexer.js'
+export { FAIL, expect, createState, buildParseError, ParseError, LexError } from './state.js'
+export { createLexer } from './lexer.js'
 export { createValueMethods } from './values.js'