grammatik 0.1.0 → 0.1.1

package/dist/cjs/index.js

@@ -407,7 +407,7 @@ function createLexer(defs, options = {}) {
     return id;
   };
   for (const def of defs) for (const mode of modesOf(def)) idOf(mode);
-  const compiled = defs.map((def) => {
+  const compiled = defs.map((def, id) => {
     if (!def.name) throw new Error("createLexer: every token definition needs a name");
     if (!def.pattern && !def.literal) throw new Error(`createLexer: token '${def.name}' needs a pattern or a literal`);
     if (def.pattern && def.literal) throw new Error(`createLexer: token '${def.name}' has both a pattern and a literal; pick one`);
@@ -422,6 +422,11 @@ function createLexer(defs, options = {}) {
     }
     return {
       name: def.name,
+      // The declaration index. `createParser` derives the identical mapping
+      // from `tokenNames` independently (same array, same order), so a
+      // combinator that resolves a `consume` target's id at build time agrees
+      // with what a token actually carries without the two ever comparing notes.
+      id,
       skip: !!def.skip,
       literal: def.literal,
       longerAlt: def.longerAlt || null,
@@ -520,14 +525,7 @@ function createLexer(defs, options = {}) {
         }
         if (!winner.skip) {
           const image = winner.literal !== void 0 ? winner.literal : winner === def && matched !== null ? matched : text.slice(offset, end);
-          tokens.push({
-            type: winner.name,
-            image,
-            start: offset,
-            end,
-            line: trackLines ? line : 0,
-            col: trackLines ? offset - lineStart + 1 : 0
-          });
+          tokens.push(trackLines ? { type: winner.name, image, start: offset, end, line, col: offset - lineStart + 1 } : { type: winner.name, id: winner.id, image, start: offset, end });
         }
         if (trackLines) {
           for (let j = offset; j < end; j++) {
@@ -803,6 +801,47 @@ function isPlainParser(node) {
   const op = opOf(node);
   return op !== null && op !== "label" && op !== "action";
 }
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (Array.isArray(a) || Array.isArray(b)) {
+    if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) if (!deepEqual(a[i], b[i])) return false;
+    return true;
+  }
+  if (a === null || b === null || typeof a !== "object" || typeof b !== "object") return false;
+  const keysA = Object.keys(a);
+  const keysB = Object.keys(b);
+  if (keysA.length !== keysB.length) return false;
+  for (const key of keysA) {
+    if (!Object.prototype.hasOwnProperty.call(b, key) || !deepEqual(a[key], b[key])) return false;
+  }
+  return true;
+}
+function recognizeAltPrefixIdiom(branches) {
+  if (!Array.isArray(branches) || branches.length < 2) return null;
+  if (!branches.every((b) => opOf(b) === "seq" && Array.isArray(b.seq))) return null;
+  const seqs = branches.map((b) => b.seq);
+  const minLen = Math.min(...seqs.map((s) => s.length));
+  let n = 0;
+  while (n < minLen) {
+    const candidate = seqs[0][n];
+    const op = opOf(candidate);
+    if (op === "label" || op === "action") break;
+    if (!seqs.every((s) => deepEqual(s[n], candidate))) break;
+    n++;
+  }
+  if (n === 0) return null;
+  if (!seqs.every((s) => s.slice(n).some((child) => opOf(child) === "action"))) return null;
+  const prefix = seqs[0].slice(0, n);
+  const remainders = seqs.map((s) => ({ seq: s.slice(n) }));
+  return {
+    seq: [
+      ...prefix,
+      { label: ["__grammatik_rest", { alt: remainders }] },
+      { action: { val: "__grammatik_rest" } }
+    ]
+  };
+}
 var BAIL = /* @__PURE__ */ Symbol("grammatik.bail");
 function rebaseOnLabel(logic, name) {
   if (Array.isArray(logic)) {
@@ -1052,16 +1091,38 @@ function analyze(grammar, options = {}) {
   return { firsts, errors, leftRecursive };
 }
 function buildDispatch(branches, firsts) {
-  const table = /* @__PURE__ */ new Map();
-  for (const branch of branches) {
-    const first = firstOf(branch, firsts);
+  const perBranch = branches.map((branch) => firstOf(branch, firsts));
+  for (const first of perBranch) {
     if (first.nullable || first.unknown || first.tokens.size === 0) return null;
-    for (const token of first.tokens) {
-      if (table.has(token)) return null;
-      table.set(token, branch);
+  }
+  const byToken = /* @__PURE__ */ new Map();
+  for (let i = 0; i < branches.length; i++) {
+    for (const token of perBranch[i].tokens) {
+      let list = byToken.get(token);
+      if (list === void 0) {
+        list = [];
+        byToken.set(token, list);
+      }
+      list.push(branches[i]);
     }
   }
-  return table.size > 0 ? table : null;
+  if (byToken.size === 0) return null;
+  const canonical = /* @__PURE__ */ new Map();
+  const table = /* @__PURE__ */ new Map();
+  for (const [token, list] of byToken) {
+    if (list.length === 1) {
+      table.set(token, list[0]);
+      continue;
+    }
+    const key = list.map((branch) => branches.indexOf(branch)).join(",");
+    let group = canonical.get(key);
+    if (group === void 0) {
+      group = list;
+      canonical.set(key, group);
+    }
+    table.set(token, group);
+  }
+  return table;
 }
 
 // src/plan.js
@@ -1096,7 +1157,7 @@ function mayReturnContext(logic) {
   return false;
 }
 function createPlanner(config) {
-  const { engine, rules, firsts, memo, maxSteps, unsafeEval = true } = config;
+  const { engine, rules, firsts, memo, maxSteps, unsafeEval = true, tokenIds = null } = config;
   const slots = /* @__PURE__ */ new Map();
   const planCache = /* @__PURE__ */ new WeakMap();
   const opCaches = /* @__PURE__ */ new Map();
@@ -1179,9 +1240,10 @@ function createPlanner(config) {
   }
   const buildConsume = (arg) => {
     const name = typeof arg === "string" ? arg : arg[0];
+    const id = tokenIds ? tokenIds.get(name) : void 0;
     return (state) => {
       const token = state.tokens[state.idx];
-      if (token !== void 0 && token.type === name) {
+      if (token !== void 0 && (token.id === id || token.id === void 0 && token.type === name)) {
         state.idx++;
         return token;
       }
@@ -1277,10 +1339,11 @@ function createPlanner(config) {
   const buildOneOf = (arg) => {
     const names = Array.isArray(arg) ? arg : [arg];
     const set = new Set(names);
+    const idSet = tokenIds ? new Set(names.map((n) => tokenIds.get(n))) : null;
     const count = names.length;
     return (state) => {
       const token = state.tokens[state.idx];
-      if (token !== void 0 && set.has(token.type)) {
+      if (token !== void 0 && (idSet ? token.id !== void 0 ? idSet.has(token.id) : set.has(token.type) : set.has(token.type))) {
         state.idx++;
         return token;
       }
@@ -1556,6 +1619,7 @@ function createPlanner(config) {
     const kinds = [];
     const names = [];
     const tokens = [];
+    const tokenIdList = [];
     const fns = [];
     const actionLogic = [];
     let hasAction = false;
@@ -1576,11 +1640,13 @@ function createPlanner(config) {
           kinds.push(LABELLED_TERMINAL);
           names.push(name);
           tokens.push(terminal2);
+          tokenIdList.push(tokenIds ? tokenIds.get(terminal2) : void 0);
           fns.push(null);
         } else {
           kinds.push(LABELLED);
           names.push(name);
           tokens.push(null);
+          tokenIdList.push(void 0);
           fns.push(plan(inner));
         }
         continue;
@@ -1589,6 +1655,7 @@ function createPlanner(config) {
         kinds.push(ACTION);
         names.push(null);
         tokens.push(null);
+        tokenIdList.push(void 0);
         fns.push(planAction(child.action));
         actionLogic.push(child.action);
         hasAction = true;
@@ -1599,11 +1666,13 @@ function createPlanner(config) {
         kinds.push(TERMINAL);
         names.push(null);
         tokens.push(terminal);
+        tokenIdList.push(tokenIds ? tokenIds.get(terminal) : void 0);
         fns.push(null);
       } else {
         kinds.push(PARSER);
         names.push(null);
         tokens.push(null);
+        tokenIdList.push(void 0);
         fns.push(plan(child));
       }
     }
@@ -1624,7 +1693,7 @@ function createPlanner(config) {
         let value;
         if (kind === TERMINAL || kind === LABELLED_TERMINAL) {
           const token = state.tokens[state.idx];
-          if (token !== void 0 && token.type === tokens[i]) {
+          if (token !== void 0 && (token.id === tokenIdList[i] || token.id === void 0 && token.type === tokens[i])) {
             state.idx++;
             value = token;
           } else {
@@ -1654,18 +1723,44 @@ function createPlanner(config) {
   }
   function planAlt(branches) {
     if (!Array.isArray(branches)) throw new Error("'alt' expects an array of branches");
+    const factored = recognizeAltPrefixIdiom(branches);
+    if (factored !== null) return planUncached(factored);
     const planned = branches.map(plan);
     const table = firsts ? buildDispatch(branches, firsts) : null;
     if (table !== null) {
       const byToken = /* @__PURE__ */ Object.create(null);
-      for (const [token, branch] of table) byToken[token] = planned[branches.indexOf(branch)];
+      const byId = tokenIds ? [] : null;
+      const groupClosures = /* @__PURE__ */ new Map();
+      for (const [token, branch] of table) {
+        let closure = groupClosures.get(branch);
+        if (closure === void 0) {
+          closure = Array.isArray(branch) ? (() => {
+            const group = branch.map((b) => planned[branches.indexOf(b)]);
+            return (state) => {
+              const start = state.idx;
+              for (let i = 0; i < group.length; i++) {
+                const value = group[i](state);
+                if (value !== FAIL) return value;
+                state.idx = start;
+              }
+              return FAIL;
+            };
+          })() : planned[branches.indexOf(branch)];
+          groupClosures.set(branch, closure);
+        }
+        byToken[token] = closure;
+        if (byId) {
+          const id = tokenIds.get(token);
+          if (id !== void 0) byId[id] = closure;
+        }
+      }
       const expectedTokens = [...table.keys()];
       const expectedCount = expectedTokens.length;
       return (state) => {
         tick(state);
         const token = state.tokens[state.idx];
         if (token !== void 0) {
-          const branch = byToken[token.type];
+          const branch = byId && token.id !== void 0 ? byId[token.id] : byToken[token.type];
           if (branch !== void 0) return branch(state);
         }
         for (let i = 0; i < expectedCount; i++) expect(state, expectedTokens[i]);
@@ -1804,7 +1899,12 @@ var createValueMethods = () => ({
 
 // src/codegen.js
 function createCodegen(config) {
-  const { planner, rules, firsts, memo, maxSteps } = config;
+  const { planner, rules, firsts, memo, maxSteps, tokenIds = null } = config;
+  const typeCheck = (tokenVar, name) => {
+    const id = tokenIds ? tokenIds.get(name) : void 0;
+    if (id === void 0) return `${tokenVar}.type === ${JSON.stringify(name)}`;
+    return `(${tokenVar}.id === ${id} || (${tokenVar}.id === undefined && ${tokenVar}.type === ${JSON.stringify(name)}))`;
+  };
   const HELPERS = [];
   const ORIGINS = [];
   const RULES = [];
@@ -1846,7 +1946,7 @@ function createCodegen(config) {
         out.push(
           `let ${result};`,
           `const ${token} = c.tokens[c.idx];`,
-          `if (${token} !== undefined && ${token}.type === ${JSON.stringify(name)}) { c.idx++; ${result} = ${token}; }`,
+          `if (${token} !== undefined && ${typeCheck(token, name)}) { c.idx++; ${result} = ${token}; }`,
           `else { E(c, ${JSON.stringify(name)}); ${result} = F; }`
         );
         return result;
@@ -1942,7 +2042,7 @@ function createCodegen(config) {
       case "oneOf": {
         const names = Array.isArray(arg) ? arg : [arg];
         const token = nextId();
-        const test = names.map((n) => `${token}.type === ${JSON.stringify(n)}`).join(" || ");
+        const test = names.map((n) => typeCheck(token, n)).join(" || ");
         out.push(
           `let ${result};`,
           `const ${token} = c.tokens[c.idx];`,
@@ -2221,24 +2321,51 @@ function createCodegen(config) {
     out.push(`${result} = ${hasAction ? actionResult : "[" + collected.join(", ") + "]"};`, "}");
     return result;
   }
+  function genOrderedBacktrack(list, out, bindingsVar) {
+    const result = nextId();
+    const label2 = "L" + nextId();
+    const start = nextId();
+    out.push(`let ${result};`, `${label2}: {`, `const ${start} = c.idx;`);
+    for (const branch of list) {
+      out.push("{");
+      const value = gen(branch, out, bindingsVar);
+      out.push(`if (${value} !== F) { ${result} = ${value}; break ${label2}; }`, `c.idx = ${start};`, "}");
+    }
+    out.push(`${result} = F;`, "}");
+    return result;
+  }
   function genAlt(branches, out, result, bindingsVar) {
     if (!Array.isArray(branches)) throw new Error("'alt' expects an array of branches");
+    const factored = recognizeAltPrefixIdiom(branches);
+    if (factored !== null) return gen(factored, out, bindingsVar);
     const table = firsts ? buildDispatch(branches, firsts) : null;
     const label2 = "L" + nextId();
     if (table !== null) {
       const expected = [...table.keys()];
+      const expectedIds = tokenIds ? expected.map((t) => tokenIds.get(t)) : null;
+      const numeric = expectedIds !== null && expectedIds.every((id) => id !== void 0);
       const token = nextId();
       out.push(tickSrc, `let ${result};`, `${label2}: {`, `const ${token} = c.tokens[c.idx];`);
       out.push(`if (${token} === undefined) { ${expected.map((t) => `E(c, ${JSON.stringify(t)});`).join(" ")} ${result} = F; break ${label2}; }`);
-      out.push(`switch (${token}.type) {`);
+      let discriminant;
+      if (numeric) {
+        discriminant = nextId();
+        out.push(`const ${discriminant} = ${token}.id !== undefined ? ${token}.id : I[${token}.type];`);
+        out.push(`switch (${discriminant}) {`);
+      } else {
+        discriminant = `${token}.type`;
+        out.push(`switch (${discriminant}) {`);
+      }
       const byBranch = /* @__PURE__ */ new Map();
-      for (const [tokenName, branch] of table) {
+      for (const tokenName of expected) {
+        const branch = table.get(tokenName);
         if (!byBranch.has(branch)) byBranch.set(branch, []);
         byBranch.get(branch).push(tokenName);
       }
+      const caseLabel = (t) => numeric ? tokenIds.get(t) : JSON.stringify(t);
       for (const [branch, tokenNames] of byBranch) {
-        out.push(tokenNames.map((t) => `case ${JSON.stringify(t)}:`).join(" ") + " {");
-        const value = gen(branch, out, bindingsVar);
+        out.push(tokenNames.map((t) => `case ${caseLabel(t)}:`).join(" ") + " {");
+        const value = Array.isArray(branch) ? genOrderedBacktrack(branch, out, bindingsVar) : gen(branch, out, bindingsVar);
         out.push(`${result} = ${value}; break ${label2};`, "}");
       }
       out.push(
@@ -2248,16 +2375,10 @@ function createCodegen(config) {
       );
       return result;
     }
-    const start = nextId();
-    out.push(tickSrc, `let ${result};`, `${label2}: {`, `const ${start} = c.idx;`);
-    for (const branch of branches) {
-      out.push("{");
-      const value = gen(branch, out, bindingsVar);
-      out.push(`if (${value} !== F) { ${result} = ${value}; break ${label2}; }`, `c.idx = ${start};`, "}");
-    }
-    out.push(`${result} = F;`, "}");
-    return result;
+    out.push(tickSrc);
+    return genOrderedBacktrack(branches, out, bindingsVar);
   }
+  const idByName = tokenIds ? Object.fromEntries(tokenIds) : null;
   function buildRule(name) {
     const out = [];
     const value = gen(rules[name], out, null);
@@ -2267,8 +2388,8 @@ return ${value};
 }`;
     SOURCES[ruleIndex.get(name)] = body;
     const source = `return ${body}`;
-    const factory = new Function("F", "E", "P", "T", "R", "H", source);
-    return factory(FAIL, expect, progressError, stepsError, RULES, HELPERS);
+    const factory = new Function("F", "E", "P", "T", "R", "H", "I", source);
+    return factory(FAIL, expect, progressError, stepsError, RULES, HELPERS, idByName);
   }
   function progressError(op, at) {
     throw new Error(
@@ -2350,9 +2471,10 @@ function createParser(spec, options = {}) {
   for (const [name, method] of Object.entries(extraMethods || {})) engine.addMethod(name, method);
   const startRule = rules[start];
   const firsts = ll1 ? analysis.firsts : null;
-  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval });
+  const tokenIds = new Map(lexer.tokenNames.map((name, id) => [name, id]));
+  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval, tokenIds });
   engine.__jlGrammar = { planner };
-  const codegen = compile && unsafeEval ? createCodegen({ planner, rules, firsts, memo, maxSteps }) : null;
+  const codegen = compile && unsafeEval ? createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds }) : null;
   const compiledStart = codegen && codegen.compile(start);
   if (codegen && !engine.disableInterpretedOptimization) {
     for (const name of Object.keys(rules)) {
@@ -2795,12 +2917,13 @@ function emitModule(spec, options = {}) {
   const analysis = analyze(rules, { tokenNames: lexer.tokenNames });
   if (validate && analysis.errors.length > 0) throw new GrammarError(analysis.errors);
   const firsts = ll1 ? analysis.firsts : null;
+  const tokenIds = new Map(lexer.tokenNames.map((name, id) => [name, id]));
   const engine = { build: () => {
     throw new Error("not compiled during emit");
   }, run: () => {
   } };
-  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval: false });
-  const codegen = createCodegen({ planner, rules, firsts, memo, maxSteps });
+  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval: false, tokenIds });
+  const codegen = createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds });
   codegen.compile(start);
   const reached = [...codegen.ruleIndex.entries()].sort((a, b) => a[1] - b[1]);
   const unsupported = codegen.ORIGINS.map((origin, index) => ({ origin, index })).filter(({ origin }) => origin.kind === "unknownRule");
@@ -2943,6 +3066,14 @@ const H = [
 ${helpers.map((source, index) => `  /* ${index} */ ${source}`).join(",\n")}
 ]
 
+/**
+ * Name -> id, the same numbering \`lexer\` assigns. A generated \`alt\`'s switch
+ * dispatches on a token's id when it has one; this is only read as a fallback,
+ * for a hand-built token from \`parseTokens\` that bypassed this file's own
+ * lexer and so has no id of its own.
+ */
+const I = ${literal(Object.fromEntries(tokenIds))}
+
 ${ruleSources}
 
 const RULES = ${literal(Object.fromEntries(reached.map(([name, index]) => [name, index])))}

package/dist/cjs/runtime.js

@@ -442,7 +442,7 @@ function createLexer(defs, options = {}) {
     return id;
   };
   for (const def of defs) for (const mode of modesOf(def)) idOf(mode);
-  const compiled = defs.map((def) => {
+  const compiled = defs.map((def, id) => {
     if (!def.name) throw new Error("createLexer: every token definition needs a name");
     if (!def.pattern && !def.literal) throw new Error(`createLexer: token '${def.name}' needs a pattern or a literal`);
     if (def.pattern && def.literal) throw new Error(`createLexer: token '${def.name}' has both a pattern and a literal; pick one`);
@@ -457,6 +457,11 @@ function createLexer(defs, options = {}) {
     }
     return {
       name: def.name,
+      // The declaration index. `createParser` derives the identical mapping
+      // from `tokenNames` independently (same array, same order), so a
+      // combinator that resolves a `consume` target's id at build time agrees
+      // with what a token actually carries without the two ever comparing notes.
+      id,
       skip: !!def.skip,
       literal: def.literal,
       longerAlt: def.longerAlt || null,
@@ -555,14 +560,7 @@ function createLexer(defs, options = {}) {
         }
         if (!winner.skip) {
           const image = winner.literal !== void 0 ? winner.literal : winner === def && matched !== null ? matched : text.slice(offset, end);
-          tokens.push({
-            type: winner.name,
-            image,
-            start: offset,
-            end,
-            line: trackLines ? line : 0,
-            col: trackLines ? offset - lineStart + 1 : 0
-          });
+          tokens.push(trackLines ? { type: winner.name, image, start: offset, end, line, col: offset - lineStart + 1 } : { type: winner.name, id: winner.id, image, start: offset, end });
         }
         if (trackLines) {
           for (let j = offset; j < end; j++) {