meta-parser-generator 1.0.4 → 1.1.1

package/dist/utils.js

@@ -0,0 +1,185 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.printTree = exports.displayError = exports.checkGrammarAndTokens = exports.preprocessGrammar = exports.streamContext = void 0;
+const RED = '\x1B[0;31m';
+const YELLOW = '\x1B[1;33m';
+const NC = '\x1B[0m';
+function replaceInvisibleChars(v) {
+    v = v.replace(/\r/g, '⏎\r');
+    v = v.replace(/\n/g, '⏎\n');
+    v = v.replace(/\t/g, '⇥');
+    v = v.replace('\xa0', 'nbsp');
+    return v.replace(/[ ]/g, '␣');
+}
+function tokenPosition(token) {
+    const lineNumber = token.line_start;
+    const charNumber = token.column_start;
+    const end = charNumber + token.len;
+    return { lineNumber, charNumber, end };
+}
+function streamContext(token, firstToken, stream) {
+    const index = token.stream_index;
+    const firstTokenIndex = firstToken.stream_index;
+    const { lineNumber } = tokenPosition(token);
+    let lineNb = 1;
+    let streamIndex = 0;
+    let str = NC;
+    function char(v) {
+        if (streamIndex === index) {
+            return RED + replaceInvisibleChars(v) + NC;
+        }
+        if (streamIndex >= firstTokenIndex && streamIndex < index) {
+            return YELLOW + replaceInvisibleChars(v) + NC;
+        }
+        return v;
+    }
+    while (lineNb < (lineNumber + 4) && stream[streamIndex]) {
+        const v = stream[streamIndex].value;
+        if (v.match(/\n/)) {
+            lineNb++;
+            if (lineNb > (lineNumber + 3)) {
+                return str;
+            }
+            if (lineNb >= (lineNumber - 1)) {
+                str += `${char(v)}${String(`     ${lineNb}`).slice(-5)}: `;
+            }
+        }
+        else if (lineNb >= (lineNumber - 1)) {
+            if (streamIndex === 0) {
+                str += `\n${String(`     ${lineNb}`).slice(-5)}: `;
+            }
+            str += char(v);
+        }
+        streamIndex++;
+    }
+    return str;
+}
+exports.streamContext = streamContext;
+/**
+ * Displays a formatted error message for parsing failures
+ * @param stream - Token stream
+ * @param tokensDefinition - Token definitions
+ * @param grammar - Grammar rules
+ * @param bestFailure - The parse failure with details
+ */
+function displayError(stream, tokensDefinition, grammar, bestFailure) {
+    const sub_rules = grammar[bestFailure.type][bestFailure.sub_rule_index];
+    let rule = '';
+    const { token } = bestFailure;
+    const firstToken = bestFailure.first_token;
+    const positions = tokenPosition(token);
+    let failingToken = '';
+    for (let i = 0; i < sub_rules.length; i++) {
+        let sr = sub_rules[i];
+        if (tokensDefinition[sr] && tokensDefinition[sr].verbose) {
+            sr = tokensDefinition[sr].verbose.replace(/\s/g, '-');
+        }
+        if (i === bestFailure.sub_rule_token_index) {
+            rule += `${RED}${sr}${NC} `;
+            failingToken = `${sr}`;
+        }
+        else {
+            rule += `${YELLOW}${sr}${NC} `;
+        }
+    }
+    // If multiple alternatives failed at same position, show them
+    let expectedTokens = '';
+    if (bestFailure.best_failure_array && bestFailure.best_failure_array.length > 1) {
+        const uniqueExpected = new Set();
+        bestFailure.best_failure_array.forEach(f => {
+            var _a;
+            const expectedRule = grammar[f.type][f.sub_rule_index][f.sub_rule_token_index];
+            if (expectedRule) {
+                const verbose = ((_a = tokensDefinition[expectedRule]) === null || _a === void 0 ? void 0 : _a.verbose) || expectedRule;
+                uniqueExpected.add(verbose);
+            }
+        });
+        if (uniqueExpected.size > 1) {
+            expectedTokens = `\n  Expected one of: ${YELLOW}${Array.from(uniqueExpected).join(', ')}${NC}`;
+        }
+    }
+    throw new Error(`
+  ${RED}Parser error at line ${positions.lineNumber + 1} char ${positions.charNumber} to ${positions.end} ${NC}
+  Unexpected ${YELLOW}${replaceInvisibleChars(token.value)}${NC}
+  Best match was at rule ${bestFailure.type}[${bestFailure.sub_rule_index}][${bestFailure.sub_rule_token_index}] ${rule}
+  token "${YELLOW}${replaceInvisibleChars(token.value)}${NC}" (type:${token.type}) doesn't match rule item ${YELLOW}${failingToken}${NC}${expectedTokens}
+  Context:
+${streamContext(token, firstToken, stream)}
+`);
+}
+exports.displayError = displayError;
+function isRule(node) {
+    return typeof node.type === 'string';
+}
+/**
+ * Prints the AST tree structure to console
+ * @param node - Root AST node to print
+ * @param sp - Spacing/indentation string
+ */
+function printTree(node, sp) {
+    if (isRule(node)) {
+        console.log(`${sp}r ${node.type}(${node.sub_rule_index})`);
+        if (node.children) {
+            for (let i = 0; i < node.children.length; i++) {
+                printTree(node.children[i], `${sp}  `);
+            }
+        }
+    }
+    else {
+        console.log(`${sp}t ${node.type} ${node.value}`);
+    }
+}
+exports.printTree = printTree;
+/**
+ * Validates that grammar and token definitions don't have overlapping keys
+ * @param grammar - Grammar rules
+ * @param tokensDefinition - Token definitions
+ */
+function checkGrammarAndTokens(grammar, tokensDefinition) {
+    const gkeys = Object.keys(grammar);
+    const tkeys = Object.keys(tokensDefinition);
+    const intersection = gkeys.filter(n => tkeys.indexOf(n) > -1);
+    if (intersection.length > 0) {
+        throw new Error(`Grammar and token have keys in common: ${intersection}`);
+    }
+}
+exports.checkGrammarAndTokens = checkGrammarAndTokens;
+/**
+ * Preprocesses grammar rules to extract modifiers and aliases
+ * @param rules - Raw grammar rules
+ * @returns Processed grammar with parsed modifiers and metadata
+ */
+function preprocessGrammar(rules) {
+    return Object.keys(rules).reduce((accu, key) => {
+        accu[key] = rules[key].map((ruleItems) => ruleItems.map((ruleItem, index) => {
+            if (typeof ruleItem === 'function') {
+                return { function: true, value: ruleItem, optional: false, repeatable: false, leftRecursion: false };
+            }
+            const values = ruleItem.split(':');
+            let optional = false;
+            let repeatable = false;
+            let leftRecursion = false;
+            if (values[0].endsWith('?')) {
+                values[0] = values[0].substring(0, values[0].length - 1);
+                optional = true;
+            }
+            if (values[0].endsWith('*')) {
+                values[0] = values[0].substring(0, values[0].length - 1);
+                repeatable = true;
+            }
+            if (index === 0 && values[0] === key) {
+                leftRecursion = true;
+            }
+            return {
+                value: values[0],
+                alias: values[1],
+                optional,
+                repeatable,
+                leftRecursion,
+            };
+        }));
+        return accu;
+    }, {});
+}
+exports.preprocessGrammar = preprocessGrammar;
+//# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../utils.ts"],"names":[],"mappings":";;;AAEA,MAAM,GAAG,GAAG,YAAY,CAAC;AACzB,MAAM,MAAM,GAAG,YAAY,CAAC;AAC5B,MAAM,EAAE,GAAG,SAAS,CAAC;AAErB,SAAS,qBAAqB,CAAC,CAAS;IACtC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC1B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,aAAa,CAAC,KAAY;IACjC,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IACpC,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,CAAC;IACtC,MAAM,GAAG,GAAG,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC;IACnC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,aAAa,CAAC,KAAY,EAAE,UAAiB,EAAE,MAAe;IACrE,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC;IACjC,MAAM,eAAe,GAAG,UAAU,CAAC,YAAY,CAAC;IAChD,MAAM,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAE5C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,GAAG,GAAG,EAAE,CAAC;IAEb,SAAS,IAAI,CAAC,CAAS;QACrB,IAAI,WAAW,KAAK,KAAK,EAAE;YACzB,OAAO,GAAG,GAAG,qBAAqB,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;SAC5C;QACD,IAAI,WAAW,IAAI,eAAe,IAAI,WAAW,GAAG,KAAK,EAAE;YACzD,OAAO,MAAM,GAAG,qBAAqB,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;SAC/C;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,OAAO,MAAM,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,EAAE;QACvD,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC;QACpC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;YACjB,MAAM,EAAE,CAAC;YACT,IAAI,MAAM,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,EAAE;gBAC7B,OAAO,GAAG,CAAC;aACZ;YACD,IAAI,MAAM,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,EAAE;gBAC9B,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;aAC5D;SACF;aAAM,IAAI,MAAM,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,EAAE;YACrC,IAAI,WAAW,KAAK,CAAC,EAAE;gBACrB,GAAG,IAAI,KAAK,MAAM,CAAC,QAAQ,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;aACpD;YACD,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;SAChB;QACD,WAAW,EAAE,CAAC;KACf;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAoIC,sCAAa;AAlIf;;;;;;GAMG;AACH,SAAS,YAAY,CAAC,MAAe,EAAE,gBAAkC,EAAE,OAAgB,EAAE,WAAyB;IACpH,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;IACxE,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,MAAM,EAAE,KAAK,EAAE,GAAG,WAAW,CAAC;IAC9B,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC;IAC3C,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACzC,IAAI,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,gBAAgB,CAAC,EAAE,CAAC,IAAI,gBAAgB,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE;YACxD,EAAE,GAAG,gBAAgB,CAAC,EAAE,CAAC,CAAC,OAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;SACxD;QACD,IAAI,CAAC,KAAK,WAAW,CAAC,oBAAoB,EAAE;YAC1C,IAAI,IAAI,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;YAC5B,YAAY,GAAG,GAAG,EAAE,EAAE,CAAC;SACxB;aAAM;YACL,IAAI,IAAI,GAAG,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;SAChC;KACF;IAED,8DAA8D;IAC9D,IAAI,cAAc,GAAG,EAAE,CAAC;IACxB,IAAI,WAAW,CAAC,kBAAkB,IAAI,WAAW,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;QAC/E,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QACzC,WAAW,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;;YACzC,MAAM,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC;YAC/E,IAAI,YAAY,EAAE;gBAChB,MAAM,OAAO,GAAG,CAAA,MAAA,gBAAgB,CAAC,YAAY,CAAC,0CAAE,OAAO,KAAI,YAAY,CAAC;gBACxE,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;aAC7B;QACH,CAAC,CAAC,CAAC;QACH,IAAI,cAAc,CAAC,IAAI,GAAG,CAAC,EAAE;YAC3B,cAAc,GAAG,wBAAwB,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SAChG;KACF;IAED,MAAM,IAAI,KAAK,CAAC;IACd,GAAG,wBAAwB,SAAS,CAAC,UAAU,GAAG,CAAC,SAAS,SAAS,CAAC,UAAU,OAAO,SAAS,CAAC,GAAG,IAAI,EAAE;eAC/F,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;2BACpC,WAAW,CAAC,IAAI,IAAI,WAAW,CAAC,cAAc,KAAK,WAAW,CAAC,oBAAoB,KAAK,IAAI;WAC5G,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,WAAW,KAAK,CAAC,IAAI,6BAA6B,MAAM,GAAG,YAAY,GAAG,EAAE,GAAG,cAAc;;EAEtJ,aAAa,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC;CACzC,CAAC,CAAC;AACH,CAAC;AAkFC,oCAAY;AAhFd,SAAS,MAAM,CAAC,IAAS;IACvB,OAAO,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAA;AACtC,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAa,EAAE,EAAU;IAC1C,IAAG,MAAM,CAAC,IAAI,CAAC,EAAE;QACf,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;QAC3D,IAAI,IAAI,CAAC,QAAQ,EAAE;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC7C,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;aACxC;SACF;KACF;SAAM;QACL,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;KAClD;AACH,CAAC;AA6DC,8BAAS;AA3DX;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,OAAgB,EAAE,gBAAkC;IACjF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC5C,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC9D,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;QAC3B,MAAM,IAAI,KAAK,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAC;KAC3E;AACH,CAAC;AA6CC,sDAAqB;AA3CvB;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,IAAsB,EAAE,GAAG,EAAE,EAAE;QAC/D,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CACxB,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;YAC/C,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE;gBAClC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,QAAe,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;aAC7G;YACD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,IAAI,UAAU,GAAG,KAAK,CAAC;YACvB,IAAI,aAAa,GAAG,KAAK,CAAC;YAC1B,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBAC3B,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACzD,QAAQ,GAAG,IAAI,CAAC;aACjB;YACD,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBAC3B,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACzD,UAAU,GAAG,IAAI,CAAC;aACnB;YACD,IAAI,KAAK,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE;gBACpC,aAAa,GAAG,IAAI,CAAC;aACtB;YACD,OAAO;gBACL,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;gBAChB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;gBAChB,QAAQ;gBACR,UAAU;gBACV,aAAa;aACd,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC,EAAE,EAAE,CAAC,CAAC;AACT,CAAC;AAIC,8CAAiB"}

package/package.json

@@ -1,9 +1,17 @@
 {
   "name": "meta-parser-generator",
-  "version": "1.0.4",
-  "description": "A peg parser generator written in JavaScript for JavaScript",
-  "main": "metaParserGenerator.js",
+  "version": "1.1.1",
+  "description": "A PEG parser generator with packrat parsing written in JavaScript for JavaScript",
+  "main": "dist/metaParserGenerator.js",
+  "types": "dist/metaParserGenerator.d.ts",
+  "files": [
+    "dist/",
+    "README.md",
+    "LICENSE"
+  ],
   "scripts": {
+    "build": "tsc",
+    "prepublish": "npm run build",
     "test": "jest --no-cache",
     "gentest": "node ./tests/generateParser.js && jest --no-cache"
   },
@@ -24,6 +32,7 @@
   },
   "homepage": "https://github.com/batiste/meta-parser-generator#readme",
   "devDependencies": {
+    "@types/node": "^18.13.0",
     "jest": "^28.1.3"
   }
 }

package/.editorconfig

@@ -1,8 +0,0 @@
-
-[*.js]
-indent_style = space
-indent_size = 2
-
-[*.blop]
-indent_style = space
-indent_size = 2

package/.eslintrc.js

@@ -1,26 +0,0 @@
-module.exports = {
-  "extends": "airbnb-base",
-  "globals": {
-    "window": true,
-    "test": true,
-    "expect": true,
-  },
-  "rules": {
-    "no-plusplus": "off",
-    "no-unused-expressions": "off",
-    "quote-props": "off",
-    "camelcase": "off",
-    "no-underscore-dangle": "off",
-    "no-param-reassign": "off",
-    "no-shadow": "off",
-    // for the generated parser
-    "consistent-return": "off",
-    "object-property-newline": "off",
-    "dot-notation": "off",
-    "no-use-before-define": "off",
-    "no-console": ["error", { allow: ["log", "error", "warn"] }]
-  },
-  "plugins": [
-    "ie11"
-  ]
-};

package/metaParserGenerator.js

@@ -1,312 +0,0 @@
-
-const fs = require('fs');
-// const path = require('path');
-const { preprocessGrammar, checkGrammarAndTokens } = require('./utils');
-
-const recordFailure = `
-let best_failure;
-let best_failure_array = [];
-let best_failure_index = 0;
-
-function record_failure(failure, i) {
-  if (i > best_failure_index) {
-    best_failure_array = [];
-  }
-  if (best_failure_array.length === 0) {
-    best_failure = failure;
-  }
-  best_failure_array.push(failure);
-  best_failure_index = i;
-}
-
-let cache = {};
-
-function memoize(name, func) {
-  return function memoize_inner(stream, index) {
-    const key = \`\${name}-\${index}\`;
-    let value = cache[key];
-    if (value !== undefined) {
-      return value;
-    }
-    value = func(stream, index);
-    cache[key] = value;
-    return value;
-  };
-}
-
-let cacheR = {};
-
-// based on https://medium.com/@gvanrossum_83706/left-recursive-peg-grammars-65dab3c580e1
-function memoize_left_recur(name, func) {
-  return function memoize_inner(stream, index) {
-    const key = \`\${name}-\${index}\`;
-    let value = cacheR[key];
-    if (value !== undefined) {
-      return value;
-    }
-    // prime this rule with a failure
-    cacheR[key] = false;
-    let lastpos;
-    let lastvalue = value;
-    while (true) {
-      value = func(stream, index);
-      if (!value) break;
-      if (value.last_index <= lastpos) break;
-      lastpos = value.last_index;
-      lastvalue = value;
-      cacheR[key] = value;
-    }
-    return lastvalue;
-  };
-}
-
-`;
-
-function generateTokenizer(tokenDef) {
-  const output = [];
-  const keys = Object.keys(tokenDef);
-  for (let i = 0; i < keys.length; i++) {
-    const key = keys[i];
-    if ((/:|\?/g).test(key)) {
-      throw new Error('Reserved word in token name');
-    }
-  }
-
-  output.push('function _tokenize(tokenDef, input, stream) {');
-  output.push('  let match;');
-  let key;
-  for (let i = 0; i < keys.length; i++) {
-    key = keys[i];
-    const token = tokenDef[key];
-    if (token.str) {
-      if (token.str.indexOf("'") > -1 || token.str.indexOf('\n') > -1) {
-        output.push(`  if (input.startsWith(\`${token.str}\`)) {`);
-        output.push(`    return [\`${token.str}\`, '${key}'];`);
-      } else {
-        output.push(`  if (input.startsWith('${token.str}')) {`);
-        output.push(`    return ['${token.str}', '${key}'];`);
-      }
-      output.push('  }');
-    } else if (token.reg) {
-      output.push(`  match = input.match(tokenDef.${key}.reg);`);
-      output.push('  if (match !== null) {');
-      output.push(`    return [match[0], '${key}'];`);
-      output.push('  }');
-    } else if (token.func) {
-      output.push(`  match = tokenDef.${key}.func(input, stream);`);
-      output.push('  if (match !== undefined) {');
-      output.push(`    return [match, '${key}'];`);
-      output.push('  }');
-    } else {
-      throw new Error(`Tokenizer error: Invalid token ${key} without a reg, str or func property`);
-    }
-  }
-  output.push(`  return [null, '${key}'];`);
-  output.push('}');
-
-  output.push('function tokenize(tokenDef, input) {');
-  output.push(`  const stream = [];
-  let lastToken;
-  let key;
-  let candidate = null;
-  const len = input.length;
-  let char = 0;
-  let index = 0;
-  let line = 0;
-  let column = 0;
-  while (char < len) {
-    [candidate, key] = _tokenize(tokenDef, input, stream);
-    if (candidate !== null) {
-      lastToken = {
-        type: key,
-        value: candidate,
-        start: char,
-        stream_index: index,
-        len: candidate.length,
-        lineStart: line,
-        columnStart: column,
-      };
-      const lines = candidate.split('\\n');
-      if (lines.length > 1) {
-        line += lines.length - 1;
-        column = lines[lines.length - 1].length;
-      } else {
-        column += candidate.length;
-      }
-      lastToken.lineEnd = line;
-      lastToken.columnEnd = column;
-      stream.push(lastToken);
-      index++;
-      char += candidate.length;
-      input = input.substr(candidate.length);
-    } else {
-      if (stream.length === 0) {
-        throw new Error('Tokenizer error: total match failure');
-      }
-      if (lastToken) {
-        lastToken.pointer += lastToken.value.length;
-      }
-      let msg = \`Tokenizer error, no matching token found for \${input.slice(0, 26)}\`;
-      if (lastToken) {
-        msg += \`Before token of type \${lastToken.type}: \${lastToken.value}\`;
-      }
-      const error = new Error(msg);
-      error.token = lastToken;
-      throw error;
-    }
-  }
-  stream.push({
-    type: 'EOS', value: '<End Of Stream>', char, index,
-  });
-  return stream;
-}
-`);
-  return output;
-}
-
-function generateSubRule(name, index, subRule, tokensDef, debug) {
-  const output = [];
-  output.push(`let ${name}_${index} = (stream, index) => {`);
-  let i = 0;
-  output.push('  let i = index;');
-  output.push('  const children = [];');
-  output.push('  const named = {};');
-  output.push(`  const node = {
-    children, stream_index: index, name: '${name}',
-    subRule: ${index}, type: '${name}', named,
-  };`);
-  subRule.forEach((rule) => {
-    // terminal rule
-    if (tokensDef[rule.value] || rule.value === 'EOS') {
-      debug ? output.push('  console.log(i, stream[i])') : null;
-      if (rule.repeatable) {
-        output.push(`  while(stream[i].type === '${rule.value}') {`);
-        if (rule.alias) {
-          output.push(`    named['${rule.alias}'] ? null : named['${rule.alias}'] = []`);
-          output.push(`    named['${rule.alias}'].push(stream[i])`);
-        }
-        output.push('    children.push(stream[i]); i++;');
-        output.push('  }');
-      } else if (rule.optional) {
-        output.push(`  if (stream[i].type === '${rule.value}') {`);
-        rule.alias ? output.push(`    named['${rule.alias}'] = stream[i];`) : null;
-        output.push('    children.push(stream[i]); i++;');
-        output.push('  }');
-      } else {
-        output.push(`
-  if (stream[i].type !== '${rule.value}') {
-    if (i >= best_failure_index) {
-      const failure = {
-        rule_name: '${name}', sub_rule_index: ${index},
-        sub_rule_stream_index: i - index, sub_rule_token_index: ${i},
-        stream_index: i, token: stream[i], first_token: stream[index], success: false,
-      };
-      record_failure(failure, i);
-    }
-    return false;
-  }
-`);
-        rule.alias ? output.push(`  named['${rule.alias}'] = stream[i];`) : null;
-        output.push('  children.push(stream[i]); i++;');
-      }
-      i++;
-    // calling another rule in the grammar
-    } else {
-      if (rule.function) {
-        output.push(`  if (!(${rule.value})(node)) { return false; }`);
-      } else if (rule.repeatable) {
-        output.push(`  let _rule_${i} = ${rule.value}(stream, i);`); // doing the call
-        output.push(`  while (_rule_${i}) {`);
-        if (rule.alias) {
-          output.push(`    named['${rule.alias}'] ? null : named['${rule.alias}'] = [];`);
-          output.push(`    named['${rule.alias}'].push(_rule_${i});`);
-        }
-        output.push(`    children.push(_rule_${i});`);
-        output.push(`    i = _rule_${i}.last_index;`);
-        output.push(`    _rule_${i} = ${rule.value}(stream, i);`);
-        output.push('  }');
-      } else if (!rule.optional) {
-        output.push(`  const _rule_${i} = ${rule.value}(stream, i);`); // doing the call
-        output.push(`  if (!_rule_${i}) return false;`);
-        rule.alias ? output.push(`  named['${rule.alias}'] = _rule_${i};`) : null;
-        output.push(`  children.push(_rule_${i});`);
-        output.push(`  i = _rule_${i}.last_index;`);
-      } else {
-        output.push(`  const _rule_${i} = ${rule.value}(stream, i);`); // doing the call
-        output.push(`  if (_rule_${i}) {`);
-        output.push(`    children.push(_rule_${i});`);
-        rule.alias ? output.push(`    named['${rule.alias}'] = _rule_${i};`) : null;
-        output.push(`    i = _rule_${i}.last_index;`);
-        output.push('  }');
-      }
-      i++;
-    }
-  });
-  output.push('  node.success = i === stream.length; node.last_index = i;');
-  output.push('  return node;');
-  output.push('};');
-  if (subRule[0].leftRecursion) {
-    output.push(`${name}_${index} = memoize_left_recur('${name}_${index}', ${name}_${index});`);
-  } else {
-    output.push(`${name}_${index} = memoize('${name}_${index}', ${name}_${index});`);
-  }
-  output.push('\n');
-  return output;
-}
-
-function generate(grammar, tokensDef, debug) {
-  let output = [];
-  checkGrammarAndTokens(grammar, tokensDef);
-  const newGrammar = preprocessGrammar(grammar);
-  const entries = Object.keys(newGrammar);
-  output.push('// This code is automatically generated by the meta parser, do not modify');
-  output.push('// produced with metaParserGenerator.js');
-  output.push(recordFailure);
-  entries.forEach((key) => {
-    let i = 0;
-    const metaSub = [];
-    newGrammar[key].forEach((subRule) => {
-      output = output.concat(generateSubRule(key, i, subRule, tokensDef, debug));
-      metaSub.push(`${key}_${i}`);
-      i++;
-    });
-    output.push(`function ${key}(stream, index) {`);
-    const st = metaSub.map(sub => `${sub}(stream, index)`).join('\n    || ');
-    output.push(`  return ${st};`);
-    output.push('}');
-  });
-  output = output.concat(generateTokenizer(tokensDef));
-  output.push(`module.exports = {
-  parse: (stream) => {
-    best_failure = null;
-    best_failure_index = 0;
-    best_failure_array = [];
-    cache = {};
-    cacheR = {};
-    const result = START(stream, 0);
-    if (!result) {
-      return best_failure;
-    }
-    return result;
-  },
-  tokenize,
-};
-`);
-  return output;
-}
-
-function generateParser(grammar, tokensDefinition, filename) {
-  fs.writeFileSync(filename,
-    generate(grammar, tokensDefinition, false).join('\n'), (err) => {
-      if (err) {
-        // eslint-disable-next-line no-console
-        console.log(err);
-      }
-    });
-}
-
-module.exports = {
-  generateParser,
-  generate,
-  generateTokenizer,
-};

package/tests/generateParser.js

@@ -1,9 +0,0 @@
-
-const path = require('path');
-const { grammar } = require('./grammar');
-const { tokensDefinition } = require('./tokensDefinition');
-const { generateParser } = require('../metaParserGenerator');
-
-generateParser(grammar, tokensDefinition, path.resolve(__dirname, './parser.js'));
-
-console.log('parser generated');

package/tests/grammar.js

@@ -1,23 +0,0 @@
-
-const grammar = {
-  'START': [
-    // necessary to accept the firt line to not be a newline
-    ['GLOBAL_STATEMENT', 'GLOBAL_STATEMENTS*', 'EOS'],
-    ['GLOBAL_STATEMENTS*', 'EOS'],
-  ],
-  'GLOBAL_STATEMENTS': [
-    ['newline', 'GLOBAL_STATEMENT'],
-    ['newline'],
-  ],
-  'GLOBAL_STATEMENT': [
-    ['math_operation'],
-  ],
-  'math_operation': [
-    ['math_operation', 'math_operator', 'number'],
-    ['number'],
-  ],
-};
-
-module.exports = {
-  grammar,
-};