meta-parser-generator 1.0.4 → 1.1.1

package/tests/parser.js

@@ -1,445 +0,0 @@
-// This code is automatically generated by the meta parser, do not modify
-// produced with metaParserGenerator.js
-
-let best_failure;
-let best_failure_array = [];
-let best_failure_index = 0;
-
-function record_failure(failure, i) {
-  if (i > best_failure_index) {
-    best_failure_array = [];
-  }
-  if (best_failure_array.length === 0) {
-    best_failure = failure;
-  }
-  best_failure_array.push(failure);
-  best_failure_index = i;
-}
-
-let cache = {};
-
-function memoize(name, func) {
-  return function memoize_inner(stream, index) {
-    const key = `${name}-${index}`;
-    let value = cache[key];
-    if (value !== undefined) {
-      return value;
-    }
-    value = func(stream, index);
-    cache[key] = value;
-    return value;
-  };
-}
-
-let cacheR = {};
-
-// based on https://medium.com/@gvanrossum_83706/left-recursive-peg-grammars-65dab3c580e1
-function memoize_left_recur(name, func) {
-  return function memoize_inner(stream, index) {
-    const key = `${name}-${index}`;
-    let value = cacheR[key];
-    if (value !== undefined) {
-      return value;
-    }
-    // prime this rule with a failure
-    cacheR[key] = false;
-    let lastpos;
-    let lastvalue = value;
-    while (true) {
-      value = func(stream, index);
-      if (!value) break;
-      if (value.last_index <= lastpos) break;
-      lastpos = value.last_index;
-      lastvalue = value;
-      cacheR[key] = value;
-    }
-    return lastvalue;
-  };
-}
-
-
-let START_0 = (stream, index) => {
-  let i = index;
-  const children = [];
-  const named = {};
-  const node = {
-    children, stream_index: index, name: 'START',
-    subRule: 0, type: 'START', named,
-  };
-  const _rule_0 = GLOBAL_STATEMENT(stream, i);
-  if (!_rule_0) return false;
-  children.push(_rule_0);
-  i = _rule_0.last_index;
-  let _rule_1 = GLOBAL_STATEMENTS(stream, i);
-  while (_rule_1) {
-    children.push(_rule_1);
-    i = _rule_1.last_index;
-    _rule_1 = GLOBAL_STATEMENTS(stream, i);
-  }
-
-  if (stream[i].type !== 'EOS') {
-    if (i >= best_failure_index) {
-      const failure = {
-        rule_name: 'START', sub_rule_index: 0,
-        sub_rule_stream_index: i - index, sub_rule_token_index: 2,
-        stream_index: i, token: stream[i], first_token: stream[index], success: false,
-      };
-      record_failure(failure, i);
-    }
-    return false;
-  }
-
-  children.push(stream[i]); i++;
-  node.success = i === stream.length; node.last_index = i;
-  return node;
-};
-START_0 = memoize('START_0', START_0);
-
-
-let START_1 = (stream, index) => {
-  let i = index;
-  const children = [];
-  const named = {};
-  const node = {
-    children, stream_index: index, name: 'START',
-    subRule: 1, type: 'START', named,
-  };
-  let _rule_0 = GLOBAL_STATEMENTS(stream, i);
-  while (_rule_0) {
-    children.push(_rule_0);
-    i = _rule_0.last_index;
-    _rule_0 = GLOBAL_STATEMENTS(stream, i);
-  }
-
-  if (stream[i].type !== 'EOS') {
-    if (i >= best_failure_index) {
-      const failure = {
-        rule_name: 'START', sub_rule_index: 1,
-        sub_rule_stream_index: i - index, sub_rule_token_index: 1,
-        stream_index: i, token: stream[i], first_token: stream[index], success: false,
-      };
-      record_failure(failure, i);
-    }
-    return false;
-  }
-
-  children.push(stream[i]); i++;
-  node.success = i === stream.length; node.last_index = i;
-  return node;
-};
-START_1 = memoize('START_1', START_1);
-
-
-function START(stream, index) {
-  return START_0(stream, index)
-    || START_1(stream, index);
-}
-let GLOBAL_STATEMENTS_0 = (stream, index) => {
-  let i = index;
-  const children = [];
-  const named = {};
-  const node = {
-    children, stream_index: index, name: 'GLOBAL_STATEMENTS',
-    subRule: 0, type: 'GLOBAL_STATEMENTS', named,
-  };
-
-  if (stream[i].type !== 'newline') {
-    if (i >= best_failure_index) {
-      const failure = {
-        rule_name: 'GLOBAL_STATEMENTS', sub_rule_index: 0,
-        sub_rule_stream_index: i - index, sub_rule_token_index: 0,
-        stream_index: i, token: stream[i], first_token: stream[index], success: false,
-      };
-      record_failure(failure, i);
-    }
-    return false;
-  }
-
-  children.push(stream[i]); i++;
-  const _rule_1 = GLOBAL_STATEMENT(stream, i);
-  if (!_rule_1) return false;
-  children.push(_rule_1);
-  i = _rule_1.last_index;
-  node.success = i === stream.length; node.last_index = i;
-  return node;
-};
-GLOBAL_STATEMENTS_0 = memoize('GLOBAL_STATEMENTS_0', GLOBAL_STATEMENTS_0);
-
-
-let GLOBAL_STATEMENTS_1 = (stream, index) => {
-  let i = index;
-  const children = [];
-  const named = {};
-  const node = {
-    children, stream_index: index, name: 'GLOBAL_STATEMENTS',
-    subRule: 1, type: 'GLOBAL_STATEMENTS', named,
-  };
-
-  if (stream[i].type !== 'newline') {
-    if (i >= best_failure_index) {
-      const failure = {
-        rule_name: 'GLOBAL_STATEMENTS', sub_rule_index: 1,
-        sub_rule_stream_index: i - index, sub_rule_token_index: 0,
-        stream_index: i, token: stream[i], first_token: stream[index], success: false,
-      };
-      record_failure(failure, i);
-    }
-    return false;
-  }
-
-  children.push(stream[i]); i++;
-  node.success = i === stream.length; node.last_index = i;
-  return node;
-};
-GLOBAL_STATEMENTS_1 = memoize('GLOBAL_STATEMENTS_1', GLOBAL_STATEMENTS_1);
-
-
-function GLOBAL_STATEMENTS(stream, index) {
-  return GLOBAL_STATEMENTS_0(stream, index)
-    || GLOBAL_STATEMENTS_1(stream, index);
-}
-let GLOBAL_STATEMENT_0 = (stream, index) => {
-  let i = index;
-  const children = [];
-  const named = {};
-  const node = {
-    children, stream_index: index, name: 'GLOBAL_STATEMENT',
-    subRule: 0, type: 'GLOBAL_STATEMENT', named,
-  };
-  const _rule_0 = math_operation(stream, i);
-  if (!_rule_0) return false;
-  children.push(_rule_0);
-  i = _rule_0.last_index;
-  node.success = i === stream.length; node.last_index = i;
-  return node;
-};
-GLOBAL_STATEMENT_0 = memoize('GLOBAL_STATEMENT_0', GLOBAL_STATEMENT_0);
-
-
-function GLOBAL_STATEMENT(stream, index) {
-  return GLOBAL_STATEMENT_0(stream, index);
-}
-let math_operation_0 = (stream, index) => {
-  let i = index;
-  const children = [];
-  const named = {};
-  const node = {
-    children, stream_index: index, name: 'math_operation',
-    subRule: 0, type: 'math_operation', named,
-  };
-  const _rule_0 = math_operation(stream, i);
-  if (!_rule_0) return false;
-  children.push(_rule_0);
-  i = _rule_0.last_index;
-
-  if (stream[i].type !== 'math_operator') {
-    if (i >= best_failure_index) {
-      const failure = {
-        rule_name: 'math_operation', sub_rule_index: 0,
-        sub_rule_stream_index: i - index, sub_rule_token_index: 1,
-        stream_index: i, token: stream[i], first_token: stream[index], success: false,
-      };
-      record_failure(failure, i);
-    }
-    return false;
-  }
-
-  children.push(stream[i]); i++;
-
-  if (stream[i].type !== 'number') {
-    if (i >= best_failure_index) {
-      const failure = {
-        rule_name: 'math_operation', sub_rule_index: 0,
-        sub_rule_stream_index: i - index, sub_rule_token_index: 2,
-        stream_index: i, token: stream[i], first_token: stream[index], success: false,
-      };
-      record_failure(failure, i);
-    }
-    return false;
-  }
-
-  children.push(stream[i]); i++;
-  node.success = i === stream.length; node.last_index = i;
-  return node;
-};
-math_operation_0 = memoize_left_recur('math_operation_0', math_operation_0);
-
-
-let math_operation_1 = (stream, index) => {
-  let i = index;
-  const children = [];
-  const named = {};
-  const node = {
-    children, stream_index: index, name: 'math_operation',
-    subRule: 1, type: 'math_operation', named,
-  };
-
-  if (stream[i].type !== 'number') {
-    if (i >= best_failure_index) {
-      const failure = {
-        rule_name: 'math_operation', sub_rule_index: 1,
-        sub_rule_stream_index: i - index, sub_rule_token_index: 0,
-        stream_index: i, token: stream[i], first_token: stream[index], success: false,
-      };
-      record_failure(failure, i);
-    }
-    return false;
-  }
-
-  children.push(stream[i]); i++;
-  node.success = i === stream.length; node.last_index = i;
-  return node;
-};
-math_operation_1 = memoize('math_operation_1', math_operation_1);
-
-
-function math_operation(stream, index) {
-  return math_operation_0(stream, index)
-    || math_operation_1(stream, index);
-}
-function _tokenize(tokenDef, input, stream) {
-  let match;
-  match = input.match(tokenDef.number.reg);
-  if (match !== null) {
-    return [match[0], 'number'];
-  }
-  match = input.match(tokenDef.comment.reg);
-  if (match !== null) {
-    return [match[0], 'comment'];
-  }
-  match = input.match(tokenDef.multiline_comment.reg);
-  if (match !== null) {
-    return [match[0], 'multiline_comment'];
-  }
-  if (input.startsWith(',')) {
-    return [',', ','];
-  }
-  if (input.startsWith('.')) {
-    return ['.', '.'];
-  }
-  if (input.startsWith('(')) {
-    return ['(', '('];
-  }
-  if (input.startsWith(')')) {
-    return [')', ')'];
-  }
-  if (input.startsWith('{')) {
-    return ['{', '{'];
-  }
-  if (input.startsWith('}')) {
-    return ['}', '}'];
-  }
-  if (input.startsWith('>')) {
-    return ['>', '>'];
-  }
-  if (input.startsWith('<')) {
-    return ['<', '<'];
-  }
-  match = input.match(tokenDef.name.reg);
-  if (match !== null) {
-    return [match[0], 'name'];
-  }
-  match = input.match(tokenDef.math_operator.reg);
-  if (match !== null) {
-    return [match[0], 'math_operator'];
-  }
-  if (input.startsWith('!')) {
-    return ['!', 'unary'];
-  }
-  if (input.startsWith('=')) {
-    return ['=', '='];
-  }
-  if (input.startsWith(':')) {
-    return [':', 'colon'];
-  }
-  if (input.startsWith(`
-`)) {
-    return [`
-`, 'newline'];
-  }
-  match = tokenDef.str.func(input, stream);
-  if (match !== undefined) {
-    return [match, 'str'];
-  }
-  match = tokenDef.w.func(input, stream);
-  if (match !== undefined) {
-    return [match, 'w'];
-  }
-  match = input.match(tokenDef.W.reg);
-  if (match !== null) {
-    return [match[0], 'W'];
-  }
-  return [null, 'W'];
-}
-function tokenize(tokenDef, input) {
-  const stream = [];
-  let lastToken;
-  let key;
-  let candidate = null;
-  const len = input.length;
-  let char = 0;
-  let index = 0;
-  let line = 0;
-  let column = 0;
-  while (char < len) {
-    [candidate, key] = _tokenize(tokenDef, input, stream);
-    if (candidate !== null) {
-      lastToken = {
-        type: key,
-        value: candidate,
-        start: char,
-        stream_index: index,
-        len: candidate.length,
-        lineStart: line,
-        columnStart: column,
-      };
-      const lines = candidate.split('\n');
-      if (lines.length > 1) {
-        line += lines.length - 1;
-        column = lines[lines.length - 1].length;
-      } else {
-        column += candidate.length;
-      }
-      lastToken.lineEnd = line;
-      lastToken.columnEnd = column;
-      stream.push(lastToken);
-      index++;
-      char += candidate.length;
-      input = input.substr(candidate.length);
-    } else {
-      if (stream.length === 0) {
-        throw new Error('Tokenizer error: total match failure');
-      }
-      if (lastToken) {
-        lastToken.pointer += lastToken.value.length;
-      }
-      let msg = `Tokenizer error, no matching token found for ${input.slice(0, 26)}`;
-      if (lastToken) {
-        msg += `Before token of type ${lastToken.type}: ${lastToken.value}`;
-      }
-      const error = new Error(msg);
-      error.token = lastToken;
-      throw error;
-    }
-  }
-  stream.push({
-    type: 'EOS', value: '<End Of Stream>', char, index,
-  });
-  return stream;
-}
-
-module.exports = {
-  parse: (stream) => {
-    best_failure = null;
-    best_failure_index = 0;
-    best_failure_array = [];
-    cache = {};
-    cacheR = {};
-    const result = START(stream, 0);
-    if (!result) {
-      return best_failure;
-    }
-    return result;
-  },
-  tokenize,
-};

package/tests/test.js

@@ -1,31 +0,0 @@
-const parser = require('./parser');
-const { tokensDefinition } = require('./tokensDefinition');
-const { grammar } = require('./grammar');
-const { displayError } = require('../utils');
-
-function parse(code) {
-  const tokens = parser.tokenize(tokensDefinition, code);
-  const ast = parser.parse(tokens);
-  if (!ast.success && process.env.DEBUG) {
-    displayError(tokens, tokensDefinition, grammar, ast);
-  }
-  return ast;
-}
-
-test('correct grammar', () => {
-  const code = `9+9
-1+1+2
-`;
-  const result = parse(code);
-  expect(result.success).toBe(true);
-});
-
-test('incorrect', () => {
-  const code = `9+9
-1+1
-1+2
-1 + 4
-`;
-  const result = parse(code);
-  expect(result.success).toBe(false);
-});

package/tests/tokensDefinition.js

@@ -1,50 +0,0 @@
-
-function strDef(input) {
-  let i; let ch;
-  const first = input.charAt(0);
-  if (first === '"' || first === "'" || first === '`') {
-    i = 1;
-    while (input.charAt(i)) {
-      ch = input.charAt(i);
-      if (ch === '\\') {
-        i++;
-      } else if (ch === first) {
-        return input.slice(0, i + 1);
-      }
-      i++;
-    }
-  }
-}
-
-function singleSpace(input) {
-  if (input[0] === ' ' && input[1] !== ' ') {
-    return ' ';
-  }
-}
-
-const tokensDefinition = {
-  'number': { reg: /^[0-9]+(\.[0-9]*)?/ },
-  'comment': { reg: /^\/\/[^\n]*/, verbose: 'comment' },
-  'multiline_comment': { reg: /^\/\*+[^*]*\*+(?:[^/*][^*]*\*+)*\//, verbose: 'comment' },
-  ',': { str: ',' },
-  '.': { str: '.' },
-  '(': { str: '(' },
-  ')': { str: ')' },
-  '{': { str: '{' },
-  '}': { str: '}' },
-  '>': { str: '>' },
-  '<': { str: '<' },
-  'name': { reg: /^[\w|$|_]+/ },
-  'math_operator': { reg: /^(\+|\/|-|\*|\^|~|%)/ },
-  'unary': { str: '!' },
-  '=': { str: '=' },
-  'colon': { str: ':' },
-  'newline': { str: '\n' },
-  'str': { func: strDef, verbose: 'string' },
-  'w': { func: singleSpace, verbose: 'single white space' },
-  'W': { reg: /^[\s]+/, verbose: 'multiple white spaces' },
-};
-
-module.exports = {
-  tokensDefinition,
-};

package/utils.js

@@ -1,153 +0,0 @@
-const RED = '\x1B[0;31m';
-const YELLOW = '\x1B[1;33m';
-const NC = '\x1B[0m';
-
-function replaceInvisibleChars(v) {
-  v = v.replace(/\r/g, '⏎\r');
-  v = v.replace(/\n/g, '⏎\n');
-  v = v.replace(/\t/g, '⇥');
-  v = v.replace('\xa0', 'nbsp');
-  return v.replace(/[ ]/g, '␣');
-}
-
-function tokenPosition(token) {
-  const lineNumber = token.lineStart;
-  const charNumber = token.columnStart;
-  const end = charNumber + token.len;
-  return { lineNumber, charNumber, end };
-}
-
-function streamContext(token, firstToken, stream) {
-  const index = token.stream_index;
-  const firstTokenIndex = firstToken.stream_index;
-  const { lineNumber } = tokenPosition(token);
-
-  let lineNb = 1;
-  let streamIndex = 0;
-  let str = NC;
-
-  function char(v) {
-    if (streamIndex === index) {
-      return RED + replaceInvisibleChars(v) + NC;
-    }
-    if (streamIndex >= firstTokenIndex && streamIndex < index) {
-      return YELLOW + replaceInvisibleChars(v) + NC;
-    }
-    return v;
-  }
-
-  while (lineNb < (lineNumber + 4) && stream[streamIndex]) {
-    const v = stream[streamIndex].value;
-    if (v.match(/\n/)) {
-      lineNb++;
-      if (lineNb > (lineNumber + 3)) {
-        return str;
-      }
-      if (lineNb >= (lineNumber - 1)) {
-        str += `${char(v)}${String(`     ${lineNb}`).slice(-5)}: `;
-      }
-    } else if (lineNb >= (lineNumber - 1)) {
-      if (streamIndex === 0) {
-        str += `\n${String(`     ${lineNb}`).slice(-5)}: `;
-      }
-      str += char(v);
-    }
-    streamIndex++;
-  }
-  return str;
-}
-
-function displayError(stream, tokensDefinition, grammar, bestFailure) {
-  const sub_rules = grammar[bestFailure.rule_name][bestFailure.sub_rule_index];
-  let rule = '';
-  const { token } = bestFailure;
-  const firstToken = bestFailure.first_token;
-  const positions = tokenPosition(token);
-  let failingToken = '';
-  for (let i = 0; i < sub_rules.length; i++) {
-    let sr = sub_rules[i];
-    if (tokensDefinition[sr] && tokensDefinition[sr].verbose) {
-      sr = tokensDefinition[sr].verbose.replace(/\s/g, '-');
-    }
-    if (i === bestFailure.sub_rule_token_index) {
-      rule += `${RED}${sr}${NC} `;
-      failingToken = `${sr}`;
-    } else {
-      rule += `${YELLOW}${sr}${NC} `;
-    }
-  }
-  throw new Error(`
-  ${RED}Parser error at line ${positions.lineNumber + 1} char ${positions.charNumber} to ${positions.end} ${NC}
-  Unexpected ${YELLOW}${replaceInvisibleChars(token.value)}${NC}
-  Best match was at rule ${bestFailure.rule_name}[${bestFailure.sub_rule_index}][${bestFailure.sub_rule_token_index}] ${rule}
-  token "${YELLOW}${replaceInvisibleChars(token.value)}${NC}" (type:${token.type}) doesn't match rule item ${YELLOW}${failingToken}${NC}
-  Context:
-${streamContext(token, firstToken, stream)}
-`);
-}
-
-function printTree(node, sp) {
-  if (node.rule_name) {
-    console.log(`${sp}r ${node.rule_name}(${node.sub_rule_index})`);
-  } else {
-    console.log(`${sp}t ${node.type} ${node.value}`);
-  }
-
-  if (node.children) {
-    for (let i = 0; i < node.children.length; i++) {
-      printTree(node.children[i], `${sp}  `);
-    }
-  }
-}
-
-function checkGrammarAndTokens(grammar, tokensDefinition) {
-  const gkeys = Object.keys(grammar);
-  const tkeys = Object.keys(tokensDefinition);
-  const intersection = gkeys.filter(n => tkeys.indexOf(n) > -1);
-  if (intersection.length > 0) {
-    throw new Error(`Grammar and token have keys in common: ${intersection}`);
-  }
-}
-
-function preprocessGrammar(rules) {
-  return Object.keys(rules).reduce((accu, key) => {
-    accu[key] = rules[key].map(
-      subRule => subRule.map((subRuleItem, index) => {
-        if (subRuleItem instanceof Function) {
-          return { function: true, value: subRuleItem };
-        }
-        const values = subRuleItem.split(':');
-        let optional = false;
-        let repeatable = false;
-        let leftRecursion = false;
-        if (values[0].endsWith('?')) {
-          values[0] = values[0].substring(0, values[0].length - 1);
-          optional = true;
-        }
-        if (values[0].endsWith('*')) {
-          values[0] = values[0].substring(0, values[0].length - 1);
-          repeatable = true;
-        }
-        if (index === 0 && values[0] === key) {
-          leftRecursion = true;
-        }
-        return {
-          value: values[0],
-          alias: values[1],
-          optional,
-          repeatable,
-          leftRecursion,
-        };
-      }),
-    );
-    return accu;
-  }, {});
-}
-
-module.exports = {
-  streamContext,
-  preprocessGrammar,
-  checkGrammarAndTokens,
-  displayError,
-  printTree,
-};