python3-parser 1.0.0

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/ParserInterpreter.cpp

@@ -0,0 +1,306 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "dfa/DFA.h"
+#include "atn/RuleStartState.h"
+#include "InterpreterRuleContext.h"
+#include "atn/ParserATNSimulator.h"
+#include "ANTLRErrorStrategy.h"
+#include "atn/LoopEndState.h"
+#include "FailedPredicateException.h"
+#include "atn/StarLoopEntryState.h"
+#include "atn/AtomTransition.h"
+#include "atn/RuleTransition.h"
+#include "atn/PredicateTransition.h"
+#include "atn/PrecedencePredicateTransition.h"
+#include "atn/ActionTransition.h"
+#include "atn/ATN.h"
+#include "atn/RuleStopState.h"
+#include "Lexer.h"
+#include "Token.h"
+#include "Vocabulary.h"
+#include "InputMismatchException.h"
+#include "CommonToken.h"
+#include "tree/ErrorNode.h"
+
+#include "support/CPPUtils.h"
+
+#include "ParserInterpreter.h"
+
+using namespace antlr4;
+using namespace antlr4::atn;
+using namespace antlr4::misc;
+
+using namespace antlrcpp;
+
+ParserInterpreter::ParserInterpreter(const std::string &grammarFileName, const std::vector<std::string>& tokenNames,
+  const std::vector<std::string>& ruleNames, const atn::ATN &atn, TokenStream *input)
+  : ParserInterpreter(grammarFileName, dfa::Vocabulary::fromTokenNames(tokenNames), ruleNames, atn, input) {
+}
+
+ParserInterpreter::ParserInterpreter(const std::string &grammarFileName, const dfa::Vocabulary &vocabulary,
+  const std::vector<std::string> &ruleNames, const atn::ATN &atn, TokenStream *input)
+  : Parser(input), _grammarFileName(grammarFileName), _atn(atn), _ruleNames(ruleNames), _vocabulary(vocabulary) {
+
+  for (size_t i = 0; i < atn.maxTokenType; ++i) {
+    _tokenNames.push_back(vocabulary.getDisplayName(i));
+  }
+
+  // init decision DFA
+  for (size_t i = 0; i < atn.getNumberOfDecisions(); ++i) {
+    atn::DecisionState *decisionState = atn.getDecisionState(i);
+    _decisionToDFA.push_back(dfa::DFA(decisionState, i));
+  }
+
+  // get atn simulator that knows how to do predictions
+  _interpreter = new atn::ParserATNSimulator(this, atn, _decisionToDFA, _sharedContextCache); /* mem-check: deleted in d-tor */
+}
+
+ParserInterpreter::~ParserInterpreter() {
+  delete _interpreter;
+}
+
+void ParserInterpreter::reset() {
+  Parser::reset();
+  _overrideDecisionReached = false;
+  _overrideDecisionRoot = nullptr;
+}
+
+const atn::ATN& ParserInterpreter::getATN() const {
+  return _atn;
+}
+
+const std::vector<std::string>& ParserInterpreter::getTokenNames() const {
+  return _tokenNames;
+}
+
+const dfa::Vocabulary& ParserInterpreter::getVocabulary() const {
+  return _vocabulary;
+}
+
+const std::vector<std::string>& ParserInterpreter::getRuleNames() const {
+  return _ruleNames;
+}
+
+std::string ParserInterpreter::getGrammarFileName() const {
+  return _grammarFileName;
+}
+
+ParserRuleContext* ParserInterpreter::parse(size_t startRuleIndex) {
+  atn::RuleStartState *startRuleStartState = _atn.ruleToStartState[startRuleIndex];
+
+  _rootContext = createInterpreterRuleContext(nullptr, atn::ATNState::INVALID_STATE_NUMBER, startRuleIndex);
+
+  if (startRuleStartState->isLeftRecursiveRule) {
+    enterRecursionRule(_rootContext, startRuleStartState->stateNumber, startRuleIndex, 0);
+  } else {
+    enterRule(_rootContext, startRuleStartState->stateNumber, startRuleIndex);
+  }
+
+  while (true) {
+    atn::ATNState *p = getATNState();
+    switch (p->getStateType()) {
+      case atn::ATNState::RULE_STOP :
+        // pop; return from rule
+        if (_ctx->isEmpty()) {
+          if (startRuleStartState->isLeftRecursiveRule) {
+            ParserRuleContext *result = _ctx;
+            auto parentContext = _parentContextStack.top();
+            _parentContextStack.pop();
+            unrollRecursionContexts(parentContext.first);
+            return result;
+          } else {
+            exitRule();
+            return _rootContext;
+          }
+        }
+
+        visitRuleStopState(p);
+        break;
+
+      default :
+        try {
+          visitState(p);
+        }
+        catch (RecognitionException &e) {
+          setState(_atn.ruleToStopState[p->ruleIndex]->stateNumber);
+          getErrorHandler()->reportError(this, e);
+          getContext()->exception = std::current_exception();
+          recover(e);
+        }
+
+        break;
+    }
+  }
+}
+
+void ParserInterpreter::enterRecursionRule(ParserRuleContext *localctx, size_t state, size_t ruleIndex, int precedence) {
+  _parentContextStack.push({ _ctx, localctx->invokingState });
+  Parser::enterRecursionRule(localctx, state, ruleIndex, precedence);
+}
+
+void ParserInterpreter::addDecisionOverride(int decision, int tokenIndex, int forcedAlt) {
+  _overrideDecision = decision;
+  _overrideDecisionInputIndex = tokenIndex;
+  _overrideDecisionAlt = forcedAlt;
+}
+
+Ref<InterpreterRuleContext> ParserInterpreter::getOverrideDecisionRoot() const {
+  return _overrideDecisionRoot;
+}
+
+InterpreterRuleContext* ParserInterpreter::getRootContext() {
+  return _rootContext;
+}
+
+atn::ATNState* ParserInterpreter::getATNState() {
+  return _atn.states[getState()];
+}
+
+void ParserInterpreter::visitState(atn::ATNState *p) {
+  size_t predictedAlt = 1;
+  if (is<DecisionState *>(p)) {
+    predictedAlt = visitDecisionState(dynamic_cast<DecisionState *>(p));
+  }
+
+  atn::Transition *transition = p->transitions[predictedAlt - 1];
+  switch (transition->getSerializationType()) {
+    case atn::Transition::EPSILON:
+      if (p->getStateType() == ATNState::STAR_LOOP_ENTRY &&
+        (dynamic_cast<StarLoopEntryState *>(p))->isPrecedenceDecision &&
+        !is<LoopEndState *>(transition->target)) {
+        // We are at the start of a left recursive rule's (...)* loop
+        // and we're not taking the exit branch of loop.
+        InterpreterRuleContext *localctx = createInterpreterRuleContext(_parentContextStack.top().first,
+          _parentContextStack.top().second, static_cast<int>(_ctx->getRuleIndex()));
+        pushNewRecursionContext(localctx, _atn.ruleToStartState[p->ruleIndex]->stateNumber, static_cast<int>(_ctx->getRuleIndex()));
+      }
+      break;
+
+    case atn::Transition::ATOM:
+      match(static_cast<int>(static_cast<atn::AtomTransition*>(transition)->_label));
+      break;
+
+    case atn::Transition::RANGE:
+    case atn::Transition::SET:
+    case atn::Transition::NOT_SET:
+      if (!transition->matches(static_cast<int>(_input->LA(1)), Token::MIN_USER_TOKEN_TYPE, Lexer::MAX_CHAR_VALUE)) {
+        recoverInline();
+      }
+      matchWildcard();
+      break;
+
+    case atn::Transition::WILDCARD:
+      matchWildcard();
+      break;
+
+    case atn::Transition::RULE:
+    {
+      atn::RuleStartState *ruleStartState = static_cast<atn::RuleStartState*>(transition->target);
+      size_t ruleIndex = ruleStartState->ruleIndex;
+      InterpreterRuleContext *newctx = createInterpreterRuleContext(_ctx, p->stateNumber, ruleIndex);
+      if (ruleStartState->isLeftRecursiveRule) {
+        enterRecursionRule(newctx, ruleStartState->stateNumber, ruleIndex, static_cast<atn::RuleTransition*>(transition)->precedence);
+      } else {
+        enterRule(newctx, transition->target->stateNumber, ruleIndex);
+      }
+    }
+      break;
+
+    case atn::Transition::PREDICATE:
+    {
+      atn::PredicateTransition *predicateTransition = static_cast<atn::PredicateTransition*>(transition);
+      if (!sempred(_ctx, predicateTransition->ruleIndex, predicateTransition->predIndex)) {
+        throw FailedPredicateException(this);
+      }
+    }
+      break;
+
+    case atn::Transition::ACTION:
+    {
+      atn::ActionTransition *actionTransition = static_cast<atn::ActionTransition*>(transition);
+      action(_ctx, actionTransition->ruleIndex, actionTransition->actionIndex);
+    }
+      break;
+
+    case atn::Transition::PRECEDENCE:
+    {
+      if (!precpred(_ctx, static_cast<atn::PrecedencePredicateTransition*>(transition)->precedence)) {
+        throw FailedPredicateException(this, "precpred(_ctx, " + std::to_string(static_cast<atn::PrecedencePredicateTransition*>(transition)->precedence) +  ")");
+      }
+    }
+      break;
+
+    default:
+      throw UnsupportedOperationException("Unrecognized ATN transition type.");
+  }
+
+  setState(transition->target->stateNumber);
+}
+
+size_t ParserInterpreter::visitDecisionState(DecisionState *p) {
+  size_t predictedAlt = 1;
+  if (p->transitions.size() > 1) {
+    getErrorHandler()->sync(this);
+    int decision = p->decision;
+    if (decision == _overrideDecision && _input->index() == _overrideDecisionInputIndex && !_overrideDecisionReached) {
+      predictedAlt = _overrideDecisionAlt;
+      _overrideDecisionReached = true;
+    } else {
+      predictedAlt = getInterpreter<ParserATNSimulator>()->adaptivePredict(_input, decision, _ctx);
+    }
+  }
+  return predictedAlt;
+}
+
+InterpreterRuleContext* ParserInterpreter::createInterpreterRuleContext(ParserRuleContext *parent,
+  size_t invokingStateNumber, size_t ruleIndex) {
+  return _tracker.createInstance<InterpreterRuleContext>(parent, invokingStateNumber, ruleIndex);
+}
+
+void ParserInterpreter::visitRuleStopState(atn::ATNState *p) {
+  atn::RuleStartState *ruleStartState = _atn.ruleToStartState[p->ruleIndex];
+  if (ruleStartState->isLeftRecursiveRule) {
+    std::pair<ParserRuleContext *, size_t> parentContext = _parentContextStack.top();
+    _parentContextStack.pop();
+
+    unrollRecursionContexts(parentContext.first);
+    setState(parentContext.second);
+  } else {
+    exitRule();
+  }
+
+  atn::RuleTransition *ruleTransition = static_cast<atn::RuleTransition*>(_atn.states[getState()]->transitions[0]);
+  setState(ruleTransition->followState->stateNumber);
+}
+
+void ParserInterpreter::recover(RecognitionException &e) {
+  size_t i = _input->index();
+  getErrorHandler()->recover(this, std::make_exception_ptr(e));
+
+  if (_input->index() == i) {
+    // no input consumed, better add an error node
+    if (is<InputMismatchException *>(&e)) {
+      InputMismatchException &ime = static_cast<InputMismatchException&>(e);
+      Token *tok = e.getOffendingToken();
+      size_t expectedTokenType = ime.getExpectedTokens().getMinElement(); // get any element
+      _errorToken = getTokenFactory()->create({ tok->getTokenSource(), tok->getTokenSource()->getInputStream() },
+        expectedTokenType, tok->getText(), Token::DEFAULT_CHANNEL, INVALID_INDEX, INVALID_INDEX, // invalid start/stop
+        tok->getLine(), tok->getCharPositionInLine());
+      _ctx->addChild(createErrorNode(_errorToken.get()));
+    }
+    else { // NoViableAlt
+      Token *tok = e.getOffendingToken();
+      _errorToken = getTokenFactory()->create({ tok->getTokenSource(), tok->getTokenSource()->getInputStream() },
+        Token::INVALID_TYPE, tok->getText(), Token::DEFAULT_CHANNEL, INVALID_INDEX, INVALID_INDEX, // invalid start/stop
+        tok->getLine(), tok->getCharPositionInLine());
+      _ctx->addChild(createErrorNode(_errorToken.get()));
+    }
+  }
+}
+
+Token* ParserInterpreter::recoverInline() {
+  return _errHandler->recoverInline(this);
+}

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/ParserInterpreter.h

@@ -0,0 +1,179 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "Parser.h"
+#include "atn/ATN.h"
+#include "support/BitSet.h"
+#include "atn/PredictionContext.h"
+#include "Vocabulary.h"
+
+namespace antlr4 {
+
+  /// <summary>
+  /// A parser simulator that mimics what ANTLR's generated
+  ///  parser code does. A ParserATNSimulator is used to make
+  ///  predictions via adaptivePredict but this class moves a pointer through the
+  ///  ATN to simulate parsing. ParserATNSimulator just
+  ///  makes us efficient rather than having to backtrack, for example.
+  ///
+  ///  This properly creates parse trees even for left recursive rules.
+  ///
+  ///  We rely on the left recursive rule invocation and special predicate
+  ///  transitions to make left recursive rules work.
+  ///
+  ///  See TestParserInterpreter for examples.
+  /// </summary>
+  class ANTLR4CPP_PUBLIC ParserInterpreter : public Parser {
+  public:
+    // @deprecated
+    ParserInterpreter(const std::string &grammarFileName, const std::vector<std::string>& tokenNames,
+      const std::vector<std::string>& ruleNames, const atn::ATN &atn, TokenStream *input);
+    ParserInterpreter(const std::string &grammarFileName, const dfa::Vocabulary &vocabulary,
+                      const std::vector<std::string> &ruleNames, const atn::ATN &atn, TokenStream *input);
+    ~ParserInterpreter();
+
+    virtual void reset() override;
+
+    virtual const atn::ATN& getATN() const override;
+
+    // @deprecated
+    virtual const std::vector<std::string>& getTokenNames() const override;
+
+    virtual const dfa::Vocabulary& getVocabulary() const override;
+
+    virtual const std::vector<std::string>& getRuleNames() const override;
+    virtual std::string getGrammarFileName() const override;
+
+    /// Begin parsing at startRuleIndex
+    virtual ParserRuleContext* parse(size_t startRuleIndex);
+
+    virtual void enterRecursionRule(ParserRuleContext *localctx, size_t state, size_t ruleIndex, int precedence) override;
+
+
+    /** Override this parser interpreters normal decision-making process
+     *  at a particular decision and input token index. Instead of
+     *  allowing the adaptive prediction mechanism to choose the
+     *  first alternative within a block that leads to a successful parse,
+     *  force it to take the alternative, 1..n for n alternatives.
+     *
+     *  As an implementation limitation right now, you can only specify one
+     *  override. This is sufficient to allow construction of different
+     *  parse trees for ambiguous input. It means re-parsing the entire input
+     *  in general because you're never sure where an ambiguous sequence would
+     *  live in the various parse trees. For example, in one interpretation,
+     *  an ambiguous input sequence would be matched completely in expression
+     *  but in another it could match all the way back to the root.
+     *
+     *  s : e '!'? ;
+     *  e : ID
+     *    | ID '!'
+     *    ;
+     *
+     *  Here, x! can be matched as (s (e ID) !) or (s (e ID !)). In the first
+     *  case, the ambiguous sequence is fully contained only by the root.
+     *  In the second case, the ambiguous sequences fully contained within just
+     *  e, as in: (e ID !).
+     *
+     *  Rather than trying to optimize this and make
+     *  some intelligent decisions for optimization purposes, I settled on
+     *  just re-parsing the whole input and then using
+     *  {link Trees#getRootOfSubtreeEnclosingRegion} to find the minimal
+     *  subtree that contains the ambiguous sequence. I originally tried to
+     *  record the call stack at the point the parser detected and ambiguity but
+     *  left recursive rules create a parse tree stack that does not reflect
+     *  the actual call stack. That impedance mismatch was enough to make
+     *  it it challenging to restart the parser at a deeply nested rule
+     *  invocation.
+     *
+     *  Only parser interpreters can override decisions so as to avoid inserting
+     *  override checking code in the critical ALL(*) prediction execution path.
+     *
+     *  @since 4.5.1
+     */
+    void addDecisionOverride(int decision, int tokenIndex, int forcedAlt);
+
+    Ref<InterpreterRuleContext> getOverrideDecisionRoot() const;
+
+    /** Return the root of the parse, which can be useful if the parser
+     *  bails out. You still can access the top node. Note that,
+     *  because of the way left recursive rules add children, it's possible
+     *  that the root will not have any children if the start rule immediately
+     *  called and left recursive rule that fails.
+     *
+     * @since 4.5.1
+     */
+    InterpreterRuleContext* getRootContext();
+
+  protected:
+    const std::string _grammarFileName;
+    std::vector<std::string> _tokenNames;
+    const atn::ATN &_atn;
+
+    std::vector<std::string> _ruleNames;
+
+    std::vector<dfa::DFA> _decisionToDFA; // not shared like it is for generated parsers
+    atn::PredictionContextCache _sharedContextCache;
+
+    /** This stack corresponds to the _parentctx, _parentState pair of locals
+     *  that would exist on call stack frames with a recursive descent parser;
+     *  in the generated function for a left-recursive rule you'd see:
+     *
+     *  private EContext e(int _p) throws RecognitionException {
+     *      ParserRuleContext _parentctx = _ctx;    // Pair.a
+     *      int _parentState = getState();          // Pair.b
+     *      ...
+     *  }
+     *
+     *  Those values are used to create new recursive rule invocation contexts
+     *  associated with left operand of an alt like "expr '*' expr".
+     */
+    std::stack<std::pair<ParserRuleContext *, size_t>> _parentContextStack;
+
+    /** We need a map from (decision,inputIndex)->forced alt for computing ambiguous
+     *  parse trees. For now, we allow exactly one override.
+     */
+    int _overrideDecision = -1;
+    size_t _overrideDecisionInputIndex = INVALID_INDEX;
+    size_t _overrideDecisionAlt = INVALID_INDEX;
+    bool _overrideDecisionReached = false; // latch and only override once; error might trigger infinite loop
+
+    /** What is the current context when we override a decision? This tells
+     *  us what the root of the parse tree is when using override
+     *  for an ambiguity/lookahead check.
+     */
+    Ref<InterpreterRuleContext> _overrideDecisionRoot;
+    InterpreterRuleContext* _rootContext;
+
+    virtual atn::ATNState *getATNState();
+    virtual void visitState(atn::ATNState *p);
+
+    /** Method visitDecisionState() is called when the interpreter reaches
+     *  a decision state (instance of DecisionState). It gives an opportunity
+     *  for subclasses to track interesting things.
+     */
+    size_t visitDecisionState(atn::DecisionState *p);
+
+    /** Provide simple "factory" for InterpreterRuleContext's.
+     *  @since 4.5.1
+     */
+    InterpreterRuleContext* createInterpreterRuleContext(ParserRuleContext *parent, size_t invokingStateNumber, size_t ruleIndex);
+
+    virtual void visitRuleStopState(atn::ATNState *p);
+
+    /** Rely on the error handler for this parser but, if no tokens are consumed
+     *  to recover, add an error node. Otherwise, nothing is seen in the parse
+     *  tree.
+     */
+    void recover(RecognitionException &e);
+    Token* recoverInline();
+
+  private:
+    const dfa::Vocabulary &_vocabulary;
+    std::unique_ptr<Token> _errorToken;
+  };
+
+} // namespace antlr4