python3-parser 1.0.0

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/dfa/DFASerializer.cpp

@@ -0,0 +1,67 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "dfa/DFA.h"
+#include "Vocabulary.h"
+
+#include "dfa/DFASerializer.h"
+
+using namespace antlr4::dfa;
+
+DFASerializer::DFASerializer(const DFA *dfa, const std::vector<std::string>& tokenNames)
+  : DFASerializer(dfa, Vocabulary::fromTokenNames(tokenNames)) {
+}
+
+DFASerializer::DFASerializer(const DFA *dfa, const Vocabulary &vocabulary) : _dfa(dfa), _vocabulary(vocabulary) {
+}
+
+DFASerializer::~DFASerializer() {
+}
+
+std::string DFASerializer::toString() const {
+  if (_dfa->s0 == nullptr) {
+    return "";
+  }
+
+  std::stringstream ss;
+  std::vector<DFAState *> states = _dfa->getStates();
+  for (auto s : states) {
+    for (size_t i = 0; i < s->edges.size(); i++) {
+      DFAState *t = s->edges[i];
+      if (t != nullptr && t->stateNumber != INT32_MAX) {
+        ss << getStateString(s);
+        std::string label = getEdgeLabel(i);
+        ss << "-" << label << "->" << getStateString(t) << "\n";
+      }
+    }
+  }
+
+  return ss.str();
+}
+
+std::string DFASerializer::getEdgeLabel(size_t i) const {
+  return _vocabulary.getDisplayName(i); // ml: no longer needed -1 as we use a map for edges, without offset.
+}
+
+std::string DFASerializer::getStateString(DFAState *s) const {
+  size_t n = s->stateNumber;
+
+  const std::string baseStateStr = std::string(s->isAcceptState ? ":" : "") + "s" + std::to_string(n) +
+    (s->requiresFullContext ? "^" : "");
+
+  if (s->isAcceptState) {
+    if (!s->predicates.empty()) {
+      std::string buf;
+      for (size_t i = 0; i < s->predicates.size(); i++) {
+        buf.append(s->predicates[i]->toString());
+      }
+      return baseStateStr + "=>" + buf;
+    } else {
+      return baseStateStr + "=>" + std::to_string(s->prediction);
+    }
+  } else {
+    return baseStateStr;
+  }
+}

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/dfa/DFASerializer.h

@@ -0,0 +1,32 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "Vocabulary.h"
+
+namespace antlr4 {
+namespace dfa {
+
+  /// A DFA walker that knows how to dump them to serialized strings.
+  class ANTLR4CPP_PUBLIC DFASerializer {
+  public:
+    DFASerializer(const DFA *dfa, const std::vector<std::string>& tnames);
+    DFASerializer(const DFA *dfa, const Vocabulary &vocabulary);
+    virtual ~DFASerializer();
+
+    virtual std::string toString() const;
+
+  protected:
+    virtual std::string getEdgeLabel(size_t i) const;
+    virtual std::string getStateString(DFAState *s) const;
+
+  private:
+    const DFA *_dfa;
+    const Vocabulary &_vocabulary;
+  };
+
+} // namespace atn
+} // namespace antlr4

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/dfa/DFAState.cpp

@@ -0,0 +1,100 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "atn/ATNConfigSet.h"
+#include "atn/SemanticContext.h"
+#include "atn/ATNConfig.h"
+#include "misc/MurmurHash.h"
+
+#include "dfa/DFAState.h"
+
+using namespace antlr4::dfa;
+using namespace antlr4::atn;
+
+DFAState::PredPrediction::PredPrediction(const Ref<SemanticContext> &pred, int alt) : pred(pred) {
+  InitializeInstanceFields();
+  this->alt = alt;
+}
+
+DFAState::PredPrediction::~PredPrediction() {
+}
+
+std::string DFAState::PredPrediction::toString() {
+  return std::string("(") + pred->toString() + ", " + std::to_string(alt) + ")";
+}
+
+void DFAState::PredPrediction::InitializeInstanceFields() {
+  alt = 0;
+}
+
+DFAState::DFAState() {
+  InitializeInstanceFields();
+}
+
+DFAState::DFAState(int state) : DFAState() {
+  stateNumber = state;
+}
+
+DFAState::DFAState(std::unique_ptr<ATNConfigSet> configs_) : DFAState() {
+  configs = std::move(configs_);
+}
+
+DFAState::~DFAState() {
+  for (auto predicate : predicates) {
+    delete predicate;
+  }
+}
+
+std::set<size_t> DFAState::getAltSet() {
+  std::set<size_t> alts;
+  if (configs != nullptr) {
+    for (size_t i = 0; i < configs->size(); i++) {
+      alts.insert(configs->get(i)->alt);
+    }
+  }
+  return alts;
+}
+
+size_t DFAState::hashCode() const {
+  size_t hash = misc::MurmurHash::initialize(7);
+  hash = misc::MurmurHash::update(hash, configs->hashCode());
+  hash = misc::MurmurHash::finish(hash, 1);
+  return hash;
+}
+
+bool DFAState::operator == (const DFAState &o) const {
+  // compare set of ATN configurations in this set with other
+  if (this == &o) {
+    return true;
+  }
+
+  return *configs == *o.configs;
+}
+
+std::string DFAState::toString() {
+  std::stringstream ss;
+  ss << stateNumber;
+  if (configs) {
+    ss << ":" << configs->toString();
+  }
+  if (isAcceptState) {
+    ss << " => ";
+    if (!predicates.empty()) {
+      for (size_t i = 0; i < predicates.size(); i++) {
+        ss << predicates[i]->toString();
+      }
+    } else {
+      ss << prediction;
+    }
+  }
+  return ss.str();
+}
+
+void DFAState::InitializeInstanceFields() {
+  stateNumber = -1;
+  isAcceptState = false;
+  prediction = 0;
+  requiresFullContext = false;
+}

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/dfa/DFAState.h

@@ -0,0 +1,144 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "antlr4-common.h"
+
+namespace antlr4 {
+namespace dfa {
+
+  /// <summary>
+  /// A DFA state represents a set of possible ATN configurations.
+  ///  As Aho, Sethi, Ullman p. 117 says "The DFA uses its state
+  ///  to keep track of all possible states the ATN can be in after
+  ///  reading each input symbol.  That is to say, after reading
+  ///  input a1a2..an, the DFA is in a state that represents the
+  ///  subset T of the states of the ATN that are reachable from the
+  ///  ATN's start state along some path labeled a1a2..an."
+  ///  In conventional NFA->DFA conversion, therefore, the subset T
+  ///  would be a bitset representing the set of states the
+  ///  ATN could be in.  We need to track the alt predicted by each
+  ///  state as well, however.  More importantly, we need to maintain
+  ///  a stack of states, tracking the closure operations as they
+  ///  jump from rule to rule, emulating rule invocations (method calls).
+  ///  I have to add a stack to simulate the proper lookahead sequences for
+  ///  the underlying LL grammar from which the ATN was derived.
+  /// <p/>
+  ///  I use a set of ATNConfig objects not simple states.  An ATNConfig
+  ///  is both a state (ala normal conversion) and a RuleContext describing
+  ///  the chain of rules (if any) followed to arrive at that state.
+  /// <p/>
+  ///  A DFA state may have multiple references to a particular state,
+  ///  but with different ATN contexts (with same or different alts)
+  ///  meaning that state was reached via a different set of rule invocations.
+  /// </summary>
+  class ANTLR4CPP_PUBLIC DFAState {
+  public:
+    class PredPrediction {
+    public:
+      Ref<atn::SemanticContext> pred; // never null; at least SemanticContext.NONE
+      int alt;
+
+      PredPrediction(const Ref<atn::SemanticContext> &pred, int alt);
+      virtual ~PredPrediction();
+
+      virtual std::string toString();
+
+    private:
+      void InitializeInstanceFields();
+    };
+
+    int stateNumber;
+
+    std::unique_ptr<atn::ATNConfigSet> configs;
+
+    /// {@code edges[symbol]} points to target of symbol. Shift up by 1 so (-1)
+    ///  <seealso cref="Token#EOF"/> maps to {@code edges[0]}.
+    // ml: this is a sparse list, so we use a map instead of a vector.
+    //     Watch out: we no longer have the -1 offset, as it isn't needed anymore.
+    std::unordered_map<size_t, DFAState *> edges;
+
+    bool isAcceptState;
+
+    /// if accept state, what ttype do we match or alt do we predict?
+    /// This is set to <seealso cref="ATN#INVALID_ALT_NUMBER"/> when <seealso cref="#predicates"/>{@code !=null} or
+    /// <seealso cref="#requiresFullContext"/>.
+    size_t prediction;
+
+    Ref<atn::LexerActionExecutor> lexerActionExecutor;
+
+    /// <summary>
+    /// Indicates that this state was created during SLL prediction that
+    /// discovered a conflict between the configurations in the state. Future
+    /// <seealso cref="ParserATNSimulator#execATN"/> invocations immediately jumped doing
+    /// full context prediction if this field is true.
+    /// </summary>
+    bool requiresFullContext;
+
+    /// <summary>
+    /// During SLL parsing, this is a list of predicates associated with the
+    ///  ATN configurations of the DFA state. When we have predicates,
+    ///  <seealso cref="#requiresFullContext"/> is {@code false} since full context prediction evaluates predicates
+    ///  on-the-fly. If this is not null, then <seealso cref="#prediction"/> is
+    ///  <seealso cref="ATN#INVALID_ALT_NUMBER"/>.
+    /// <p/>
+    ///  We only use these for non-<seealso cref="#requiresFullContext"/> but conflicting states. That
+    ///  means we know from the context (it's $ or we don't dip into outer
+    ///  context) that it's an ambiguity not a conflict.
+    /// <p/>
+    ///  This list is computed by <seealso cref="ParserATNSimulator#predicateDFAState"/>.
+    /// </summary>
+    std::vector<PredPrediction *> predicates;
+
+    /// Map a predicate to a predicted alternative.
+    DFAState();
+    DFAState(int state);
+    DFAState(std::unique_ptr<atn::ATNConfigSet> configs);
+    virtual ~DFAState();
+
+    /// <summary>
+    /// Get the set of all alts mentioned by all ATN configurations in this
+    ///  DFA state.
+    /// </summary>
+    virtual std::set<size_t> getAltSet();
+
+    virtual size_t hashCode() const;
+
+    /// Two DFAState instances are equal if their ATN configuration sets
+    /// are the same. This method is used to see if a state already exists.
+    ///
+    /// Because the number of alternatives and number of ATN configurations are
+    /// finite, there is a finite number of DFA states that can be processed.
+    /// This is necessary to show that the algorithm terminates.
+    ///
+    /// Cannot test the DFA state numbers here because in
+    /// ParserATNSimulator#addDFAState we need to know if any other state
+    /// exists that has this exact set of ATN configurations. The
+    /// stateNumber is irrelevant.
+    bool operator == (const DFAState &o) const;
+
+    virtual std::string toString();
+
+    struct Hasher
+    {
+      size_t operator()(DFAState *k) const {
+        return k->hashCode();
+      }
+    };
+
+    struct Comparer {
+      bool operator()(DFAState *lhs, DFAState *rhs) const
+      {
+        return *lhs == *rhs;
+      }
+    };
+
+  private:
+    void InitializeInstanceFields();
+  };
+
+} // namespace atn
+} // namespace antlr4

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/dfa/LexerDFASerializer.cpp

@@ -0,0 +1,20 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "Vocabulary.h"
+
+#include "dfa/LexerDFASerializer.h"
+
+using namespace antlr4::dfa;
+
+LexerDFASerializer::LexerDFASerializer(DFA *dfa) : DFASerializer(dfa, Vocabulary::EMPTY_VOCABULARY) {
+}
+
+LexerDFASerializer::~LexerDFASerializer() {
+}
+
+std::string LexerDFASerializer::getEdgeLabel(size_t i) const {
+  return std::string("'") + static_cast<char>(i) + "'";
+}

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/dfa/LexerDFASerializer.h

@@ -0,0 +1,23 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "dfa/DFASerializer.h"
+
+namespace antlr4 {
+namespace dfa {
+
+  class ANTLR4CPP_PUBLIC LexerDFASerializer : public DFASerializer {
+  public:
+    LexerDFASerializer(DFA *dfa);
+    virtual ~LexerDFASerializer();
+
+  protected:
+    virtual std::string getEdgeLabel(size_t i) const override;
+  };
+
+} // namespace atn
+} // namespace antlr4

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/misc/InterpreterDataReader.cpp

@@ -0,0 +1,124 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "atn/ATN.h"
+#include "atn/ATNDeserializer.h"
+#include "Vocabulary.h"
+
+#include "misc/InterpreterDataReader.h"
+
+using namespace antlr4::dfa;
+using namespace antlr4::atn;
+using namespace antlr4::misc;
+
+InterpreterData::InterpreterData(std::vector<std::string> const& literalNames, std::vector<std::string> const& symbolicNames)
+: vocabulary(literalNames, symbolicNames) {
+}
+
+InterpreterData InterpreterDataReader::parseFile(std::string const& fileName) {
+  // The structure of the data file is very simple. Everything is line based with empty lines
+  // separating the different parts. For lexers the layout is:
+  // token literal names:
+  // ...
+  //
+  // token symbolic names:
+  // ...
+  //
+  // rule names:
+  // ...
+  //
+  // channel names:
+  // ...
+  //
+  // mode names:
+  // ...
+  //
+  // atn:
+  // <a single line with comma separated int values> enclosed in a pair of squared brackets.
+  //
+  // Data for a parser does not contain channel and mode names.
+
+  std::ifstream input(fileName);
+  if (!input.good())
+    return {};
+
+  std::vector<std::string> literalNames;
+  std::vector<std::string> symbolicNames;
+
+  std::string line;
+
+  std::getline(input, line, '\n');
+  assert(line == "token literal names:");
+  while (true) {
+    std::getline(input, line, '\n');
+    if (line.empty())
+      break;
+
+    literalNames.push_back(line == "null" ? "" : line);
+  };
+
+  std::getline(input, line, '\n');
+  assert(line == "token symbolic names:");
+  while (true) {
+    std::getline(input, line, '\n');
+    if (line.empty())
+      break;
+
+    symbolicNames.push_back(line == "null" ? "" : line);
+  };
+  InterpreterData result(literalNames, symbolicNames);
+
+  std::getline(input, line, '\n');
+  assert(line == "rule names:");
+  while (true) {
+    std::getline(input, line, '\n');
+    if (line.empty())
+      break;
+
+    result.ruleNames.push_back(line);
+  };
+
+  std::getline(input, line, '\n');
+  if (line == "channel names:") {
+    while (true) {
+      std::getline(input, line, '\n');
+      if (line.empty())
+        break;
+
+      result.channels.push_back(line);
+    };
+
+    std::getline(input, line, '\n');
+    assert(line == "mode names:");
+    while (true) {
+      std::getline(input, line, '\n');
+      if (line.empty())
+        break;
+
+      result.modes.push_back(line);
+    };
+  }
+
+  std::vector<uint16_t> serializedATN;
+
+  std::getline(input, line, '\n');
+  assert(line == "atn:");
+  std::getline(input, line, '\n');
+  std::stringstream tokenizer(line);
+  std::string value;
+  while (tokenizer.good()) {
+    std::getline(tokenizer, value, ',');
+    unsigned long number;
+    if (value[0] == '[')
+      number = std::strtoul(&value[1], nullptr, 10);
+    else
+      number = std::strtoul(value.c_str(), nullptr, 10);
+    serializedATN.push_back(static_cast<uint16_t>(number));
+  }
+
+  ATNDeserializer deserializer;
+  result.atn = deserializer.deserialize(serializedATN);
+  return result;
+}