python3-parser 1.0.0

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/BufferedTokenStream.h

@@ -0,0 +1,200 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "TokenStream.h"
+
+namespace antlr4 {
+
+  /**
+   * This implementation of {@link TokenStream} loads tokens from a
+   * {@link TokenSource} on-demand, and places the tokens in a buffer to provide
+   * access to any previous token by index.
+   *
+   * <p>
+   * This token stream ignores the value of {@link Token#getChannel}. If your
+   * parser requires the token stream filter tokens to only those on a particular
+   * channel, such as {@link Token#DEFAULT_CHANNEL} or
+   * {@link Token#HIDDEN_CHANNEL}, use a filtering token stream such a
+   * {@link CommonTokenStream}.</p>
+   */
+  class ANTLR4CPP_PUBLIC BufferedTokenStream : public TokenStream {
+  public:
+    BufferedTokenStream(TokenSource *tokenSource);
+    BufferedTokenStream(const BufferedTokenStream& other) = delete;
+
+    BufferedTokenStream& operator = (const BufferedTokenStream& other) = delete;
+
+    virtual TokenSource* getTokenSource() const override;
+    virtual size_t index() override;
+    virtual ssize_t mark() override;
+
+    virtual void release(ssize_t marker) override;
+    virtual void reset();
+    virtual void seek(size_t index) override;
+
+    virtual size_t size() override;
+    virtual void consume() override;
+
+    virtual Token* get(size_t i) const override;
+
+    /// Get all tokens from start..stop inclusively.
+    virtual std::vector<Token *> get(size_t start, size_t stop);
+
+    virtual size_t LA(ssize_t i) override;
+    virtual Token* LT(ssize_t k) override;
+
+    /// Reset this token stream by setting its token source.
+    virtual void setTokenSource(TokenSource *tokenSource);
+    virtual std::vector<Token *> getTokens();
+    virtual std::vector<Token *> getTokens(size_t start, size_t stop);
+
+    /// <summary>
+    /// Given a start and stop index, return a List of all tokens in
+    ///  the token type BitSet.  Return null if no tokens were found.  This
+    ///  method looks at both on and off channel tokens.
+    /// </summary>
+    virtual std::vector<Token *> getTokens(size_t start, size_t stop, const std::vector<size_t> &types);
+    virtual std::vector<Token *> getTokens(size_t start, size_t stop, size_t ttype);
+
+    /// Collect all tokens on specified channel to the right of
+    ///  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or
+    ///  EOF. If channel is -1, find any non default channel token.
+    virtual std::vector<Token *> getHiddenTokensToRight(size_t tokenIndex, ssize_t channel);
+
+    /// <summary>
+    /// Collect all hidden tokens (any off-default channel) to the right of
+    ///  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL
+    ///  or EOF.
+    /// </summary>
+    virtual std::vector<Token *> getHiddenTokensToRight(size_t tokenIndex);
+
+    /// <summary>
+    /// Collect all tokens on specified channel to the left of
+    ///  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
+    ///  If channel is -1, find any non default channel token.
+    /// </summary>
+    virtual std::vector<Token *> getHiddenTokensToLeft(size_t tokenIndex, ssize_t channel);
+
+    /// <summary>
+    /// Collect all hidden tokens (any off-default channel) to the left of
+    ///  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
+    /// </summary>
+    virtual std::vector<Token *> getHiddenTokensToLeft(size_t tokenIndex);
+
+    virtual std::string getSourceName() const override;
+    virtual std::string getText() override;
+    virtual std::string getText(const misc::Interval &interval) override;
+    virtual std::string getText(RuleContext *ctx) override;
+    virtual std::string getText(Token *start, Token *stop) override;
+
+    /// Get all tokens from lexer until EOF.
+    virtual void fill();
+
+  protected:
+    /**
+     * The {@link TokenSource} from which tokens for this stream are fetched.
+     */
+    TokenSource *_tokenSource;
+
+    /**
+     * A collection of all tokens fetched from the token source. The list is
+     * considered a complete view of the input once {@link #fetchedEOF} is set
+     * to {@code true}.
+     */
+    std::vector<std::unique_ptr<Token>> _tokens;
+
+    /**
+     * The index into {@link #tokens} of the current token (next token to
+     * {@link #consume}). {@link #tokens}{@code [}{@link #p}{@code ]} should be
+     * {@link #LT LT(1)}.
+     *
+     * <p>This field is set to -1 when the stream is first constructed or when
+     * {@link #setTokenSource} is called, indicating that the first token has
+     * not yet been fetched from the token source. For additional information,
+     * see the documentation of {@link IntStream} for a description of
+     * Initializing Methods.</p>
+     */
+    // ml: since -1 requires to make this member signed for just this single aspect we use a member _needSetup instead.
+    //     Use bool isInitialized() to find out if this stream has started reading.
+    size_t _p;
+
+    /**
+     * Indicates whether the {@link Token#EOF} token has been fetched from
+     * {@link #tokenSource} and added to {@link #tokens}. This field improves
+     * performance for the following cases:
+     *
+     * <ul>
+     * <li>{@link #consume}: The lookahead check in {@link #consume} to prevent
+     * consuming the EOF symbol is optimized by checking the values of
+     * {@link #fetchedEOF} and {@link #p} instead of calling {@link #LA}.</li>
+     * <li>{@link #fetch}: The check to prevent adding multiple EOF symbols into
+     * {@link #tokens} is trivial with this field.</li>
+     * <ul>
+     */
+    bool _fetchedEOF;
+
+    /// <summary>
+    /// Make sure index {@code i} in tokens has a token.
+    /// </summary>
+    /// <returns> {@code true} if a token is located at index {@code i}, otherwise
+    ///    {@code false}. </returns>
+    /// <seealso cref= #get(int i) </seealso>
+    virtual bool sync(size_t i);
+
+    /// <summary>
+    /// Add {@code n} elements to buffer.
+    /// </summary>
+    /// <returns> The actual number of elements added to the buffer. </returns>
+    virtual size_t fetch(size_t n);
+
+    virtual Token* LB(size_t k);
+
+    /// Allowed derived classes to modify the behavior of operations which change
+    /// the current stream position by adjusting the target token index of a seek
+    /// operation. The default implementation simply returns {@code i}. If an
+    /// exception is thrown in this method, the current stream index should not be
+    /// changed.
+    /// <p/>
+    /// For example, <seealso cref="CommonTokenStream"/> overrides this method to ensure that
+    /// the seek target is always an on-channel token.
+    ///
+    /// <param name="i"> The target token index. </param>
+    /// <returns> The adjusted target token index. </returns>
+    virtual ssize_t adjustSeekIndex(size_t i);
+    void lazyInit();
+    virtual void setup();
+
+    /**
+     * Given a starting index, return the index of the next token on channel.
+     * Return {@code i} if {@code tokens[i]} is on channel. Return the index of
+     * the EOF token if there are no tokens on channel between {@code i} and
+     * EOF.
+     */
+    virtual ssize_t nextTokenOnChannel(size_t i, size_t channel);
+
+    /**
+     * Given a starting index, return the index of the previous token on
+     * channel. Return {@code i} if {@code tokens[i]} is on channel. Return -1
+     * if there are no tokens on channel between {@code i} and 0.
+     *
+     * <p>
+     * If {@code i} specifies an index at or after the EOF token, the EOF token
+     * index is returned. This is due to the fact that the EOF token is treated
+     * as though it were on every channel.</p>
+     */
+    virtual ssize_t previousTokenOnChannel(size_t i, size_t channel);
+
+    virtual std::vector<Token *> filterForChannel(size_t from, size_t to, ssize_t channel);
+
+    bool isInitialized() const;
+
+  private:
+    bool _needSetup;
+    void InitializeInstanceFields();
+  };
+
+} // namespace antlr4

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/CharStream.cpp

@@ -0,0 +1,11 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "CharStream.h"
+
+using namespace antlr4;
+
+CharStream::~CharStream() {
+}

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/CharStream.h

@@ -0,0 +1,37 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "IntStream.h"
+#include "misc/Interval.h"
+
+namespace antlr4 {
+
+  /// A source of characters for an ANTLR lexer.
+  class ANTLR4CPP_PUBLIC CharStream : public IntStream {
+  public:
+    virtual ~CharStream();
+
+    /// This method returns the text for a range of characters within this input
+    /// stream. This method is guaranteed to not throw an exception if the
+    /// specified interval lies entirely within a marked range. For more
+    /// information about marked ranges, see IntStream::mark.
+    ///
+    /// <param name="interval"> an interval within the stream </param>
+    /// <returns> the text of the specified interval
+    /// </returns>
+    /// <exception cref="NullPointerException"> if {@code interval} is {@code null} </exception>
+    /// <exception cref="IllegalArgumentException"> if {@code interval.a < 0}, or if
+    /// {@code interval.b < interval.a - 1}, or if {@code interval.b} lies at or
+    /// past the end of the stream </exception>
+    /// <exception cref="UnsupportedOperationException"> if the stream does not support
+    /// getting the text of the specified interval </exception>
+    virtual std::string getText(const misc::Interval &interval) = 0;
+
+    virtual std::string toString() const = 0;
+  };
+
+} // namespace antlr4

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/CommonToken.cpp

@@ -0,0 +1,195 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "TokenSource.h"
+#include "CharStream.h"
+#include "Recognizer.h"
+#include "Vocabulary.h"
+
+#include "misc/Interval.h"
+
+#include "support/StringUtils.h"
+#include "support/CPPUtils.h"
+
+#include "CommonToken.h"
+
+using namespace antlr4;
+using namespace antlr4::misc;
+
+using namespace antlrcpp;
+
+const std::pair<TokenSource*, CharStream*> CommonToken::EMPTY_SOURCE;
+
+CommonToken::CommonToken(size_t type) {
+  InitializeInstanceFields();
+  _type = type;
+}
+
+CommonToken::CommonToken(std::pair<TokenSource*, CharStream*> source, size_t type, size_t channel, size_t start, size_t stop) {
+  InitializeInstanceFields();
+  _source = source;
+  _type = type;
+  _channel = channel;
+  _start = start;
+  _stop = stop;
+  if (_source.first != nullptr) {
+    _line = static_cast<int>(source.first->getLine());
+    _charPositionInLine = source.first->getCharPositionInLine();
+  }
+}
+
+CommonToken::CommonToken(size_t type, const std::string &text) {
+  InitializeInstanceFields();
+  _type = type;
+  _channel = DEFAULT_CHANNEL;
+  _text = text;
+  _source = EMPTY_SOURCE;
+}
+
+CommonToken::CommonToken(Token *oldToken) {
+  InitializeInstanceFields();
+  _type = oldToken->getType();
+  _line = oldToken->getLine();
+  _index = oldToken->getTokenIndex();
+  _charPositionInLine = oldToken->getCharPositionInLine();
+  _channel = oldToken->getChannel();
+  _start = oldToken->getStartIndex();
+  _stop = oldToken->getStopIndex();
+
+  if (is<CommonToken *>(oldToken)) {
+    _text = (static_cast<CommonToken *>(oldToken))->_text;
+    _source = (static_cast<CommonToken *>(oldToken))->_source;
+  } else {
+    _text = oldToken->getText();
+    _source = { oldToken->getTokenSource(), oldToken->getInputStream() };
+  }
+}
+
+size_t CommonToken::getType() const {
+  return _type;
+}
+
+void CommonToken::setLine(size_t line) {
+  _line = line;
+}
+
+std::string CommonToken::getText() const {
+  if (!_text.empty()) {
+    return _text;
+  }
+
+  CharStream *input = getInputStream();
+  if (input == nullptr) {
+    return "";
+  }
+  size_t n = input->size();
+  if (_start < n && _stop < n) {
+    return input->getText(misc::Interval(_start, _stop));
+  } else {
+    return "<EOF>";
+  }
+}
+
+void CommonToken::setText(const std::string &text) {
+  _text = text;
+}
+
+size_t CommonToken::getLine() const {
+  return _line;
+}
+
+size_t CommonToken::getCharPositionInLine() const {
+  return _charPositionInLine;
+}
+
+void CommonToken::setCharPositionInLine(size_t charPositionInLine) {
+  _charPositionInLine = charPositionInLine;
+}
+
+size_t CommonToken::getChannel() const {
+  return _channel;
+}
+
+void CommonToken::setChannel(size_t channel) {
+  _channel = channel;
+}
+
+void CommonToken::setType(size_t type) {
+  _type = type;
+}
+
+size_t CommonToken::getStartIndex() const {
+  return _start;
+}
+
+void CommonToken::setStartIndex(size_t start) {
+  _start = start;
+}
+
+size_t CommonToken::getStopIndex() const {
+  return _stop;
+}
+
+void CommonToken::setStopIndex(size_t stop) {
+  _stop = stop;
+}
+
+size_t CommonToken::getTokenIndex() const {
+  return _index;
+}
+
+void CommonToken::setTokenIndex(size_t index) {
+  _index = index;
+}
+
+antlr4::TokenSource *CommonToken::getTokenSource() const {
+  return _source.first;
+}
+
+antlr4::CharStream *CommonToken::getInputStream() const {
+  return _source.second;
+}
+
+std::string CommonToken::toString() const {
+  return toString(nullptr);
+}
+
+std::string CommonToken::toString(Recognizer *r) const {
+  std::stringstream ss;
+
+  std::string channelStr;
+  if (_channel > 0) {
+    channelStr = ",channel=" + std::to_string(_channel);
+  }
+  std::string txt = getText();
+  if (!txt.empty()) {
+    antlrcpp::replaceAll(txt, "\n", "\\n");
+    antlrcpp::replaceAll(txt, "\r", "\\r");
+    antlrcpp::replaceAll(txt, "\t", "\\t");
+  } else {
+    txt = "<no text>";
+  }
+
+  std::string typeString = std::to_string(symbolToNumeric(_type));
+  if (r != nullptr)
+    typeString = r->getVocabulary().getDisplayName(_type);
+
+  ss << "[@" << symbolToNumeric(getTokenIndex()) << "," << symbolToNumeric(_start) << ":" << symbolToNumeric(_stop)
+    << "='" << txt << "',<" << typeString << ">" << channelStr << "," << _line << ":"
+    << getCharPositionInLine() << "]";
+
+  return ss.str();
+}
+
+void CommonToken::InitializeInstanceFields() {
+  _type = 0;
+  _line = 0;
+  _charPositionInLine = INVALID_INDEX;
+  _channel = DEFAULT_CHANNEL;
+  _index = INVALID_INDEX;
+  _start = 0;
+  _stop = 0;
+  _source = EMPTY_SOURCE;
+}

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/CommonToken.h

@@ -0,0 +1,158 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "WritableToken.h"
+
+namespace antlr4 {
+
+  class ANTLR4CPP_PUBLIC CommonToken : public WritableToken {
+  protected:
+    /**
+     * An empty {@link Pair} which is used as the default value of
+     * {@link #source} for tokens that do not have a source.
+     */
+    static const std::pair<TokenSource *, CharStream *> EMPTY_SOURCE;
+
+    /**
+     * This is the backing field for {@link #getType} and {@link #setType}.
+     */
+    size_t _type;
+
+    /**
+     * This is the backing field for {@link #getLine} and {@link #setLine}.
+     */
+    size_t _line;
+
+    /**
+     * This is the backing field for {@link #getCharPositionInLine} and
+     * {@link #setCharPositionInLine}.
+     */
+    size_t _charPositionInLine; // set to invalid position
+
+    /**
+     * This is the backing field for {@link #getChannel} and
+     * {@link #setChannel}.
+     */
+    size_t _channel;
+
+    /**
+     * This is the backing field for {@link #getTokenSource} and
+     * {@link #getInputStream}.
+     *
+     * <p>
+     * These properties share a field to reduce the memory footprint of
+     * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from
+     * the same source and input stream share a reference to the same
+     * {@link Pair} containing these values.</p>
+     */
+
+    std::pair<TokenSource *, CharStream *> _source; // ml: pure references, usually from statically allocated classes.
+
+    /**
+     * This is the backing field for {@link #getText} when the token text is
+     * explicitly set in the constructor or via {@link #setText}.
+     *
+     * @see #getText()
+     */
+    std::string _text;
+
+    /**
+     * This is the backing field for {@link #getTokenIndex} and
+     * {@link #setTokenIndex}.
+     */
+    size_t _index;
+
+    /**
+     * This is the backing field for {@link #getStartIndex} and
+     * {@link #setStartIndex}.
+     */
+    size_t _start;
+
+    /**
+     * This is the backing field for {@link #getStopIndex} and
+     * {@link #setStopIndex}.
+     */
+    size_t _stop;
+
+  public:
+    /**
+     * Constructs a new {@link CommonToken} with the specified token type.
+     *
+     * @param type The token type.
+     */
+    CommonToken(size_t type);
+    CommonToken(std::pair<TokenSource*, CharStream*> source, size_t type, size_t channel, size_t start, size_t stop);
+
+    /**
+     * Constructs a new {@link CommonToken} with the specified token type and
+     * text.
+     *
+     * @param type The token type.
+     * @param text The text of the token.
+     */
+    CommonToken(size_t type, const std::string &text);
+
+    /**
+     * Constructs a new {@link CommonToken} as a copy of another {@link Token}.
+     *
+     * <p>
+     * If {@code oldToken} is also a {@link CommonToken} instance, the newly
+     * constructed token will share a reference to the {@link #text} field and
+     * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will
+     * be assigned the result of calling {@link #getText}, and {@link #source}
+     * will be constructed from the result of {@link Token#getTokenSource} and
+     * {@link Token#getInputStream}.</p>
+     *
+     * @param oldToken The token to copy.
+     */
+    CommonToken(Token *oldToken);
+
+    virtual size_t getType() const override;
+
+    /**
+     * Explicitly set the text for this token. If {code text} is not
+     * {@code null}, then {@link #getText} will return this value rather than
+     * extracting the text from the input.
+     *
+     * @param text The explicit text of the token, or {@code null} if the text
+     * should be obtained from the input along with the start and stop indexes
+     * of the token.
+     */
+    virtual void setText(const std::string &text) override;
+    virtual std::string getText() const override;
+
+    virtual void setLine(size_t line) override;
+    virtual size_t getLine() const override;
+
+    virtual size_t getCharPositionInLine() const override;
+    virtual void setCharPositionInLine(size_t charPositionInLine) override;
+
+    virtual size_t getChannel() const override;
+    virtual void setChannel(size_t channel) override;
+
+    virtual void setType(size_t type) override;
+
+    virtual size_t getStartIndex() const override;
+    virtual void setStartIndex(size_t start);
+
+    virtual size_t getStopIndex() const override;
+    virtual void setStopIndex(size_t stop);
+
+    virtual size_t getTokenIndex() const override;
+    virtual void setTokenIndex(size_t index) override;
+
+    virtual TokenSource *getTokenSource() const override;
+    virtual CharStream *getInputStream() const override;
+
+    virtual std::string toString() const override;
+
+    virtual std::string toString(Recognizer *r) const;
+  private:
+    void InitializeInstanceFields();
+  };
+
+} // namespace antlr4