expressir 1.4.3 → 2.1.0

data/ext/express_parser/antlr4-upstream/runtime/Cpp/runtime/src/TokenStreamRewriter.h

@@ -1,295 +0,0 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-#pragma once
-
-#include "antlr4-common.h"
-
-namespace antlr4 {
-
-  /**
-   * Useful for rewriting out a buffered input token stream after doing some
-   * augmentation or other manipulations on it.
-   *
-   * <p>
-   * You can insert stuff, replace, and delete chunks. Note that the operations
-   * are done lazily--only if you convert the buffer to a {@link String} with
-   * {@link TokenStream#getText()}. This is very efficient because you are not
-   * moving data around all the time. As the buffer of tokens is converted to
-   * strings, the {@link #getText()} method(s) scan the input token stream and
-   * check to see if there is an operation at the current index. If so, the
-   * operation is done and then normal {@link String} rendering continues on the
-   * buffer. This is like having multiple Turing machine instruction streams
-   * (programs) operating on a single input tape. :)</p>
-   *
-   * <p>
-   * This rewriter makes no modifications to the token stream. It does not ask the
-   * stream to fill itself up nor does it advance the input cursor. The token
-   * stream {@link TokenStream#index()} will return the same value before and
-   * after any {@link #getText()} call.</p>
-   *
-   * <p>
-   * The rewriter only works on tokens that you have in the buffer and ignores the
-   * current input cursor. If you are buffering tokens on-demand, calling
-   * {@link #getText()} halfway through the input will only do rewrites for those
-   * tokens in the first half of the file.</p>
-   *
-   * <p>
-   * Since the operations are done lazily at {@link #getText}-time, operations do
-   * not screw up the token index values. That is, an insert operation at token
-   * index {@code i} does not change the index values for tokens
-   * {@code i}+1..n-1.</p>
-   *
-   * <p>
-   * Because operations never actually alter the buffer, you may always get the
-   * original token stream back without undoing anything. Since the instructions
-   * are queued up, you can easily simulate transactions and roll back any changes
-   * if there is an error just by removing instructions. For example,</p>
-   *
-   * <pre>
-   * CharStream input = new ANTLRFileStream("input");
-   * TLexer lex = new TLexer(input);
-   * CommonTokenStream tokens = new CommonTokenStream(lex);
-   * T parser = new T(tokens);
-   * TokenStreamRewriter rewriter = new TokenStreamRewriter(tokens);
-   * parser.startRule();
-   * </pre>
-   *
-   * <p>
-   * Then in the rules, you can execute (assuming rewriter is visible):</p>
-   *
-   * <pre>
-   * Token t,u;
-   * ...
-   * rewriter.insertAfter(t, "text to put after t");}
-   * rewriter.insertAfter(u, "text after u");}
-   * System.out.println(rewriter.getText());
-   * </pre>
-   *
-   * <p>
-   * You can also have multiple "instruction streams" and get multiple rewrites
-   * from a single pass over the input. Just name the instruction streams and use
-   * that name again when printing the buffer. This could be useful for generating
-   * a C file and also its header file--all from the same buffer:</p>
-   *
-   * <pre>
-   * rewriter.insertAfter("pass1", t, "text to put after t");}
-   * rewriter.insertAfter("pass2", u, "text after u");}
-   * System.out.println(rewriter.getText("pass1"));
-   * System.out.println(rewriter.getText("pass2"));
-   * </pre>
-   *
-   * <p>
-   * If you don't use named rewrite streams, a "default" stream is used as the
-   * first example shows.</p>
-   */
-  class ANTLR4CPP_PUBLIC TokenStreamRewriter {
-  public:
-    static const std::string DEFAULT_PROGRAM_NAME;
-    static constexpr size_t PROGRAM_INIT_SIZE = 100;
-    static constexpr size_t MIN_TOKEN_INDEX = 0;
-
-    TokenStreamRewriter(TokenStream *tokens);
-    virtual ~TokenStreamRewriter();
-
-    TokenStream *getTokenStream();
-
-    virtual void rollback(size_t instructionIndex);
-
-    /// Rollback the instruction stream for a program so that
-    /// the indicated instruction (via instructionIndex) is no
-    /// longer in the stream.  UNTESTED!
-    virtual void rollback(const std::string &programName, size_t instructionIndex);
-
-    virtual void deleteProgram();
-
-    /// Reset the program so that no instructions exist.
-    virtual void deleteProgram(const std::string &programName);
-    virtual void insertAfter(Token *t, const std::string& text);
-    virtual void insertAfter(size_t index, const std::string& text);
-    virtual void insertAfter(const std::string &programName, Token *t, const std::string& text);
-    virtual void insertAfter(const std::string &programName, size_t index, const std::string& text);
-
-    virtual void insertBefore(Token *t, const std::string& text);
-    virtual void insertBefore(size_t index, const std::string& text);
-    virtual void insertBefore(const std::string &programName, Token *t, const std::string& text);
-    virtual void insertBefore(const std::string &programName, size_t index, const std::string& text);
-
-    virtual void replace(size_t index, const std::string& text);
-    virtual void replace(size_t from, size_t to, const std::string& text);
-    virtual void replace(Token *indexT, const std::string& text);
-    virtual void replace(Token *from, Token *to, const std::string& text);
-    virtual void replace(const std::string &programName, size_t from, size_t to, const std::string& text);
-    virtual void replace(const std::string &programName, Token *from, Token *to, const std::string& text);
-
-    virtual void Delete(size_t index);
-    virtual void Delete(size_t from, size_t to);
-    virtual void Delete(Token *indexT);
-    virtual void Delete(Token *from, Token *to);
-    virtual void Delete(const std::string &programName, size_t from, size_t to);
-    virtual void Delete(const std::string &programName, Token *from, Token *to);
-
-    virtual size_t getLastRewriteTokenIndex();
-
-    /// Return the text from the original tokens altered per the
-    /// instructions given to this rewriter.
-    virtual std::string getText();
-
-    /** Return the text from the original tokens altered per the
-     *  instructions given to this rewriter in programName.
-     */
-    std::string getText(std::string programName);
-
-    /// Return the text associated with the tokens in the interval from the
-    /// original token stream but with the alterations given to this rewriter.
-    /// The interval refers to the indexes in the original token stream.
-    /// We do not alter the token stream in any way, so the indexes
-    /// and intervals are still consistent. Includes any operations done
-    /// to the first and last token in the interval. So, if you did an
-    /// insertBefore on the first token, you would get that insertion.
-    /// The same is true if you do an insertAfter the stop token.
-    virtual std::string getText(const misc::Interval &interval);
-
-    virtual std::string getText(const std::string &programName, const misc::Interval &interval);
-
-  protected:
-    class RewriteOperation {
-    public:
-      /// What index into rewrites List are we?
-      size_t index;
-      std::string text;
-
-      /// Token buffer index.
-      size_t instructionIndex;
-
-      RewriteOperation(TokenStreamRewriter *outerInstance, size_t index);
-      RewriteOperation(TokenStreamRewriter *outerInstance, size_t index, const std::string& text);
-      virtual ~RewriteOperation();
-
-      /// Execute the rewrite operation by possibly adding to the buffer.
-      /// Return the index of the next token to operate on.
-
-      virtual size_t execute(std::string *buf);
-      virtual std::string toString();
-
-    private:
-      TokenStreamRewriter *const outerInstance;
-      void InitializeInstanceFields();
-    };
-
-    class InsertBeforeOp : public RewriteOperation {
-    private:
-      TokenStreamRewriter *const outerInstance;
-
-    public:
-      InsertBeforeOp(TokenStreamRewriter *outerInstance, size_t index, const std::string& text);
-
-      virtual size_t execute(std::string *buf) override;
-    };
-
-    class ReplaceOp : public RewriteOperation {
-    private:
-      TokenStreamRewriter *const outerInstance;
-
-    public:
-      size_t lastIndex;
-
-      ReplaceOp(TokenStreamRewriter *outerInstance, size_t from, size_t to, const std::string& text);
-      virtual size_t execute(std::string *buf) override;
-      virtual std::string toString() override;
-
-    private:
-      void InitializeInstanceFields();
-    };
-
-    /// Our source stream
-    TokenStream *const tokens;
-
-    /// You may have multiple, named streams of rewrite operations.
-    /// I'm calling these things "programs."
-    /// Maps String (name) -> rewrite (List)
-    std::map<std::string, std::vector<RewriteOperation*>> _programs;
-
-    /// <summary>
-    /// Map String (program name) -> Integer index </summary>
-    std::map<std::string, size_t> _lastRewriteTokenIndexes;
-    virtual size_t getLastRewriteTokenIndex(const std::string &programName);
-    virtual void setLastRewriteTokenIndex(const std::string &programName, size_t i);
-    virtual std::vector<RewriteOperation*>& getProgram(const std::string &name);
-
-    /// <summary>
-    /// We need to combine operations and report invalid operations (like
-    ///  overlapping replaces that are not completed nested).  Inserts to
-    ///  same index need to be combined etc...   Here are the cases:
-    ///
-    ///  I.i.u I.j.v                                leave alone, nonoverlapping
-    ///  I.i.u I.i.v                                combine: Iivu
-    ///
-    ///  R.i-j.u R.x-y.v    | i-j in x-y            delete first R
-    ///  R.i-j.u R.i-j.v                            delete first R
-    ///  R.i-j.u R.x-y.v    | x-y in i-j            ERROR
-    ///  R.i-j.u R.x-y.v    | boundaries overlap    ERROR
-    ///
-    ///  Delete special case of replace (text==null):
-    ///  D.i-j.u D.x-y.v    | boundaries overlap    combine to max(min)..max(right)
-    ///
-    ///  I.i.u R.x-y.v | i in (x+1)-y           delete I (since insert before
-    ///                                         we're not deleting i)
-    ///  I.i.u R.x-y.v | i not in (x+1)-y       leave alone, nonoverlapping
-    ///  R.x-y.v I.i.u | i in x-y               ERROR
-    ///  R.x-y.v I.x.u                          R.x-y.uv (combine, delete I)
-    ///  R.x-y.v I.i.u | i not in x-y           leave alone, nonoverlapping
-    ///
-    ///  I.i.u = insert u before op @ index i
-    ///  R.x-y.u = replace x-y indexed tokens with u
-    ///
-    ///  First we need to examine replaces.  For any replace op:
-    ///
-    ///         1. wipe out any insertions before op within that range.
-    ///     2. Drop any replace op before that is contained completely within
-    ///         that range.
-    ///     3. Throw exception upon boundary overlap with any previous replace.
-    ///
-    ///  Then we can deal with inserts:
-    ///
-    ///         1. for any inserts to same index, combine even if not adjacent.
-    ///         2. for any prior replace with same left boundary, combine this
-    ///         insert with replace and delete this replace.
-    ///         3. throw exception if index in same range as previous replace
-    ///
-    ///  Don't actually delete; make op null in list. Easier to walk list.
-    ///  Later we can throw as we add to index -> op map.
-    ///
-    ///  Note that I.2 R.2-2 will wipe out I.2 even though, technically, the
-    ///  inserted stuff would be before the replace range.  But, if you
-    ///  add tokens in front of a method body '{' and then delete the method
-    ///  body, I think the stuff before the '{' you added should disappear too.
-    ///
-    ///  Return a map from token index to operation.
-    /// </summary>
-    virtual std::unordered_map<size_t, RewriteOperation*> reduceToSingleOperationPerIndex(std::vector<RewriteOperation*> &rewrites);
-
-    virtual std::string catOpText(std::string *a, std::string *b);
-
-    /// Get all operations before an index of a particular kind.
-    template <typename T>
-    std::vector<T *> getKindOfOps(std::vector<RewriteOperation *> rewrites, size_t before) {
-      std::vector<T *> ops;
-      for (size_t i = 0; i < before && i < rewrites.size(); i++) {
-        T *op = dynamic_cast<T *>(rewrites[i]);
-        if (op == nullptr) { // ignore deleted or non matching entries
-          continue;
-        }
-        ops.push_back(op);
-      }
-      return ops;
-    }
-
-  private:
-    std::vector<RewriteOperation *>& initializeProgram(const std::string &name);
-
-  };
-
-} // namespace antlr4

data/ext/express_parser/antlr4-upstream/runtime/Cpp/runtime/src/UnbufferedCharStream.cpp

@@ -1,208 +0,0 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-#include "misc/Interval.h"
-#include "Exceptions.h"
-#include "support/Utf8.h"
-
-#include "UnbufferedCharStream.h"
-
-using namespace antlrcpp;
-using namespace antlr4;
-using namespace antlr4::misc;
-
-UnbufferedCharStream::UnbufferedCharStream(std::wistream &input)
-    : _p(0), _numMarkers(0), _lastChar(0), _lastCharBufferStart(0), _currentCharIndex(0), _input(input) {
-  // The vector's size is what used to be n in Java code.
-  fill(1); // prime
-}
-
-void UnbufferedCharStream::consume() {
-  if (LA(1) == EOF) {
-    throw IllegalStateException("cannot consume EOF");
-  }
-
-  // buf always has at least data[p==0] in this method due to ctor
-  _lastChar = _data[_p]; // track last char for LA(-1)
-
-  if (_p == _data.size() - 1 && _numMarkers == 0) {
-    size_t capacity = _data.capacity();
-    _data.clear();
-    _data.reserve(capacity);
-
-    _p = 0;
-    _lastCharBufferStart = _lastChar;
-  } else {
-    _p++;
-  }
-
-  _currentCharIndex++;
-  sync(1);
-}
-
-void UnbufferedCharStream::sync(size_t want) {
-  if (_p + want <= _data.size()) // Already enough data loaded?
-    return;
-
-  fill(_p + want - _data.size());
-}
-
-size_t UnbufferedCharStream::fill(size_t n) {
-  for (size_t i = 0; i < n; i++) {
-    if (_data.size() > 0 && _data.back() == 0xFFFF) {
-      return i;
-    }
-
-    try {
-      char32_t c = nextChar();
-      add(c);
-#if defined(_MSC_FULL_VER) && _MSC_FULL_VER < 190023026
-    } catch (IOException &ioe) {
-      // throw_with_nested is not available before VS 2015.
-      throw ioe;
-#else
-    } catch (IOException & /*ioe*/) {
-      std::throw_with_nested(RuntimeException());
-#endif
-    }
-  }
-
-  return n;
-}
-
-char32_t UnbufferedCharStream::nextChar()  {
-  return _input.get();
-}
-
-void UnbufferedCharStream::add(char32_t c) {
-  _data += c;
-}
-
-size_t UnbufferedCharStream::LA(ssize_t i) {
-  if (i == -1) { // special case
-    return _lastChar;
-  }
-
-  // We can look back only as many chars as we have buffered.
-  ssize_t index = static_cast<ssize_t>(_p) + i - 1;
-  if (index < 0) {
-    throw IndexOutOfBoundsException();
-  }
-
-  if (i > 0) {
-    sync(static_cast<size_t>(i)); // No need to sync if we look back.
-  }
-  if (static_cast<size_t>(index) >= _data.size()) {
-    return EOF;
-  }
-
-  if (_data[static_cast<size_t>(index)] == std::char_traits<wchar_t>::eof()) {
-    return EOF;
-  }
-
-  return _data[static_cast<size_t>(index)];
-}
-
-ssize_t UnbufferedCharStream::mark() {
-  if (_numMarkers == 0) {
-    _lastCharBufferStart = _lastChar;
-  }
-
-  ssize_t mark = -static_cast<ssize_t>(_numMarkers) - 1;
-  _numMarkers++;
-  return mark;
-}
-
-void UnbufferedCharStream::release(ssize_t marker) {
-  ssize_t expectedMark = -static_cast<ssize_t>(_numMarkers);
-  if (marker != expectedMark) {
-    throw IllegalStateException("release() called with an invalid marker.");
-  }
-
-  _numMarkers--;
-  if (_numMarkers == 0 && _p > 0) {
-    _data.erase(0, _p);
-    _p = 0;
-    _lastCharBufferStart = _lastChar;
-  }
-}
-
-size_t UnbufferedCharStream::index() {
-  return _currentCharIndex;
-}
-
-void UnbufferedCharStream::seek(size_t index) {
-  if (index == _currentCharIndex) {
-    return;
-  }
-
-  if (index > _currentCharIndex) {
-    sync(index - _currentCharIndex);
-    index = std::min(index, getBufferStartIndex() + _data.size() - 1);
-  }
-
-  // index == to bufferStartIndex should set p to 0
-  ssize_t i = static_cast<ssize_t>(index) - static_cast<ssize_t>(getBufferStartIndex());
-  if (i < 0) {
-    throw IllegalArgumentException(std::string("cannot seek to negative index ") + std::to_string(index));
-  } else if (i >= static_cast<ssize_t>(_data.size())) {
-    throw UnsupportedOperationException("Seek to index outside buffer: " + std::to_string(index) +
-                                        " not in " + std::to_string(getBufferStartIndex()) + ".." +
-                                        std::to_string(getBufferStartIndex() + _data.size()));
-  }
-
-  _p = static_cast<size_t>(i);
-  _currentCharIndex = index;
-  if (_p == 0) {
-    _lastChar = _lastCharBufferStart;
-  } else {
-    _lastChar = _data[_p - 1];
-  }
-}
-
-size_t UnbufferedCharStream::size() {
-  throw UnsupportedOperationException("Unbuffered stream cannot know its size");
-}
-
-std::string UnbufferedCharStream::getSourceName() const {
-  if (name.empty()) {
-    return UNKNOWN_SOURCE_NAME;
-  }
-
-  return name;
-}
-
-std::string UnbufferedCharStream::getText(const misc::Interval &interval) {
-  if (interval.a < 0 || interval.b < interval.a - 1) {
-    throw IllegalArgumentException("invalid interval");
-  }
-
-  size_t bufferStartIndex = getBufferStartIndex();
-  if (!_data.empty() && _data.back() == 0xFFFF) {
-    if (interval.a + interval.length() > bufferStartIndex + _data.size()) {
-      throw IllegalArgumentException("the interval extends past the end of the stream");
-    }
-  }
-
-  if (interval.a < static_cast<ssize_t>(bufferStartIndex) || interval.b >= ssize_t(bufferStartIndex + _data.size())) {
-    throw UnsupportedOperationException("interval " + interval.toString() + " outside buffer: " +
-      std::to_string(bufferStartIndex) + ".." + std::to_string(bufferStartIndex + _data.size() - 1));
-  }
-  // convert from absolute to local index
-  size_t i = interval.a - bufferStartIndex;
-  auto maybeUtf8 = Utf8::strictEncode(std::u32string_view(_data).substr(i, interval.length()));
-  if (!maybeUtf8.has_value()) {
-    throw IllegalArgumentException("Unbuffered stream contains invalid Unicode code points");
-  }
-  return std::move(maybeUtf8).value();
-}
-
-std::string UnbufferedCharStream::toString() const {
-  throw UnsupportedOperationException("Unbuffered stream cannot be materialized to a string");
-}
-
-size_t UnbufferedCharStream::getBufferStartIndex() const {
-  return _currentCharIndex - _p;
-}

data/ext/express_parser/antlr4-upstream/runtime/Cpp/runtime/src/UnbufferedCharStream.h

@@ -1,117 +0,0 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-#pragma once
-
-#include "CharStream.h"
-
-namespace antlr4 {
-
-  /// Do not buffer up the entire char stream. It does keep a small buffer
-  /// for efficiency and also buffers while a mark exists (set by the
-  /// lookahead prediction in parser). "Unbuffered" here refers to fact
-  /// that it doesn't buffer all data, not that's it's on demand loading of char.
-  class ANTLR4CPP_PUBLIC UnbufferedCharStream : public CharStream {
-  public:
-    /// The name or source of this char stream.
-    std::string name;
-
-    explicit UnbufferedCharStream(std::wistream &input);
-
-    void consume() override;
-    size_t LA(ssize_t i) override;
-
-    /// <summary>
-    /// Return a marker that we can release later.
-    /// <p/>
-    /// The specific marker value used for this class allows for some level of
-    /// protection against misuse where {@code seek()} is called on a mark or
-    /// {@code release()} is called in the wrong order.
-    /// </summary>
-    ssize_t mark() override;
-
-    /// <summary>
-    /// Decrement number of markers, resetting buffer if we hit 0. </summary>
-    /// <param name="marker"> </param>
-    void release(ssize_t marker) override;
-    size_t index() override;
-
-    /// <summary>
-    /// Seek to absolute character index, which might not be in the current
-    ///  sliding window.  Move {@code p} to {@code index-bufferStartIndex}.
-    /// </summary>
-    void seek(size_t index) override;
-    size_t size() override;
-    std::string getSourceName() const override;
-    std::string getText(const misc::Interval &interval) override;
-
-    std::string toString() const override;
-
-  protected:
-    /// A moving window buffer of the data being scanned. While there's a marker,
-    /// we keep adding to buffer. Otherwise, <seealso cref="#consume consume()"/> resets so
-    /// we start filling at index 0 again.
-    // UTF-32 encoded.
-    std::u32string _data;
-    typedef char32_t storage_type;
-
-    /// <summary>
-    /// 0..n-1 index into <seealso cref="#data data"/> of next character.
-    /// <p/>
-    /// The {@code LA(1)} character is {@code data[p]}. If {@code p == n}, we are
-    /// out of buffered characters.
-    /// </summary>
-    size_t _p;
-
-    /// <summary>
-    /// Count up with <seealso cref="#mark mark()"/> and down with
-    /// <seealso cref="#release release()"/>. When we {@code release()} the last mark,
-    /// {@code numMarkers} reaches 0 and we reset the buffer. Copy
-    /// {@code data[p]..data[n-1]} to {@code data[0]..data[(n-1)-p]}.
-    /// </summary>
-    size_t _numMarkers;
-
-    /// This is the {@code LA(-1)} character for the current position.
-    size_t _lastChar; // UTF-32
-
-    /// <summary>
-    /// When {@code numMarkers > 0}, this is the {@code LA(-1)} character for the
-    /// first character in <seealso cref="#data data"/>. Otherwise, this is unspecified.
-    /// </summary>
-    size_t _lastCharBufferStart; // UTF-32
-
-    /// <summary>
-    /// Absolute character index. It's the index of the character about to be
-    /// read via {@code LA(1)}. Goes from 0 to the number of characters in the
-    /// entire stream, although the stream size is unknown before the end is
-    /// reached.
-    /// </summary>
-    size_t _currentCharIndex;
-
-    std::wistream &_input;
-
-    /// <summary>
-    /// Make sure we have 'want' elements from current position <seealso cref="#p p"/>.
-    /// Last valid {@code p} index is {@code data.length-1}. {@code p+need-1} is
-    /// the char index 'need' elements ahead. If we need 1 element,
-    /// {@code (p+1-1)==p} must be less than {@code data.length}.
-    /// </summary>
-    virtual void sync(size_t want);
-
-    /// <summary>
-    /// Add {@code n} characters to the buffer. Returns the number of characters
-    /// actually added to the buffer. If the return value is less than {@code n},
-    /// then EOF was reached before {@code n} characters could be added.
-    /// </summary>
-    virtual size_t fill(size_t n);
-
-    /// Override to provide different source of characters than
-    /// <seealso cref="#input input"/>.
-    virtual char32_t nextChar();
-    virtual void add(char32_t c);
-    size_t getBufferStartIndex() const;
-  };
-
-} // namespace antlr4