python3-parser 1.0.0

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/pattern/ParseTreeMatch.h

@@ -0,0 +1,132 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "antlr4-common.h"
+
+namespace antlr4 {
+namespace tree {
+namespace pattern {
+
+  /// Represents the result of matching a ParseTree against a tree pattern.
+  class ANTLR4CPP_PUBLIC ParseTreeMatch {
+  private:
+    /// This is the backing field for getTree().
+    ParseTree *_tree;
+
+    /// This is the backing field for getPattern().
+    const ParseTreePattern &_pattern;
+
+    /// This is the backing field for getLabels().
+    std::map<std::string, std::vector<ParseTree *>> _labels;
+
+    /// This is the backing field for getMismatchedNode().
+    ParseTree *_mismatchedNode;
+
+  public:
+    /// <summary>
+    /// Constructs a new instance of <seealso cref="ParseTreeMatch"/> from the specified
+    /// parse tree and pattern.
+    /// </summary>
+    /// <param name="tree"> The parse tree to match against the pattern. </param>
+    /// <param name="pattern"> The parse tree pattern. </param>
+    /// <param name="labels"> A mapping from label names to collections of
+    /// <seealso cref="ParseTree"/> objects located by the tree pattern matching process. </param>
+    /// <param name="mismatchedNode"> The first node which failed to match the tree
+    /// pattern during the matching process.
+    /// </param>
+    /// <exception cref="IllegalArgumentException"> if {@code tree} is {@code null} </exception>
+    /// <exception cref="IllegalArgumentException"> if {@code pattern} is {@code null} </exception>
+    /// <exception cref="IllegalArgumentException"> if {@code labels} is {@code null} </exception>
+    ParseTreeMatch(ParseTree *tree, ParseTreePattern const& pattern,
+                   const std::map<std::string, std::vector<ParseTree *>> &labels, ParseTree *mismatchedNode);
+    ParseTreeMatch(ParseTreeMatch const&) = default;
+    virtual ~ParseTreeMatch();
+
+    /// <summary>
+    /// Get the last node associated with a specific {@code label}.
+    /// <p/>
+    /// For example, for pattern {@code <id:ID>}, {@code get("id")} returns the
+    /// node matched for that {@code ID}. If more than one node
+    /// matched the specified label, only the last is returned. If there is
+    /// no node associated with the label, this returns {@code null}.
+    /// <p/>
+    /// Pattern tags like {@code <ID>} and {@code <expr>} without labels are
+    /// considered to be labeled with {@code ID} and {@code expr}, respectively.
+    /// </summary>
+    /// <param name="labe"> The label to check.
+    /// </param>
+    /// <returns> The last <seealso cref="ParseTree"/> to match a tag with the specified
+    /// label, or {@code null} if no parse tree matched a tag with the label. </returns>
+    virtual ParseTree* get(const std::string &label);
+
+    /// <summary>
+    /// Return all nodes matching a rule or token tag with the specified label.
+    /// <p/>
+    /// If the {@code label} is the name of a parser rule or token in the
+    /// grammar, the resulting list will contain both the parse trees matching
+    /// rule or tags explicitly labeled with the label and the complete set of
+    /// parse trees matching the labeled and unlabeled tags in the pattern for
+    /// the parser rule or token. For example, if {@code label} is {@code "foo"},
+    /// the result will contain <em>all</em> of the following.
+    ///
+    /// <ul>
+    /// <li>Parse tree nodes matching tags of the form {@code <foo:anyRuleName>} and
+    /// {@code <foo:AnyTokenName>}.</li>
+    /// <li>Parse tree nodes matching tags of the form {@code <anyLabel:foo>}.</li>
+    /// <li>Parse tree nodes matching tags of the form {@code <foo>}.</li>
+    /// </ul>
+    /// </summary>
+    /// <param name="labe"> The label.
+    /// </param>
+    /// <returns> A collection of all <seealso cref="ParseTree"/> nodes matching tags with
+    /// the specified {@code label}. If no nodes matched the label, an empty list
+    /// is returned. </returns>
+    virtual std::vector<ParseTree *> getAll(const std::string &label);
+
+    /// <summary>
+    /// Return a mapping from label &rarr; [list of nodes].
+    /// <p/>
+    /// The map includes special entries corresponding to the names of rules and
+    /// tokens referenced in tags in the original pattern. For additional
+    /// information, see the description of <seealso cref="#getAll(String)"/>.
+    /// </summary>
+    /// <returns> A mapping from labels to parse tree nodes. If the parse tree
+    /// pattern did not contain any rule or token tags, this map will be empty. </returns>
+    virtual std::map<std::string, std::vector<ParseTree *>>& getLabels();
+
+    /// <summary>
+    /// Get the node at which we first detected a mismatch.
+    /// </summary>
+    /// <returns> the node at which we first detected a mismatch, or {@code null}
+    /// if the match was successful. </returns>
+    virtual ParseTree* getMismatchedNode();
+
+    /// <summary>
+    /// Gets a value indicating whether the match operation succeeded.
+    /// </summary>
+    /// <returns> {@code true} if the match operation succeeded; otherwise,
+    /// {@code false}. </returns>
+    virtual bool succeeded();
+
+    /// <summary>
+    /// Get the tree pattern we are matching against.
+    /// </summary>
+    /// <returns> The tree pattern we are matching against. </returns>
+    virtual const ParseTreePattern& getPattern();
+
+    /// <summary>
+    /// Get the parse tree we are trying to match to a pattern.
+    /// </summary>
+    /// <returns> The <seealso cref="ParseTree"/> we are trying to match to a pattern. </returns>
+    virtual ParseTree* getTree();
+
+    virtual std::string toString();
+  };
+
+} // namespace pattern
+} // namespace tree
+} // namespace antlr4

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/pattern/ParseTreePattern.cpp

@@ -0,0 +1,64 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "tree/ParseTree.h"
+#include "tree/pattern/ParseTreePatternMatcher.h"
+#include "tree/pattern/ParseTreeMatch.h"
+
+#include "tree/xpath/XPath.h"
+#include "tree/xpath/XPathElement.h"
+
+#include "tree/pattern/ParseTreePattern.h"
+
+using namespace antlr4::tree;
+using namespace antlr4::tree::pattern;
+
+using namespace antlrcpp;
+
+ParseTreePattern::ParseTreePattern(ParseTreePatternMatcher *matcher, const std::string &pattern, int patternRuleIndex_,
+                                   ParseTree *patternTree)
+  : patternRuleIndex(patternRuleIndex_), _pattern(pattern), _patternTree(patternTree), _matcher(matcher) {
+}
+
+ParseTreePattern::~ParseTreePattern() {
+}
+
+ParseTreeMatch ParseTreePattern::match(ParseTree *tree) {
+  return _matcher->match(tree, *this);
+}
+
+bool ParseTreePattern::matches(ParseTree *tree) {
+  return _matcher->match(tree, *this).succeeded();
+}
+
+std::vector<ParseTreeMatch> ParseTreePattern::findAll(ParseTree *tree, const std::string &xpath) {
+  xpath::XPath finder(_matcher->getParser(), xpath);
+  std::vector<ParseTree *> subtrees = finder.evaluate(tree);
+  std::vector<ParseTreeMatch> matches;
+  for (auto t : subtrees) {
+    ParseTreeMatch aMatch = match(t);
+    if (aMatch.succeeded()) {
+      matches.push_back(aMatch);
+    }
+  }
+  return matches;
+}
+
+
+ParseTreePatternMatcher *ParseTreePattern::getMatcher() const {
+  return _matcher;
+}
+
+std::string ParseTreePattern::getPattern() const {
+  return _pattern;
+}
+
+int ParseTreePattern::getPatternRuleIndex() const {
+  return patternRuleIndex;
+}
+
+ParseTree* ParseTreePattern::getPatternTree() const {
+  return _patternTree;
+}

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/pattern/ParseTreePattern.h

@@ -0,0 +1,105 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "antlr4-common.h"
+
+namespace antlr4 {
+namespace tree {
+namespace pattern {
+
+  /// <summary>
+  /// A pattern like {@code <ID> = <expr>;} converted to a <seealso cref="ParseTree"/> by
+  /// <seealso cref="ParseTreePatternMatcher#compile(String, int)"/>.
+  /// </summary>
+  class ANTLR4CPP_PUBLIC ParseTreePattern {
+  public:
+    /// <summary>
+    /// Construct a new instance of the <seealso cref="ParseTreePattern"/> class.
+    /// </summary>
+    /// <param name="matcher"> The <seealso cref="ParseTreePatternMatcher"/> which created this
+    /// tree pattern. </param>
+    /// <param name="pattern"> The tree pattern in concrete syntax form. </param>
+    /// <param name="patternRuleIndex"> The parser rule which serves as the root of the
+    /// tree pattern. </param>
+    /// <param name="patternTree"> The tree pattern in <seealso cref="ParseTree"/> form. </param>
+    ParseTreePattern(ParseTreePatternMatcher *matcher, const std::string &pattern, int patternRuleIndex,
+                     ParseTree *patternTree);
+    ParseTreePattern(ParseTreePattern const&) = default;
+    virtual ~ParseTreePattern();
+
+    /// <summary>
+    /// Match a specific parse tree against this tree pattern.
+    /// </summary>
+    /// <param name="tree"> The parse tree to match against this tree pattern. </param>
+    /// <returns> A <seealso cref="ParseTreeMatch"/> object describing the result of the
+    /// match operation. The <seealso cref="ParseTreeMatch#succeeded()"/> method can be
+    /// used to determine whether or not the match was successful. </returns>
+    virtual ParseTreeMatch match(ParseTree *tree);
+
+    /// <summary>
+    /// Determine whether or not a parse tree matches this tree pattern.
+    /// </summary>
+    /// <param name="tree"> The parse tree to match against this tree pattern. </param>
+    /// <returns> {@code true} if {@code tree} is a match for the current tree
+    /// pattern; otherwise, {@code false}. </returns>
+    virtual bool matches(ParseTree *tree);
+
+    /// Find all nodes using XPath and then try to match those subtrees against
+    /// this tree pattern.
+    /// @param tree The ParseTree to match against this pattern.
+    /// @param xpath An expression matching the nodes
+    ///
+    /// @returns A collection of ParseTreeMatch objects describing the
+    /// successful matches. Unsuccessful matches are omitted from the result,
+    /// regardless of the reason for the failure.
+    virtual std::vector<ParseTreeMatch> findAll(ParseTree *tree, const std::string &xpath);
+
+    /// <summary>
+    /// Get the <seealso cref="ParseTreePatternMatcher"/> which created this tree pattern.
+    /// </summary>
+    /// <returns> The <seealso cref="ParseTreePatternMatcher"/> which created this tree
+    /// pattern. </returns>
+    virtual ParseTreePatternMatcher *getMatcher() const;
+
+    /// <summary>
+    /// Get the tree pattern in concrete syntax form.
+    /// </summary>
+    /// <returns> The tree pattern in concrete syntax form. </returns>
+    virtual std::string getPattern() const;
+
+    /// <summary>
+    /// Get the parser rule which serves as the outermost rule for the tree
+    /// pattern.
+    /// </summary>
+    /// <returns> The parser rule which serves as the outermost rule for the tree
+    /// pattern. </returns>
+    virtual int getPatternRuleIndex() const;
+
+    /// <summary>
+    /// Get the tree pattern as a <seealso cref="ParseTree"/>. The rule and token tags from
+    /// the pattern are present in the parse tree as terminal nodes with a symbol
+    /// of type <seealso cref="RuleTagToken"/> or <seealso cref="TokenTagToken"/>.
+    /// </summary>
+    /// <returns> The tree pattern as a <seealso cref="ParseTree"/>. </returns>
+    virtual ParseTree* getPatternTree() const;
+
+  private:
+    const int patternRuleIndex;
+
+    /// This is the backing field for <seealso cref="#getPattern()"/>.
+    const std::string _pattern;
+
+    /// This is the backing field for <seealso cref="#getPatternTree()"/>.
+    ParseTree *_patternTree;
+
+    /// This is the backing field for <seealso cref="#getMatcher()"/>.
+    ParseTreePatternMatcher *const _matcher;
+  };
+
+} // namespace pattern
+} // namespace tree
+} // namespace antlr4

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/pattern/ParseTreePatternMatcher.cpp

@@ -0,0 +1,371 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "tree/pattern/ParseTreePattern.h"
+#include "tree/pattern/ParseTreeMatch.h"
+#include "tree/TerminalNode.h"
+#include "CommonTokenStream.h"
+#include "ParserInterpreter.h"
+#include "tree/pattern/TokenTagToken.h"
+#include "ParserRuleContext.h"
+#include "tree/pattern/RuleTagToken.h"
+#include "tree/pattern/TagChunk.h"
+#include "atn/ATN.h"
+#include "Lexer.h"
+#include "BailErrorStrategy.h"
+
+#include "ListTokenSource.h"
+#include "tree/pattern/TextChunk.h"
+#include "ANTLRInputStream.h"
+#include "support/Arrays.h"
+#include "Exceptions.h"
+#include "support/StringUtils.h"
+#include "support/CPPUtils.h"
+
+#include "tree/pattern/ParseTreePatternMatcher.h"
+
+using namespace antlr4;
+using namespace antlr4::tree;
+using namespace antlr4::tree::pattern;
+using namespace antlrcpp;
+
+ParseTreePatternMatcher::CannotInvokeStartRule::CannotInvokeStartRule(const RuntimeException &e) : RuntimeException(e.what()) {
+}
+
+ParseTreePatternMatcher::CannotInvokeStartRule::~CannotInvokeStartRule() {
+}
+
+ParseTreePatternMatcher::StartRuleDoesNotConsumeFullPattern::~StartRuleDoesNotConsumeFullPattern() {
+}
+
+ParseTreePatternMatcher::ParseTreePatternMatcher(Lexer *lexer, Parser *parser) : _lexer(lexer), _parser(parser) {
+  InitializeInstanceFields();
+}
+
+ParseTreePatternMatcher::~ParseTreePatternMatcher() {
+}
+
+void ParseTreePatternMatcher::setDelimiters(const std::string &start, const std::string &stop, const std::string &escapeLeft) {
+  if (start.empty()) {
+    throw IllegalArgumentException("start cannot be null or empty");
+  }
+
+  if (stop.empty()) {
+    throw IllegalArgumentException("stop cannot be null or empty");
+  }
+
+ _start = start;
+  _stop = stop;
+  _escape = escapeLeft;
+}
+
+bool ParseTreePatternMatcher::matches(ParseTree *tree, const std::string &pattern, int patternRuleIndex) {
+  ParseTreePattern p = compile(pattern, patternRuleIndex);
+  return matches(tree, p);
+}
+
+bool ParseTreePatternMatcher::matches(ParseTree *tree, const ParseTreePattern &pattern) {
+  std::map<std::string, std::vector<ParseTree *>> labels;
+  ParseTree *mismatchedNode = matchImpl(tree, pattern.getPatternTree(), labels);
+  return mismatchedNode == nullptr;
+}
+
+ParseTreeMatch ParseTreePatternMatcher::match(ParseTree *tree, const std::string &pattern, int patternRuleIndex) {
+  ParseTreePattern p = compile(pattern, patternRuleIndex);
+  return match(tree, p);
+}
+
+ParseTreeMatch ParseTreePatternMatcher::match(ParseTree *tree, const ParseTreePattern &pattern) {
+  std::map<std::string, std::vector<ParseTree *>> labels;
+  tree::ParseTree *mismatchedNode = matchImpl(tree, pattern.getPatternTree(), labels);
+  return ParseTreeMatch(tree, pattern, labels, mismatchedNode);
+}
+
+ParseTreePattern ParseTreePatternMatcher::compile(const std::string &pattern, int patternRuleIndex) {
+  ListTokenSource tokenSrc(tokenize(pattern));
+  CommonTokenStream tokens(&tokenSrc);
+
+  ParserInterpreter parserInterp(_parser->getGrammarFileName(), _parser->getVocabulary(),
+                                 _parser->getRuleNames(), _parser->getATNWithBypassAlts(), &tokens);
+
+  ParserRuleContext *tree = nullptr;
+  try {
+    parserInterp.setErrorHandler(std::make_shared<BailErrorStrategy>());
+    tree = parserInterp.parse(patternRuleIndex);
+  } catch (ParseCancellationException &e) {
+#if defined(_MSC_FULL_VER) && _MSC_FULL_VER < 190023026
+    // rethrow_if_nested is not available before VS 2015.
+    throw e;
+#else
+    std::rethrow_if_nested(e); // Unwrap the nested exception.
+#endif
+  } catch (RecognitionException &re) {
+    throw re;
+#if defined(_MSC_FULL_VER) && _MSC_FULL_VER < 190023026
+  } catch (std::exception &e) {
+    // throw_with_nested is not available before VS 2015.
+    throw e;
+#else
+  } catch (std::exception & /*e*/) {
+    std::throw_with_nested((const char*)"Cannot invoke start rule"); // Wrap any other exception. We should however probably use one of the ANTLR exceptions here.
+#endif
+  }
+
+  // Make sure tree pattern compilation checks for a complete parse
+  if (tokens.LA(1) != Token::EOF) {
+    throw StartRuleDoesNotConsumeFullPattern();
+  }
+
+  return ParseTreePattern(this, pattern, patternRuleIndex, tree);
+}
+
+Lexer* ParseTreePatternMatcher::getLexer() {
+  return _lexer;
+}
+
+Parser* ParseTreePatternMatcher::getParser() {
+  return _parser;
+}
+
+ParseTree* ParseTreePatternMatcher::matchImpl(ParseTree *tree, ParseTree *patternTree,
+                                              std::map<std::string, std::vector<ParseTree *>> &labels) {
+  if (tree == nullptr) {
+    throw IllegalArgumentException("tree cannot be nul");
+  }
+
+  if (patternTree == nullptr) {
+    throw IllegalArgumentException("patternTree cannot be nul");
+  }
+
+  // x and <ID>, x and y, or x and x; or could be mismatched types
+  if (is<TerminalNode *>(tree) && is<TerminalNode *>(patternTree)) {
+    TerminalNode *t1 = dynamic_cast<TerminalNode *>(tree);
+    TerminalNode *t2 = dynamic_cast<TerminalNode *>(patternTree);
+
+    ParseTree *mismatchedNode = nullptr;
+    // both are tokens and they have same type
+    if (t1->getSymbol()->getType() == t2->getSymbol()->getType()) {
+      if (is<TokenTagToken *>(t2->getSymbol())) { // x and <ID>
+        TokenTagToken *tokenTagToken = dynamic_cast<TokenTagToken *>(t2->getSymbol());
+
+        // track label->list-of-nodes for both token name and label (if any)
+        labels[tokenTagToken->getTokenName()].push_back(tree);
+        if (tokenTagToken->getLabel() != "") {
+          labels[tokenTagToken->getLabel()].push_back(tree);
+        }
+      } else if (t1->getText() == t2->getText()) {
+        // x and x
+      } else {
+        // x and y
+        if (mismatchedNode == nullptr) {
+          mismatchedNode = t1;
+        }
+      }
+    } else {
+      if (mismatchedNode == nullptr) {
+        mismatchedNode = t1;
+      }
+    }
+
+    return mismatchedNode;
+  }
+
+  if (is<ParserRuleContext *>(tree) && is<ParserRuleContext *>(patternTree)) {
+    ParserRuleContext *r1 = dynamic_cast<ParserRuleContext *>(tree);
+    ParserRuleContext *r2 = dynamic_cast<ParserRuleContext *>(patternTree);
+    ParseTree *mismatchedNode = nullptr;
+
+    // (expr ...) and <expr>
+    RuleTagToken *ruleTagToken = getRuleTagToken(r2);
+    if (ruleTagToken != nullptr) {
+      //ParseTreeMatch *m = nullptr; // unused?
+      if (r1->getRuleIndex() == r2->getRuleIndex()) {
+        // track label->list-of-nodes for both rule name and label (if any)
+        labels[ruleTagToken->getRuleName()].push_back(tree);
+        if (ruleTagToken->getLabel() != "") {
+          labels[ruleTagToken->getLabel()].push_back(tree);
+        }
+      } else {
+        if (!mismatchedNode) {
+          mismatchedNode = r1;
+        }
+      }
+
+      return mismatchedNode;
+    }
+
+    // (expr ...) and (expr ...)
+    if (r1->children.size() != r2->children.size()) {
+      if (mismatchedNode == nullptr) {
+        mismatchedNode = r1;
+      }
+
+      return mismatchedNode;
+    }
+
+    std::size_t n = r1->children.size();
+    for (size_t i = 0; i < n; i++) {
+      ParseTree *childMatch = matchImpl(r1->children[i], patternTree->children[i], labels);
+      if (childMatch) {
+        return childMatch;
+      }
+    }
+
+    return mismatchedNode;
+  }
+
+  // if nodes aren't both tokens or both rule nodes, can't match
+  return tree;
+}
+
+RuleTagToken* ParseTreePatternMatcher::getRuleTagToken(ParseTree *t) {
+  if (t->children.size() == 1 && is<TerminalNode *>(t->children[0])) {
+    TerminalNode *c = dynamic_cast<TerminalNode *>(t->children[0]);
+    if (is<RuleTagToken *>(c->getSymbol())) {
+      return dynamic_cast<RuleTagToken *>(c->getSymbol());
+    }
+  }
+  return nullptr;
+}
+
+std::vector<std::unique_ptr<Token>> ParseTreePatternMatcher::tokenize(const std::string &pattern) {
+  // split pattern into chunks: sea (raw input) and islands (<ID>, <expr>)
+  std::vector<Chunk> chunks = split(pattern);
+
+  // create token stream from text and tags
+  std::vector<std::unique_ptr<Token>> tokens;
+  for (auto chunk : chunks) {
+    if (is<TagChunk *>(&chunk)) {
+      TagChunk &tagChunk = (TagChunk&)chunk;
+      // add special rule token or conjure up new token from name
+      if (isupper(tagChunk.getTag()[0])) {
+        size_t ttype = _parser->getTokenType(tagChunk.getTag());
+        if (ttype == Token::INVALID_TYPE) {
+          throw IllegalArgumentException("Unknown token " + tagChunk.getTag() + " in pattern: " + pattern);
+        }
+        tokens.emplace_back(new TokenTagToken(tagChunk.getTag(), (int)ttype, tagChunk.getLabel()));
+      } else if (islower(tagChunk.getTag()[0])) {
+        size_t ruleIndex = _parser->getRuleIndex(tagChunk.getTag());
+        if (ruleIndex == INVALID_INDEX) {
+          throw IllegalArgumentException("Unknown rule " + tagChunk.getTag() + " in pattern: " + pattern);
+        }
+        size_t ruleImaginaryTokenType = _parser->getATNWithBypassAlts().ruleToTokenType[ruleIndex];
+        tokens.emplace_back(new RuleTagToken(tagChunk.getTag(), ruleImaginaryTokenType, tagChunk.getLabel()));
+      } else {
+        throw IllegalArgumentException("invalid tag: " + tagChunk.getTag() + " in pattern: " + pattern);
+      }
+    } else {
+      TextChunk &textChunk = (TextChunk&)chunk;
+      ANTLRInputStream input(textChunk.getText());
+      _lexer->setInputStream(&input);
+      std::unique_ptr<Token> t(_lexer->nextToken());
+      while (t->getType() != Token::EOF) {
+        tokens.push_back(std::move(t));
+        t = _lexer->nextToken();
+      }
+      _lexer->setInputStream(nullptr);
+    }
+  }
+
+  return tokens;
+}
+
+std::vector<Chunk> ParseTreePatternMatcher::split(const std::string &pattern) {
+  size_t p = 0;
+  size_t n = pattern.length();
+  std::vector<Chunk> chunks;
+
+  // find all start and stop indexes first, then collect
+  std::vector<size_t> starts;
+  std::vector<size_t> stops;
+  while (p < n) {
+    if (p == pattern.find(_escape + _start,p)) {
+      p += _escape.length() + _start.length();
+    } else if (p == pattern.find(_escape + _stop,p)) {
+      p += _escape.length() + _stop.length();
+    } else if (p == pattern.find(_start,p)) {
+      starts.push_back(p);
+      p += _start.length();
+    } else if (p == pattern.find(_stop,p)) {
+      stops.push_back(p);
+      p += _stop.length();
+    } else {
+      p++;
+    }
+  }
+
+  if (starts.size() > stops.size()) {
+    throw IllegalArgumentException("unterminated tag in pattern: " + pattern);
+  }
+
+  if (starts.size() < stops.size()) {
+    throw IllegalArgumentException("missing start tag in pattern: " + pattern);
+  }
+
+  size_t ntags = starts.size();
+  for (size_t i = 0; i < ntags; i++) {
+    if (starts[i] >= stops[i]) {
+      throw IllegalArgumentException("tag delimiters out of order in pattern: " + pattern);
+    }
+  }
+
+  // collect into chunks now
+  if (ntags == 0) {
+    std::string text = pattern.substr(0, n);
+    chunks.push_back(TextChunk(text));
+  }
+
+  if (ntags > 0 && starts[0] > 0) { // copy text up to first tag into chunks
+    std::string text = pattern.substr(0, starts[0]);
+    chunks.push_back(TextChunk(text));
+  }
+
+  for (size_t i = 0; i < ntags; i++) {
+    // copy inside of <tag>
+    std::string tag = pattern.substr(starts[i] + _start.length(), stops[i] - (starts[i] + _start.length()));
+    std::string ruleOrToken = tag;
+    std::string label = "";
+    size_t colon = tag.find(':');
+    if (colon != std::string::npos) {
+      label = tag.substr(0,colon);
+      ruleOrToken = tag.substr(colon + 1, tag.length() - (colon + 1));
+    }
+    chunks.push_back(TagChunk(label, ruleOrToken));
+    if (i + 1 < ntags) {
+      // copy from end of <tag> to start of next
+      std::string text = pattern.substr(stops[i] + _stop.length(), starts[i + 1] - (stops[i] + _stop.length()));
+      chunks.push_back(TextChunk(text));
+    }
+  }
+
+  if (ntags > 0) {
+    size_t afterLastTag = stops[ntags - 1] + _stop.length();
+    if (afterLastTag < n) { // copy text from end of last tag to end
+      std::string text = pattern.substr(afterLastTag, n - afterLastTag);
+      chunks.push_back(TextChunk(text));
+    }
+  }
+
+  // strip out all backslashes from text chunks but not tags
+  for (size_t i = 0; i < chunks.size(); i++) {
+    Chunk &c = chunks[i];
+    if (is<TextChunk *>(&c)) {
+      TextChunk &tc = (TextChunk&)c;
+      std::string unescaped = tc.getText();
+      unescaped.erase(std::remove(unescaped.begin(), unescaped.end(), '\\'), unescaped.end());
+      if (unescaped.length() < tc.getText().length()) {
+        chunks[i] = TextChunk(unescaped);
+      }
+    }
+  }
+
+  return chunks;
+}
+
+void ParseTreePatternMatcher::InitializeInstanceFields() {
+  _start = "<";
+  _stop = ">";
+  _escape = "\\";
+}