grpc 1.21.0 → 1.32.0

data/third_party/re2/re2/filtered_re2.cc

@@ -0,0 +1,121 @@
+// Copyright 2009 The RE2 Authors.  All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "re2/filtered_re2.h"
+
+#include <stddef.h>
+#include <string>
+
+#include "util/util.h"
+#include "util/logging.h"
+#include "re2/prefilter.h"
+#include "re2/prefilter_tree.h"
+
+namespace re2 {
+
+FilteredRE2::FilteredRE2()
+    : compiled_(false),
+      prefilter_tree_(new PrefilterTree()) {
+}
+
+FilteredRE2::FilteredRE2(int min_atom_len)
+    : compiled_(false),
+      prefilter_tree_(new PrefilterTree(min_atom_len)) {
+}
+
+FilteredRE2::~FilteredRE2() {
+  for (size_t i = 0; i < re2_vec_.size(); i++)
+    delete re2_vec_[i];
+  delete prefilter_tree_;
+}
+
+RE2::ErrorCode FilteredRE2::Add(const StringPiece& pattern,
+                                const RE2::Options& options, int* id) {
+  RE2* re = new RE2(pattern, options);
+  RE2::ErrorCode code = re->error_code();
+
+  if (!re->ok()) {
+    if (options.log_errors()) {
+      LOG(ERROR) << "Couldn't compile regular expression, skipping: "
+                 << re << " due to error " << re->error();
+    }
+    delete re;
+  } else {
+    *id = static_cast<int>(re2_vec_.size());
+    re2_vec_.push_back(re);
+  }
+
+  return code;
+}
+
+void FilteredRE2::Compile(std::vector<std::string>* atoms) {
+  if (compiled_) {
+    LOG(ERROR) << "Compile called already.";
+    return;
+  }
+
+  if (re2_vec_.empty()) {
+    LOG(ERROR) << "Compile called before Add.";
+    return;
+  }
+
+  for (size_t i = 0; i < re2_vec_.size(); i++) {
+    Prefilter* prefilter = Prefilter::FromRE2(re2_vec_[i]);
+    prefilter_tree_->Add(prefilter);
+  }
+  atoms->clear();
+  prefilter_tree_->Compile(atoms);
+  compiled_ = true;
+}
+
+int FilteredRE2::SlowFirstMatch(const StringPiece& text) const {
+  for (size_t i = 0; i < re2_vec_.size(); i++)
+    if (RE2::PartialMatch(text, *re2_vec_[i]))
+      return static_cast<int>(i);
+  return -1;
+}
+
+int FilteredRE2::FirstMatch(const StringPiece& text,
+                            const std::vector<int>& atoms) const {
+  if (!compiled_) {
+    LOG(DFATAL) << "FirstMatch called before Compile.";
+    return -1;
+  }
+  std::vector<int> regexps;
+  prefilter_tree_->RegexpsGivenStrings(atoms, &regexps);
+  for (size_t i = 0; i < regexps.size(); i++)
+    if (RE2::PartialMatch(text, *re2_vec_[regexps[i]]))
+      return regexps[i];
+  return -1;
+}
+
+bool FilteredRE2::AllMatches(
+    const StringPiece& text,
+    const std::vector<int>& atoms,
+    std::vector<int>* matching_regexps) const {
+  matching_regexps->clear();
+  std::vector<int> regexps;
+  prefilter_tree_->RegexpsGivenStrings(atoms, &regexps);
+  for (size_t i = 0; i < regexps.size(); i++)
+    if (RE2::PartialMatch(text, *re2_vec_[regexps[i]]))
+      matching_regexps->push_back(regexps[i]);
+  return !matching_regexps->empty();
+}
+
+void FilteredRE2::AllPotentials(
+    const std::vector<int>& atoms,
+    std::vector<int>* potential_regexps) const {
+  prefilter_tree_->RegexpsGivenStrings(atoms, potential_regexps);
+}
+
+void FilteredRE2::RegexpsGivenStrings(const std::vector<int>& matched_atoms,
+                                      std::vector<int>* passed_regexps) {
+  prefilter_tree_->RegexpsGivenStrings(matched_atoms, passed_regexps);
+}
+
+void FilteredRE2::PrintPrefilter(int regexpid) {
+  prefilter_tree_->PrintPrefilter(regexpid);
+}
+
+}  // namespace re2

data/third_party/re2/re2/filtered_re2.h

@@ -0,0 +1,109 @@
+// Copyright 2009 The RE2 Authors.  All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#ifndef RE2_FILTERED_RE2_H_
+#define RE2_FILTERED_RE2_H_
+
+// The class FilteredRE2 is used as a wrapper to multiple RE2 regexps.
+// It provides a prefilter mechanism that helps in cutting down the
+// number of regexps that need to be actually searched.
+//
+// By design, it does not include a string matching engine. This is to
+// allow the user of the class to use their favorite string matching
+// engine. The overall flow is: Add all the regexps using Add, then
+// Compile the FilteredRE2. Compile returns strings that need to be
+// matched. Note that the returned strings are lowercased and distinct.
+// For applying regexps to a search text, the caller does the string
+// matching using the returned strings. When doing the string match,
+// note that the caller has to do that in a case-insensitive way or
+// on a lowercased version of the search text. Then call FirstMatch
+// or AllMatches with a vector of indices of strings that were found
+// in the text to get the actual regexp matches.
+
+#include <string>
+#include <vector>
+
+#include "re2/re2.h"
+
+namespace re2 {
+
+class PrefilterTree;
+
+class FilteredRE2 {
+ public:
+  FilteredRE2();
+  explicit FilteredRE2(int min_atom_len);
+  ~FilteredRE2();
+
+  // Uses RE2 constructor to create a RE2 object (re). Returns
+  // re->error_code(). If error_code is other than NoError, then re is
+  // deleted and not added to re2_vec_.
+  RE2::ErrorCode Add(const StringPiece& pattern,
+                     const RE2::Options& options,
+                     int *id);
+
+  // Prepares the regexps added by Add for filtering.  Returns a set
+  // of strings that the caller should check for in candidate texts.
+  // The returned strings are lowercased and distinct. When doing
+  // string matching, it should be performed in a case-insensitive
+  // way or the search text should be lowercased first.  Call after
+  // all Add calls are done.
+  void Compile(std::vector<std::string>* strings_to_match);
+
+  // Returns the index of the first matching regexp.
+  // Returns -1 on no match. Can be called prior to Compile.
+  // Does not do any filtering: simply tries to Match the
+  // regexps in a loop.
+  int SlowFirstMatch(const StringPiece& text) const;
+
+  // Returns the index of the first matching regexp.
+  // Returns -1 on no match. Compile has to be called before
+  // calling this.
+  int FirstMatch(const StringPiece& text,
+                 const std::vector<int>& atoms) const;
+
+  // Returns the indices of all matching regexps, after first clearing
+  // matched_regexps.
+  bool AllMatches(const StringPiece& text,
+                  const std::vector<int>& atoms,
+                  std::vector<int>* matching_regexps) const;
+
+  // Returns the indices of all potentially matching regexps after first
+  // clearing potential_regexps.
+  // A regexp is potentially matching if it passes the filter.
+  // If a regexp passes the filter it may still not match.
+  // A regexp that does not pass the filter is guaranteed to not match.
+  void AllPotentials(const std::vector<int>& atoms,
+                     std::vector<int>* potential_regexps) const;
+
+  // The number of regexps added.
+  int NumRegexps() const { return static_cast<int>(re2_vec_.size()); }
+
+  // Get the individual RE2 objects.
+  const RE2& GetRE2(int regexpid) const { return *re2_vec_[regexpid]; }
+
+ private:
+  // Print prefilter.
+  void PrintPrefilter(int regexpid);
+
+  // Useful for testing and debugging.
+  void RegexpsGivenStrings(const std::vector<int>& matched_atoms,
+                           std::vector<int>* passed_regexps);
+
+  // All the regexps in the FilteredRE2.
+  std::vector<RE2*> re2_vec_;
+
+  // Has the FilteredRE2 been compiled using Compile()
+  bool compiled_;
+
+  // An AND-OR tree of string atoms used for filtering regexps.
+  PrefilterTree* prefilter_tree_;
+
+  FilteredRE2(const FilteredRE2&) = delete;
+  FilteredRE2& operator=(const FilteredRE2&) = delete;
+};
+
+}  // namespace re2
+
+#endif  // RE2_FILTERED_RE2_H_

data/third_party/re2/re2/mimics_pcre.cc

@@ -0,0 +1,197 @@
+// Copyright 2008 The RE2 Authors.  All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Determine whether this library should match PCRE exactly
+// for a particular Regexp.  (If so, the testing framework can
+// check that it does.)
+//
+// This library matches PCRE except in these cases:
+//   * the regexp contains a repetition of an empty string,
+//     like (a*)* or (a*)+.  In this case, PCRE will treat
+//     the repetition sequence as ending with an empty string,
+//     while this library does not.
+//   * Perl and PCRE differ on whether \v matches \n.
+//     For historical reasons, this library implements the Perl behavior.
+//   * Perl and PCRE allow $ in one-line mode to match either the very
+//     end of the text or just before a \n at the end of the text.
+//     This library requires it to match only the end of the text.
+//   * Similarly, Perl and PCRE do not allow ^ in multi-line mode to
+//     match the end of the text if the last character is a \n.
+//     This library does allow it.
+//
+// Regexp::MimicsPCRE checks for any of these conditions.
+
+#include "util/util.h"
+#include "util/logging.h"
+#include "re2/regexp.h"
+#include "re2/walker-inl.h"
+
+namespace re2 {
+
+// Returns whether re might match an empty string.
+static bool CanBeEmptyString(Regexp *re);
+
+// Walker class to compute whether library handles a regexp
+// exactly as PCRE would.  See comment at top for conditions.
+
+class PCREWalker : public Regexp::Walker<bool> {
+ public:
+  PCREWalker() {}
+
+  virtual bool PostVisit(Regexp* re, bool parent_arg, bool pre_arg,
+                         bool* child_args, int nchild_args);
+
+  virtual bool ShortVisit(Regexp* re, bool a) {
+    // Should never be called: we use Walk(), not WalkExponential().
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    LOG(DFATAL) << "PCREWalker::ShortVisit called";
+#endif
+    return a;
+  }
+
+ private:
+  PCREWalker(const PCREWalker&) = delete;
+  PCREWalker& operator=(const PCREWalker&) = delete;
+};
+
+// Called after visiting each of re's children and accumulating
+// the return values in child_args.  So child_args contains whether
+// this library mimics PCRE for those subexpressions.
+bool PCREWalker::PostVisit(Regexp* re, bool parent_arg, bool pre_arg,
+                           bool* child_args, int nchild_args) {
+  // If children failed, so do we.
+  for (int i = 0; i < nchild_args; i++)
+    if (!child_args[i])
+      return false;
+
+  // Otherwise look for other reasons to fail.
+  switch (re->op()) {
+    // Look for repeated empty string.
+    case kRegexpStar:
+    case kRegexpPlus:
+    case kRegexpQuest:
+      if (CanBeEmptyString(re->sub()[0]))
+        return false;
+      break;
+    case kRegexpRepeat:
+      if (re->max() == -1 && CanBeEmptyString(re->sub()[0]))
+        return false;
+      break;
+
+    // Look for \v
+    case kRegexpLiteral:
+      if (re->rune() == '\v')
+        return false;
+      break;
+
+    // Look for $ in single-line mode.
+    case kRegexpEndText:
+    case kRegexpEmptyMatch:
+      if (re->parse_flags() & Regexp::WasDollar)
+        return false;
+      break;
+
+    // Look for ^ in multi-line mode.
+    case kRegexpBeginLine:
+      // No condition: in single-line mode ^ becomes kRegexpBeginText.
+      return false;
+
+    default:
+      break;
+  }
+
+  // Not proven guilty.
+  return true;
+}
+
+// Returns whether this regexp's behavior will mimic PCRE's exactly.
+bool Regexp::MimicsPCRE() {
+  PCREWalker w;
+  return w.Walk(this, true);
+}
+
+
+// Walker class to compute whether a Regexp can match an empty string.
+// It is okay to overestimate.  For example, \b\B cannot match an empty
+// string, because \b and \B are mutually exclusive, but this isn't
+// that smart and will say it can.  Spurious empty strings
+// will reduce the number of regexps we sanity check against PCRE,
+// but they won't break anything.
+
+class EmptyStringWalker : public Regexp::Walker<bool> {
+ public:
+  EmptyStringWalker() {}
+
+  virtual bool PostVisit(Regexp* re, bool parent_arg, bool pre_arg,
+                         bool* child_args, int nchild_args);
+
+  virtual bool ShortVisit(Regexp* re, bool a) {
+    // Should never be called: we use Walk(), not WalkExponential().
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    LOG(DFATAL) << "EmptyStringWalker::ShortVisit called";
+#endif
+    return a;
+  }
+
+ private:
+  EmptyStringWalker(const EmptyStringWalker&) = delete;
+  EmptyStringWalker& operator=(const EmptyStringWalker&) = delete;
+};
+
+// Called after visiting re's children.  child_args contains the return
+// value from each of the children's PostVisits (i.e., whether each child
+// can match an empty string).  Returns whether this clause can match an
+// empty string.
+bool EmptyStringWalker::PostVisit(Regexp* re, bool parent_arg, bool pre_arg,
+                                  bool* child_args, int nchild_args) {
+  switch (re->op()) {
+    case kRegexpNoMatch:               // never empty
+    case kRegexpLiteral:
+    case kRegexpAnyChar:
+    case kRegexpAnyByte:
+    case kRegexpCharClass:
+    case kRegexpLiteralString:
+      return false;
+
+    case kRegexpEmptyMatch:            // always empty
+    case kRegexpBeginLine:             // always empty, when they match
+    case kRegexpEndLine:
+    case kRegexpNoWordBoundary:
+    case kRegexpWordBoundary:
+    case kRegexpBeginText:
+    case kRegexpEndText:
+    case kRegexpStar:                  // can always be empty
+    case kRegexpQuest:
+    case kRegexpHaveMatch:
+      return true;
+
+    case kRegexpConcat:                // can be empty if all children can
+      for (int i = 0; i < nchild_args; i++)
+        if (!child_args[i])
+          return false;
+      return true;
+
+    case kRegexpAlternate:             // can be empty if any child can
+      for (int i = 0; i < nchild_args; i++)
+        if (child_args[i])
+          return true;
+      return false;
+
+    case kRegexpPlus:                  // can be empty if the child can
+    case kRegexpCapture:
+      return child_args[0];
+
+    case kRegexpRepeat:                // can be empty if child can or is x{0}
+      return child_args[0] || re->min() == 0;
+  }
+  return false;
+}
+
+// Returns whether re can match an empty string.
+static bool CanBeEmptyString(Regexp* re) {
+  EmptyStringWalker w;
+  return w.Walk(re, true);
+}
+
+}  // namespace re2

data/third_party/re2/re2/nfa.cc

@@ -0,0 +1,713 @@
+// Copyright 2006-2007 The RE2 Authors.  All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Tested by search_test.cc.
+//
+// Prog::SearchNFA, an NFA search.
+// This is an actual NFA like the theorists talk about,
+// not the pseudo-NFA found in backtracking regexp implementations.
+//
+// IMPLEMENTATION
+//
+// This algorithm is a variant of one that appeared in Rob Pike's sam editor,
+// which is a variant of the one described in Thompson's 1968 CACM paper.
+// See http://swtch.com/~rsc/regexp/ for various history.  The main feature
+// over the DFA implementation is that it tracks submatch boundaries.
+//
+// When the choice of submatch boundaries is ambiguous, this particular
+// implementation makes the same choices that traditional backtracking
+// implementations (in particular, Perl and PCRE) do.
+// Note that unlike in Perl and PCRE, this algorithm *cannot* take exponential
+// time in the length of the input.
+//
+// Like Thompson's original machine and like the DFA implementation, this
+// implementation notices a match only once it is one byte past it.
+
+#include <stdio.h>
+#include <string.h>
+#include <algorithm>
+#include <deque>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "util/logging.h"
+#include "util/strutil.h"
+#include "re2/pod_array.h"
+#include "re2/prog.h"
+#include "re2/regexp.h"
+#include "re2/sparse_array.h"
+#include "re2/sparse_set.h"
+
+namespace re2 {
+
+static const bool ExtraDebug = false;
+
+class NFA {
+ public:
+  NFA(Prog* prog);
+  ~NFA();
+
+  // Searches for a matching string.
+  //   * If anchored is true, only considers matches starting at offset.
+  //     Otherwise finds lefmost match at or after offset.
+  //   * If longest is true, returns the longest match starting
+  //     at the chosen start point.  Otherwise returns the so-called
+  //     left-biased match, the one traditional backtracking engines
+  //     (like Perl and PCRE) find.
+  // Records submatch boundaries in submatch[1..nsubmatch-1].
+  // Submatch[0] is the entire match.  When there is a choice in
+  // which text matches each subexpression, the submatch boundaries
+  // are chosen to match what a backtracking implementation would choose.
+  bool Search(const StringPiece& text, const StringPiece& context,
+              bool anchored, bool longest,
+              StringPiece* submatch, int nsubmatch);
+
+ private:
+  struct Thread {
+    union {
+      int ref;
+      Thread* next;  // when on free list
+    };
+    const char** capture;
+  };
+
+  // State for explicit stack in AddToThreadq.
+  struct AddState {
+    int id;     // Inst to process
+    Thread* t;  // if not null, set t0 = t before processing id
+  };
+
+  // Threadq is a list of threads.  The list is sorted by the order
+  // in which Perl would explore that particular state -- the earlier
+  // choices appear earlier in the list.
+  typedef SparseArray<Thread*> Threadq;
+
+  inline Thread* AllocThread();
+  inline Thread* Incref(Thread* t);
+  inline void Decref(Thread* t);
+
+  // Follows all empty arrows from id0 and enqueues all the states reached.
+  // Enqueues only the ByteRange instructions that match byte c.
+  // context is used (with p) for evaluating empty-width specials.
+  // p is the current input position, and t0 is the current thread.
+  void AddToThreadq(Threadq* q, int id0, int c, const StringPiece& context,
+                    const char* p, Thread* t0);
+
+  // Run runq on byte c, appending new states to nextq.
+  // Updates matched_ and match_ as new, better matches are found.
+  // context is used (with p) for evaluating empty-width specials.
+  // p is the position of byte c in the input string for AddToThreadq;
+  // p-1 will be used when processing Match instructions.
+  // Frees all the threads on runq.
+  // If there is a shortcut to the end, returns that shortcut.
+  int Step(Threadq* runq, Threadq* nextq, int c, const StringPiece& context,
+           const char* p);
+
+  // Returns text version of capture information, for debugging.
+  std::string FormatCapture(const char** capture);
+
+  void CopyCapture(const char** dst, const char** src) {
+    memmove(dst, src, ncapture_*sizeof src[0]);
+  }
+
+  Prog* prog_;                // underlying program
+  int start_;                 // start instruction in program
+  int ncapture_;              // number of submatches to track
+  bool longest_;              // whether searching for longest match
+  bool endmatch_;             // whether match must end at text.end()
+  const char* btext_;         // beginning of text (for FormatSubmatch)
+  const char* etext_;         // end of text (for endmatch_)
+  Threadq q0_, q1_;           // pre-allocated for Search.
+  PODArray<AddState> stack_;  // pre-allocated for AddToThreadq
+  std::deque<Thread> arena_;  // thread arena
+  Thread* freelist_;          // thread freelist
+  const char** match_;        // best match so far
+  bool matched_;              // any match so far?
+
+  NFA(const NFA&) = delete;
+  NFA& operator=(const NFA&) = delete;
+};
+
+NFA::NFA(Prog* prog) {
+  prog_ = prog;
+  start_ = prog_->start();
+  ncapture_ = 0;
+  longest_ = false;
+  endmatch_ = false;
+  btext_ = NULL;
+  etext_ = NULL;
+  q0_.resize(prog_->size());
+  q1_.resize(prog_->size());
+  // See NFA::AddToThreadq() for why this is so.
+  int nstack = 2*prog_->inst_count(kInstCapture) +
+               prog_->inst_count(kInstEmptyWidth) +
+               prog_->inst_count(kInstNop) + 1;  // + 1 for start inst
+  stack_ = PODArray<AddState>(nstack);
+  freelist_ = NULL;
+  match_ = NULL;
+  matched_ = false;
+}
+
+NFA::~NFA() {
+  delete[] match_;
+  for (const Thread& t : arena_)
+    delete[] t.capture;
+}
+
+NFA::Thread* NFA::AllocThread() {
+  Thread* t = freelist_;
+  if (t != NULL) {
+    freelist_ = t->next;
+    t->ref = 1;
+    // We don't need to touch t->capture because
+    // the caller will immediately overwrite it.
+    return t;
+  }
+  arena_.emplace_back();
+  t = &arena_.back();
+  t->ref = 1;
+  t->capture = new const char*[ncapture_];
+  return t;
+}
+
+NFA::Thread* NFA::Incref(Thread* t) {
+  DCHECK(t != NULL);
+  t->ref++;
+  return t;
+}
+
+void NFA::Decref(Thread* t) {
+  DCHECK(t != NULL);
+  t->ref--;
+  if (t->ref > 0)
+    return;
+  DCHECK_EQ(t->ref, 0);
+  t->next = freelist_;
+  freelist_ = t;
+}
+
+// Follows all empty arrows from id0 and enqueues all the states reached.
+// Enqueues only the ByteRange instructions that match byte c.
+// context is used (with p) for evaluating empty-width specials.
+// p is the current input position, and t0 is the current thread.
+void NFA::AddToThreadq(Threadq* q, int id0, int c, const StringPiece& context,
+                       const char* p, Thread* t0) {
+  if (id0 == 0)
+    return;
+
+  // Use stack_ to hold our stack of instructions yet to process.
+  // It was preallocated as follows:
+  //   two entries per Capture;
+  //   one entry per EmptyWidth; and
+  //   one entry per Nop.
+  // This reflects the maximum number of stack pushes that each can
+  // perform. (Each instruction can be processed at most once.)
+  AddState* stk = stack_.data();
+  int nstk = 0;
+
+  stk[nstk++] = {id0, NULL};
+  while (nstk > 0) {
+    DCHECK_LE(nstk, stack_.size());
+    AddState a = stk[--nstk];
+
+  Loop:
+    if (a.t != NULL) {
+      // t0 was a thread that we allocated and copied in order to
+      // record the capture, so we must now decref it.
+      Decref(t0);
+      t0 = a.t;
+    }
+
+    int id = a.id;
+    if (id == 0)
+      continue;
+    if (q->has_index(id)) {
+      if (ExtraDebug)
+        fprintf(stderr, "  [%d%s]\n", id, FormatCapture(t0->capture).c_str());
+      continue;
+    }
+
+    // Create entry in q no matter what.  We might fill it in below,
+    // or we might not.  Even if not, it is necessary to have it,
+    // so that we don't revisit id0 during the recursion.
+    q->set_new(id, NULL);
+    Thread** tp = &q->get_existing(id);
+    int j;
+    Thread* t;
+    Prog::Inst* ip = prog_->inst(id);
+    switch (ip->opcode()) {
+    default:
+      LOG(DFATAL) << "unhandled " << ip->opcode() << " in AddToThreadq";
+      break;
+
+    case kInstFail:
+      break;
+
+    case kInstAltMatch:
+      // Save state; will pick up at next byte.
+      t = Incref(t0);
+      *tp = t;
+
+      DCHECK(!ip->last());
+      a = {id+1, NULL};
+      goto Loop;
+
+    case kInstNop:
+      if (!ip->last())
+        stk[nstk++] = {id+1, NULL};
+
+      // Continue on.
+      a = {ip->out(), NULL};
+      goto Loop;
+
+    case kInstCapture:
+      if (!ip->last())
+        stk[nstk++] = {id+1, NULL};
+
+      if ((j=ip->cap()) < ncapture_) {
+        // Push a dummy whose only job is to restore t0
+        // once we finish exploring this possibility.
+        stk[nstk++] = {0, t0};
+
+        // Record capture.
+        t = AllocThread();
+        CopyCapture(t->capture, t0->capture);
+        t->capture[j] = p;
+        t0 = t;
+      }
+      a = {ip->out(), NULL};
+      goto Loop;
+
+    case kInstByteRange:
+      if (!ip->Matches(c))
+        goto Next;
+
+      // Save state; will pick up at next byte.
+      t = Incref(t0);
+      *tp = t;
+      if (ExtraDebug)
+        fprintf(stderr, " + %d%s\n", id, FormatCapture(t0->capture).c_str());
+
+      if (ip->hint() == 0)
+        break;
+      a = {id+ip->hint(), NULL};
+      goto Loop;
+
+    case kInstMatch:
+      // Save state; will pick up at next byte.
+      t = Incref(t0);
+      *tp = t;
+      if (ExtraDebug)
+        fprintf(stderr, " ! %d%s\n", id, FormatCapture(t0->capture).c_str());
+
+    Next:
+      if (ip->last())
+        break;
+      a = {id+1, NULL};
+      goto Loop;
+
+    case kInstEmptyWidth:
+      if (!ip->last())
+        stk[nstk++] = {id+1, NULL};
+
+      // Continue on if we have all the right flag bits.
+      if (ip->empty() & ~Prog::EmptyFlags(context, p))
+        break;
+      a = {ip->out(), NULL};
+      goto Loop;
+    }
+  }
+}
+
+// Run runq on byte c, appending new states to nextq.
+// Updates matched_ and match_ as new, better matches are found.
+// context is used (with p) for evaluating empty-width specials.
+// p is the position of byte c in the input string for AddToThreadq;
+// p-1 will be used when processing Match instructions.
+// Frees all the threads on runq.
+// If there is a shortcut to the end, returns that shortcut.
+int NFA::Step(Threadq* runq, Threadq* nextq, int c, const StringPiece& context,
+              const char* p) {
+  nextq->clear();
+
+  for (Threadq::iterator i = runq->begin(); i != runq->end(); ++i) {
+    Thread* t = i->value();
+    if (t == NULL)
+      continue;
+
+    if (longest_) {
+      // Can skip any threads started after our current best match.
+      if (matched_ && match_[0] < t->capture[0]) {
+        Decref(t);
+        continue;
+      }
+    }
+
+    int id = i->index();
+    Prog::Inst* ip = prog_->inst(id);
+
+    switch (ip->opcode()) {
+      default:
+        // Should only see the values handled below.
+        LOG(DFATAL) << "Unhandled " << ip->opcode() << " in step";
+        break;
+
+      case kInstByteRange:
+        AddToThreadq(nextq, ip->out(), c, context, p, t);
+        break;
+
+      case kInstAltMatch:
+        if (i != runq->begin())
+          break;
+        // The match is ours if we want it.
+        if (ip->greedy(prog_) || longest_) {
+          CopyCapture(match_, t->capture);
+          matched_ = true;
+
+          Decref(t);
+          for (++i; i != runq->end(); ++i) {
+            if (i->value() != NULL)
+              Decref(i->value());
+          }
+          runq->clear();
+          if (ip->greedy(prog_))
+            return ip->out1();
+          return ip->out();
+        }
+        break;
+
+      case kInstMatch: {
+        // Avoid invoking undefined behavior (arithmetic on a null pointer)
+        // by storing p instead of p-1. (What would the latter even mean?!)
+        // This complements the special case in NFA::Search().
+        if (p == NULL) {
+          CopyCapture(match_, t->capture);
+          match_[1] = p;
+          matched_ = true;
+          break;
+        }
+
+        if (endmatch_ && p-1 != etext_)
+          break;
+
+        if (longest_) {
+          // Leftmost-longest mode: save this match only if
+          // it is either farther to the left or at the same
+          // point but longer than an existing match.
+          if (!matched_ || t->capture[0] < match_[0] ||
+              (t->capture[0] == match_[0] && p-1 > match_[1])) {
+            CopyCapture(match_, t->capture);
+            match_[1] = p-1;
+            matched_ = true;
+          }
+        } else {
+          // Leftmost-biased mode: this match is by definition
+          // better than what we've already found (see next line).
+          CopyCapture(match_, t->capture);
+          match_[1] = p-1;
+          matched_ = true;
+
+          // Cut off the threads that can only find matches
+          // worse than the one we just found: don't run the
+          // rest of the current Threadq.
+          Decref(t);
+          for (++i; i != runq->end(); ++i) {
+            if (i->value() != NULL)
+              Decref(i->value());
+          }
+          runq->clear();
+          return 0;
+        }
+        break;
+      }
+    }
+    Decref(t);
+  }
+  runq->clear();
+  return 0;
+}
+
+std::string NFA::FormatCapture(const char** capture) {
+  std::string s;
+  for (int i = 0; i < ncapture_; i+=2) {
+    if (capture[i] == NULL)
+      s += "(?,?)";
+    else if (capture[i+1] == NULL)
+      s += StringPrintf("(%td,?)",
+                        capture[i] - btext_);
+    else
+      s += StringPrintf("(%td,%td)",
+                        capture[i] - btext_,
+                        capture[i+1] - btext_);
+  }
+  return s;
+}
+
+bool NFA::Search(const StringPiece& text, const StringPiece& const_context,
+            bool anchored, bool longest,
+            StringPiece* submatch, int nsubmatch) {
+  if (start_ == 0)
+    return false;
+
+  StringPiece context = const_context;
+  if (context.data() == NULL)
+    context = text;
+
+  // Sanity check: make sure that text lies within context.
+  if (text.begin() < context.begin() || text.end() > context.end()) {
+    LOG(DFATAL) << "context does not contain text";
+    return false;
+  }
+
+  if (prog_->anchor_start() && context.begin() != text.begin())
+    return false;
+  if (prog_->anchor_end() && context.end() != text.end())
+    return false;
+  anchored |= prog_->anchor_start();
+  if (prog_->anchor_end()) {
+    longest = true;
+    endmatch_ = true;
+  }
+
+  if (nsubmatch < 0) {
+    LOG(DFATAL) << "Bad args: nsubmatch=" << nsubmatch;
+    return false;
+  }
+
+  // Save search parameters.
+  ncapture_ = 2*nsubmatch;
+  longest_ = longest;
+
+  if (nsubmatch == 0) {
+    // We need to maintain match[0], both to distinguish the
+    // longest match (if longest is true) and also to tell
+    // whether we've seen any matches at all.
+    ncapture_ = 2;
+  }
+
+  match_ = new const char*[ncapture_];
+  memset(match_, 0, ncapture_*sizeof match_[0]);
+  matched_ = false;
+
+  // For debugging prints.
+  btext_ = context.data();
+  // For convenience.
+  etext_ = text.data() + text.size();
+
+  if (ExtraDebug)
+    fprintf(stderr, "NFA::Search %s (context: %s) anchored=%d longest=%d\n",
+            std::string(text).c_str(), std::string(context).c_str(), anchored, longest);
+
+  // Set up search.
+  Threadq* runq = &q0_;
+  Threadq* nextq = &q1_;
+  runq->clear();
+  nextq->clear();
+
+  // Loop over the text, stepping the machine.
+  for (const char* p = text.data();; p++) {
+    if (ExtraDebug) {
+      int c = 0;
+      if (p == btext_)
+        c = '^';
+      else if (p > etext_)
+        c = '$';
+      else if (p < etext_)
+        c = p[0] & 0xFF;
+
+      fprintf(stderr, "%c:", c);
+      for (Threadq::iterator i = runq->begin(); i != runq->end(); ++i) {
+        Thread* t = i->value();
+        if (t == NULL)
+          continue;
+        fprintf(stderr, " %d%s", i->index(), FormatCapture(t->capture).c_str());
+      }
+      fprintf(stderr, "\n");
+    }
+
+    // This is a no-op the first time around the loop because runq is empty.
+    int id = Step(runq, nextq, p < etext_ ? p[0] & 0xFF : -1, context, p);
+    DCHECK_EQ(runq->size(), 0);
+    using std::swap;
+    swap(nextq, runq);
+    nextq->clear();
+    if (id != 0) {
+      // We're done: full match ahead.
+      p = etext_;
+      for (;;) {
+        Prog::Inst* ip = prog_->inst(id);
+        switch (ip->opcode()) {
+          default:
+            LOG(DFATAL) << "Unexpected opcode in short circuit: " << ip->opcode();
+            break;
+
+          case kInstCapture:
+            if (ip->cap() < ncapture_)
+              match_[ip->cap()] = p;
+            id = ip->out();
+            continue;
+
+          case kInstNop:
+            id = ip->out();
+            continue;
+
+          case kInstMatch:
+            match_[1] = p;
+            matched_ = true;
+            break;
+        }
+        break;
+      }
+      break;
+    }
+
+    if (p > etext_)
+      break;
+
+    // Start a new thread if there have not been any matches.
+    // (No point in starting a new thread if there have been
+    // matches, since it would be to the right of the match
+    // we already found.)
+    if (!matched_ && (!anchored || p == text.data())) {
+      // Try to use prefix accel (e.g. memchr) to skip ahead.
+      // The search must be unanchored and there must be zero
+      // possible matches already.
+      if (!anchored && runq->size() == 0 &&
+          p < etext_ && prog_->can_prefix_accel()) {
+        p = reinterpret_cast<const char*>(prog_->PrefixAccel(p, etext_ - p));
+        if (p == NULL)
+          p = etext_;
+      }
+
+      Thread* t = AllocThread();
+      CopyCapture(t->capture, match_);
+      t->capture[0] = p;
+      AddToThreadq(runq, start_, p < etext_ ? p[0] & 0xFF : -1, context, p,
+                   t);
+      Decref(t);
+    }
+
+    // If all the threads have died, stop early.
+    if (runq->size() == 0) {
+      if (ExtraDebug)
+        fprintf(stderr, "dead\n");
+      break;
+    }
+
+    // Avoid invoking undefined behavior (arithmetic on a null pointer)
+    // by simply not continuing the loop.
+    // This complements the special case in NFA::Step().
+    if (p == NULL) {
+      (void)Step(runq, nextq, p < etext_ ? p[0] & 0xFF : -1, context, p);
+      DCHECK_EQ(runq->size(), 0);
+      using std::swap;
+      swap(nextq, runq);
+      nextq->clear();
+      break;
+    }
+  }
+
+  for (Threadq::iterator i = runq->begin(); i != runq->end(); ++i) {
+    if (i->value() != NULL)
+      Decref(i->value());
+  }
+
+  if (matched_) {
+    for (int i = 0; i < nsubmatch; i++)
+      submatch[i] =
+          StringPiece(match_[2 * i],
+                      static_cast<size_t>(match_[2 * i + 1] - match_[2 * i]));
+    if (ExtraDebug)
+      fprintf(stderr, "match (%td,%td)\n",
+              match_[0] - btext_,
+              match_[1] - btext_);
+    return true;
+  }
+  return false;
+}
+
+bool
+Prog::SearchNFA(const StringPiece& text, const StringPiece& context,
+                Anchor anchor, MatchKind kind,
+                StringPiece* match, int nmatch) {
+  if (ExtraDebug)
+    Dump();
+
+  NFA nfa(this);
+  StringPiece sp;
+  if (kind == kFullMatch) {
+    anchor = kAnchored;
+    if (nmatch == 0) {
+      match = &sp;
+      nmatch = 1;
+    }
+  }
+  if (!nfa.Search(text, context, anchor == kAnchored, kind != kFirstMatch, match, nmatch))
+    return false;
+  if (kind == kFullMatch && match[0].end() != text.end())
+    return false;
+  return true;
+}
+
+// For each instruction i in the program reachable from the start, compute the
+// number of instructions reachable from i by following only empty transitions
+// and record that count as fanout[i].
+//
+// fanout holds the results and is also the work queue for the outer iteration.
+// reachable holds the reached nodes for the inner iteration.
+void Prog::Fanout(SparseArray<int>* fanout) {
+  DCHECK_EQ(fanout->max_size(), size());
+  SparseSet reachable(size());
+  fanout->clear();
+  fanout->set_new(start(), 0);
+  for (SparseArray<int>::iterator i = fanout->begin(); i != fanout->end(); ++i) {
+    int* count = &i->value();
+    reachable.clear();
+    reachable.insert(i->index());
+    for (SparseSet::iterator j = reachable.begin(); j != reachable.end(); ++j) {
+      int id = *j;
+      Prog::Inst* ip = inst(id);
+      switch (ip->opcode()) {
+        default:
+          LOG(DFATAL) << "unhandled " << ip->opcode() << " in Prog::Fanout()";
+          break;
+
+        case kInstByteRange:
+          if (!ip->last())
+            reachable.insert(id+1);
+
+          (*count)++;
+          if (!fanout->has_index(ip->out())) {
+            fanout->set_new(ip->out(), 0);
+          }
+          break;
+
+        case kInstAltMatch:
+          DCHECK(!ip->last());
+          reachable.insert(id+1);
+          break;
+
+        case kInstCapture:
+        case kInstEmptyWidth:
+        case kInstNop:
+          if (!ip->last())
+            reachable.insert(id+1);
+
+          reachable.insert(ip->out());
+          break;
+
+        case kInstMatch:
+          if (!ip->last())
+            reachable.insert(id+1);
+          break;
+
+        case kInstFail:
+          break;
+      }
+    }
+  }
+}
+
+}  // namespace re2