duckdb 0.10.1 → 0.10.2-dev3.0

package/src/duckdb/third_party/re2/re2/compile.cc

@@ -14,8 +14,8 @@
 #include <utility>
 
 #include "util/logging.h"
-#include "util/pod_array.h"
 #include "util/utf.h"
+#include "re2/pod_array.h"
 #include "re2/prog.h"
 #include "re2/re2.h"
 #include "re2/regexp.h"
@@ -30,92 +30,60 @@ namespace duckdb_re2 {
 // See http://swtch.com/~rsc/regexp/regexp1.html for inspiration.
 //
 // Because the out and out1 fields in Inst are no longer pointers,
-// we can't use pointers directly here either.  Instead, p refers
-// to inst_[p>>1].out (p&1 == 0) or inst_[p>>1].out1 (p&1 == 1).
-// p == 0 represents the NULL list.  This is okay because instruction #0
+// we can't use pointers directly here either.  Instead, head refers
+// to inst_[head>>1].out (head&1 == 0) or inst_[head>>1].out1 (head&1 == 1).
+// head == 0 represents the NULL list.  This is okay because instruction #0
 // is always the fail instruction, which never appears on a list.
-
 struct PatchList {
-  uint32_t p;
-
   // Returns patch list containing just p.
-  static PatchList Mk(uint32_t p);
+  static PatchList Mk(uint32_t p) {
+    return {p, p};
+  }
 
-  // Patches all the entries on l to have value v.
+  // Patches all the entries on l to have value p.
   // Caller must not ever use patch list again.
-  static void Patch(Prog::Inst *inst0, PatchList l, uint32_t v);
-
-  // Deref returns the next pointer pointed at by p.
-  static PatchList Deref(Prog::Inst *inst0, PatchList l);
-
-  // Appends two patch lists and returns result.
-  static PatchList Append(Prog::Inst *inst0, PatchList l1, PatchList l2);
-};
-
-static PatchList nullPatchList = { 0 };
-
-// Returns patch list containing just p.
-PatchList PatchList::Mk(uint32_t p) {
-  PatchList l;
-  l.p = p;
-  return l;
-}
-
-// Returns the next pointer pointed at by l.
-PatchList PatchList::Deref(Prog::Inst* inst0, PatchList l) {
-  Prog::Inst* ip = &inst0[l.p>>1];
-  if (l.p&1)
-    l.p = ip->out1();
-  else
-    l.p = ip->out();
-  return l;
-}
-
-// Patches all the entries on l to have value v.
-void PatchList::Patch(Prog::Inst *inst0, PatchList l, uint32_t val) {
-  while (l.p != 0) {
-    Prog::Inst* ip = &inst0[l.p>>1];
-    if (l.p&1) {
-      l.p = ip->out1();
-      ip->out1_ = val;
-    } else {
-      l.p = ip->out();
-      ip->set_out(val);
+  static void Patch(Prog::Inst* inst0, PatchList l, uint32_t p) {
+    while (l.head != 0) {
+      Prog::Inst* ip = &inst0[l.head>>1];
+      if (l.head&1) {
+        l.head = ip->out1();
+        ip->out1_ = p;
+      } else {
+        l.head = ip->out();
+        ip->set_out(p);
+      }
     }
   }
-}
 
-// Appends two patch lists and returns result.
-PatchList PatchList::Append(Prog::Inst* inst0, PatchList l1, PatchList l2) {
-  if (l1.p == 0)
-    return l2;
-  if (l2.p == 0)
-    return l1;
-
-  PatchList l = l1;
-  for (;;) {
-    PatchList next = PatchList::Deref(inst0, l);
-    if (next.p == 0)
-      break;
-    l = next;
+  // Appends two patch lists and returns result.
+  static PatchList Append(Prog::Inst* inst0, PatchList l1, PatchList l2) {
+    if (l1.head == 0)
+      return l2;
+    if (l2.head == 0)
+      return l1;
+    Prog::Inst* ip = &inst0[l1.tail>>1];
+    if (l1.tail&1)
+      ip->out1_ = l2.head;
+    else
+      ip->set_out(l2.head);
+    return {l1.head, l2.tail};
   }
 
-  Prog::Inst* ip = &inst0[l.p>>1];
-  if (l.p&1)
-    ip->out1_ = l2.p;
-  else
-    ip->set_out(l2.p);
+  uint32_t head;
+  uint32_t tail;  // for constant-time append
+};
 
-  return l1;
-}
+static const PatchList kNullPatchList = {0, 0};
 
 // Compiled program fragment.
 struct Frag {
   uint32_t begin;
   PatchList end;
+  bool nullable;
 
-  Frag() : begin(0) { end.p = 0; }  // needed so Frag can go in vector
-  Frag(uint32_t begin, PatchList end) : begin(begin), end(end) {}
+  Frag() : begin(0), end(kNullPatchList), nullable(false) {}
+  Frag(uint32_t begin, PatchList end, bool nullable)
+      : begin(begin), end(end), nullable(nullable) {}
 };
 
 // Input encodings.
@@ -212,8 +180,8 @@ class Compiler : public Regexp::Walker<Frag> {
   int AddSuffixRecursive(int root, int id);
 
   // Finds the trie node for the given suffix. Returns a Frag in order to
-  // distinguish between pointing at the root node directly (end.p == 0)
-  // and pointing at an Alt's out1 or out (end.p&1 == 1 or 0, respectively).
+  // distinguish between pointing at the root node directly (end.head == 0)
+  // and pointing at an Alt's out1 or out (end.head&1 == 1 or 0, respectively).
   Frag FindByteRange(int root, int id);
 
   // Compares two ByteRanges and returns true iff they are equal.
@@ -225,8 +193,8 @@ class Compiler : public Regexp::Walker<Frag> {
   // Single rune.
   Frag Literal(Rune r, bool foldcase);
 
-  void Setup(Regexp::ParseFlags, int64_t, RE2::Anchor);
-  Prog* Finish();
+  void Setup(Regexp::ParseFlags flags, int64_t max_mem, RE2::Anchor anchor);
+  Prog* Finish(Regexp* re);
 
   // Returns .* where dot = any byte
   Frag DotStar();
@@ -298,7 +266,7 @@ int Compiler::AllocInst(int n) {
 
 // Returns an unmatchable fragment.
 Frag Compiler::NoMatch() {
-  return Frag(0, nullPatchList);
+  return Frag();
 }
 
 // Is a an unmatchable fragment?
@@ -314,7 +282,7 @@ Frag Compiler::Cat(Frag a, Frag b) {
   // Elide no-op.
   Prog::Inst* begin = &inst_[a.begin];
   if (begin->opcode() == kInstNop &&
-      a.end.p == (a.begin << 1) &&
+      a.end.head == (a.begin << 1) &&
       begin->out() == 0) {
     // in case refs to a somewhere
     PatchList::Patch(inst_.data(), a.end, b.begin);
@@ -324,11 +292,11 @@ Frag Compiler::Cat(Frag a, Frag b) {
   // To run backward over string, reverse all concatenations.
   if (reversed_) {
     PatchList::Patch(inst_.data(), b.end, a.begin);
-    return Frag(b.begin, a.end);
+    return Frag(b.begin, a.end, b.nullable && a.nullable);
   }
 
   PatchList::Patch(inst_.data(), a.end, b.begin);
-  return Frag(a.begin, b.end);
+  return Frag(a.begin, b.end, a.nullable && b.nullable);
 }
 
 // Given fragments for a and b, returns fragment for a|b.
@@ -344,7 +312,8 @@ Frag Compiler::Alt(Frag a, Frag b) {
     return NoMatch();
 
   inst_[id].InitAlt(a.begin, b.begin);
-  return Frag(id, PatchList::Append(inst_.data(), a.end, b.end));
+  return Frag(id, PatchList::Append(inst_.data(), a.end, b.end),
+              a.nullable || b.nullable);
 }
 
 // When capturing submatches in like-Perl mode, a kOpAlt Inst
@@ -354,27 +323,44 @@ Frag Compiler::Alt(Frag a, Frag b) {
 // then the operator is greedy.  If out1_ is the repetition
 // (and out_ moves forward), then the operator is non-greedy.
 
-// Given a fragment a, returns a fragment for a* or a*? (if nongreedy)
-Frag Compiler::Star(Frag a, bool nongreedy) {
+// Given a fragment for a, returns a fragment for a+ or a+? (if nongreedy)
+Frag Compiler::Plus(Frag a, bool nongreedy) {
   int id = AllocInst(1);
   if (id < 0)
     return NoMatch();
-  inst_[id].InitAlt(0, 0);
-  PatchList::Patch(inst_.data(), a.end, id);
+  PatchList pl;
   if (nongreedy) {
-    inst_[id].out1_ = a.begin;
-    return Frag(id, PatchList::Mk(id << 1));
+    inst_[id].InitAlt(0, a.begin);
+    pl = PatchList::Mk(id << 1);
   } else {
-    inst_[id].set_out(a.begin);
-    return Frag(id, PatchList::Mk((id << 1) | 1));
+    inst_[id].InitAlt(a.begin, 0);
+    pl = PatchList::Mk((id << 1) | 1);
   }
+  PatchList::Patch(inst_.data(), a.end, id);
+  return Frag(a.begin, pl, a.nullable);
 }
 
-// Given a fragment for a, returns a fragment for a+ or a+? (if nongreedy)
-Frag Compiler::Plus(Frag a, bool nongreedy) {
-  // a+ is just a* with a different entry point.
-  Frag f = Star(a, nongreedy);
-  return Frag(a.begin, f.end);
+// Given a fragment for a, returns a fragment for a* or a*? (if nongreedy)
+Frag Compiler::Star(Frag a, bool nongreedy) {
+  // When the subexpression is nullable, one Alt isn't enough to guarantee
+  // correct priority ordering within the transitive closure. The simplest
+  // solution is to handle it as (a+)? instead, which adds the second Alt.
+  if (a.nullable)
+    return Quest(Plus(a, nongreedy), nongreedy);
+
+  int id = AllocInst(1);
+  if (id < 0)
+    return NoMatch();
+  PatchList pl;
+  if (nongreedy) {
+    inst_[id].InitAlt(0, a.begin);
+    pl = PatchList::Mk(id << 1);
+  } else {
+    inst_[id].InitAlt(a.begin, 0);
+    pl = PatchList::Mk((id << 1) | 1);
+  }
+  PatchList::Patch(inst_.data(), a.end, id);
+  return Frag(id, pl, true);
 }
 
 // Given a fragment for a, returns a fragment for a? or a?? (if nongreedy)
@@ -392,7 +378,7 @@ Frag Compiler::Quest(Frag a, bool nongreedy) {
     inst_[id].InitAlt(a.begin, 0);
     pl = PatchList::Mk((id << 1) | 1);
   }
-  return Frag(id, PatchList::Append(inst_.data(), pl, a.end));
+  return Frag(id, PatchList::Append(inst_.data(), pl, a.end), true);
 }
 
 // Returns a fragment for the byte range lo-hi.
@@ -401,7 +387,7 @@ Frag Compiler::ByteRange(int lo, int hi, bool foldcase) {
   if (id < 0)
     return NoMatch();
   inst_[id].InitByteRange(lo, hi, foldcase, 0);
-  return Frag(id, PatchList::Mk(id << 1));
+  return Frag(id, PatchList::Mk(id << 1), false);
 }
 
 // Returns a no-op fragment.  Sometimes unavoidable.
@@ -410,7 +396,7 @@ Frag Compiler::Nop() {
   if (id < 0)
     return NoMatch();
   inst_[id].InitNop(0);
-  return Frag(id, PatchList::Mk(id << 1));
+  return Frag(id, PatchList::Mk(id << 1), true);
 }
 
 // Returns a fragment that signals a match.
@@ -419,7 +405,7 @@ Frag Compiler::Match(int32_t match_id) {
   if (id < 0)
     return NoMatch();
   inst_[id].InitMatch(match_id);
-  return Frag(id, nullPatchList);
+  return Frag(id, kNullPatchList, false);
 }
 
 // Returns a fragment matching a particular empty-width op (like ^ or $)
@@ -428,7 +414,7 @@ Frag Compiler::EmptyWidth(EmptyOp empty) {
   if (id < 0)
     return NoMatch();
   inst_[id].InitEmptyWidth(empty, 0);
-  return Frag(id, PatchList::Mk(id << 1));
+  return Frag(id, PatchList::Mk(id << 1), true);
 }
 
 // Given a fragment a, returns a fragment with capturing parens around a.
@@ -442,7 +428,7 @@ Frag Compiler::Capture(Frag a, int n) {
   inst_[id+1].InitCapture(2*n+1, 0);
   PatchList::Patch(inst_.data(), a.end, id+1);
 
-  return Frag(id, PatchList::Mk((id+1) << 1));
+  return Frag(id, PatchList::Mk((id+1) << 1), a.nullable);
 }
 
 // A Rune is a name for a Unicode code point.
@@ -467,7 +453,7 @@ static int MaxRune(int len) {
 void Compiler::BeginRange() {
   rune_cache_.clear();
   rune_range_.begin = 0;
-  rune_range_.end = nullPatchList;
+  rune_range_.end = kNullPatchList;
 }
 
 int Compiler::UncachedRuneByteSuffix(uint8_t lo, uint8_t hi, bool foldcase,
@@ -501,8 +487,8 @@ int Compiler::CachedRuneByteSuffix(uint8_t lo, uint8_t hi, bool foldcase,
 }
 
 bool Compiler::IsCachedRuneByteSuffix(int id) {
-  uint8_t lo = inst_[id].inst_byte_range_data_.lo_;
-  uint8_t hi = inst_[id].inst_byte_range_data_.hi_;
+  uint8_t lo = inst_[id].lo_;
+  uint8_t hi = inst_[id].hi_;
   bool foldcase = inst_[id].foldcase() != 0;
   int next = inst_[id].out();
 
@@ -548,9 +534,9 @@ int Compiler::AddSuffixRecursive(int root, int id) {
   }
 
   int br;
-  if (f.end.p == 0)
+  if (f.end.head == 0)
     br = root;
-  else if (f.end.p&1)
+  else if (f.end.head&1)
     br = inst_[f.begin].out1();
   else
     br = inst_[f.begin].out();
@@ -566,9 +552,9 @@ int Compiler::AddSuffixRecursive(int root, int id) {
     // Ensure that the parent points to the clone, not to the original.
     // Note that this could leave the head unreachable except via the cache.
     br = byterange;
-    if (f.end.p == 0)
+    if (f.end.head == 0)
       root = br;
-    else if (f.end.p&1)
+    else if (f.end.head&1)
       inst_[f.begin].out1_ = br;
     else
       inst_[f.begin].set_out(br);
@@ -601,7 +587,7 @@ bool Compiler::ByteRangeEqual(int id1, int id2) {
 Frag Compiler::FindByteRange(int root, int id) {
   if (inst_[root].opcode() == kInstByteRange) {
     if (ByteRangeEqual(root, id))
-      return Frag(root, nullPatchList);
+      return Frag(root, kNullPatchList, false);
     else
       return NoMatch();
   }
@@ -609,7 +595,7 @@ Frag Compiler::FindByteRange(int root, int id) {
   while (inst_[root].opcode() == kInstAlt) {
     int out1 = inst_[root].out1();
     if (ByteRangeEqual(out1, id))
-      return Frag(root, PatchList::Mk((root << 1) | 1));
+      return Frag(root, PatchList::Mk((root << 1) | 1), false);
 
     // CharClass is a sorted list of ranges, so if out1 of the root Alt wasn't
     // what we're looking for, then we can stop immediately. Unfortunately, we
@@ -621,7 +607,7 @@ Frag Compiler::FindByteRange(int root, int id) {
     if (inst_[out].opcode() == kInstAlt)
       root = out;
     else if (ByteRangeEqual(out, id))
-      return Frag(root, PatchList::Mk(root << 1));
+      return Frag(root, PatchList::Mk(root << 1), false);
     else
       return NoMatch();
   }
@@ -662,48 +648,43 @@ void Compiler::AddRuneRangeLatin1(Rune lo, Rune hi, bool foldcase) {
                                    static_cast<uint8_t>(hi), foldcase, 0));
 }
 
-// Table describing how to make a UTF-8 matching machine
-// for the rune range 80-10FFFF (Runeself-Runemax).
-// This range happens frequently enough (for example /./ and /[^a-z]/)
-// and the rune_cache_ map is slow enough that this is worth
-// special handling.  Makes compilation of a small expression
-// with a dot in it about 10% faster.
-// The * in the comments below mark whole sequences.
-static struct ByteRangeProg {
-  int next;
-  int lo;
-  int hi;
-} prog_80_10ffff[] = {
-  // Two-byte
-  { -1, 0x80, 0xBF, },  // 0:  80-BF
-  {  0, 0xC2, 0xDF, },  // 1:  C2-DF 80-BF*
-
-  // Three-byte
-  {  0, 0xA0, 0xBF, },  // 2:  A0-BF 80-BF
-  {  2, 0xE0, 0xE0, },  // 3:  E0 A0-BF 80-BF*
-  {  0, 0x80, 0xBF, },  // 4:  80-BF 80-BF
-  {  4, 0xE1, 0xEF, },  // 5:  E1-EF 80-BF 80-BF*
-
-  // Four-byte
-  {  4, 0x90, 0xBF, },  // 6:  90-BF 80-BF 80-BF
-  {  6, 0xF0, 0xF0, },  // 7:  F0 90-BF 80-BF 80-BF*
-  {  4, 0x80, 0xBF, },  // 8:  80-BF 80-BF 80-BF
-  {  8, 0xF1, 0xF3, },  // 9: F1-F3 80-BF 80-BF 80-BF*
-  {  4, 0x80, 0x8F, },  // 10: 80-8F 80-BF 80-BF
-  { 10, 0xF4, 0xF4, },  // 11: F4 80-8F 80-BF 80-BF*
-};
-
 void Compiler::Add_80_10ffff() {
-  int inst[arraysize(prog_80_10ffff)] = { 0 }; // does not need to be initialized; silences gcc warning
-  for (int i = 0; i < arraysize(prog_80_10ffff); i++) {
-    const ByteRangeProg& p = prog_80_10ffff[i];
-    int next = 0;
-    if (p.next >= 0)
-      next = inst[p.next];
-    inst[i] = UncachedRuneByteSuffix(static_cast<uint8_t>(p.lo),
-                                     static_cast<uint8_t>(p.hi), false, next);
-    if ((p.lo & 0xC0) != 0x80)
-      AddSuffix(inst[i]);
+  // The 80-10FFFF (Runeself-Runemax) rune range occurs frequently enough
+  // (for example, for /./ and /[^a-z]/) that it is worth simplifying: by
+  // permitting overlong encodings in E0 and F0 sequences and code points
+  // over 10FFFF in F4 sequences, the size of the bytecode and the number
+  // of equivalence classes are reduced significantly.
+  int id;
+  if (reversed_) {
+    // Prefix factoring matters, but we don't have to handle it here
+    // because the rune range trie logic takes care of that already.
+    id = UncachedRuneByteSuffix(0xC2, 0xDF, false, 0);
+    id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);
+    AddSuffix(id);
+
+    id = UncachedRuneByteSuffix(0xE0, 0xEF, false, 0);
+    id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);
+    id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);
+    AddSuffix(id);
+
+    id = UncachedRuneByteSuffix(0xF0, 0xF4, false, 0);
+    id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);
+    id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);
+    id = UncachedRuneByteSuffix(0x80, 0xBF, false, id);
+    AddSuffix(id);
+  } else {
+    // Suffix factoring matters - and we do have to handle it here.
+    int cont1 = UncachedRuneByteSuffix(0x80, 0xBF, false, 0);
+    id = UncachedRuneByteSuffix(0xC2, 0xDF, false, cont1);
+    AddSuffix(id);
+
+    int cont2 = UncachedRuneByteSuffix(0x80, 0xBF, false, cont1);
+    id = UncachedRuneByteSuffix(0xE0, 0xEF, false, cont2);
+    AddSuffix(id);
+
+    int cont3 = UncachedRuneByteSuffix(0x80, 0xBF, false, cont2);
+    id = UncachedRuneByteSuffix(0xF0, 0xF4, false, cont3);
+    AddSuffix(id);
   }
 }
 
@@ -711,9 +692,8 @@ void Compiler::AddRuneRangeUTF8(Rune lo, Rune hi, bool foldcase) {
   if (lo > hi)
     return;
 
-  // Pick off 80-10FFFF as a common special case
-  // that can bypass the slow rune_cache_.
-  if (lo == 0x80 && hi == 0x10ffff && !reversed_) {
+  // Pick off 80-10FFFF as a common special case.
+  if (lo == 0x80 && hi == 0x10ffff) {
     Add_80_10ffff();
     return;
   }
@@ -809,8 +789,8 @@ void Compiler::AddRuneRangeUTF8(Rune lo, Rune hi, bool foldcase) {
 // Should not be called.
 Frag Compiler::Copy(Frag arg) {
   // We're using WalkExponential; there should be no copying.
-  LOG(DFATAL) << "Compiler::Copy called!";
   failed_ = true;
+  LOG(DFATAL) << "Compiler::Copy called!";
   return NoMatch();
 }
 
@@ -936,8 +916,8 @@ Frag Compiler::PostVisit(Regexp* re, Frag, Frag, Frag* child_frags,
       CharClass* cc = re->cc();
       if (cc->empty()) {
         // This can't happen.
-        LOG(DFATAL) << "No ranges in char class";
         failed_ = true;
+        LOG(DFATAL) << "No ranges in char class";
         return NoMatch();
       }
 
@@ -994,8 +974,8 @@ Frag Compiler::PostVisit(Regexp* re, Frag, Frag, Frag* child_frags,
     case kRegexpNoWordBoundary:
       return EmptyWidth(kEmptyNonWordBoundary);
   }
-  LOG(DFATAL) << "Missing case in Compiler: " << re->op();
   failed_ = true;
+  LOG(DFATAL) << "Missing case in Compiler: " << re->op();
   return NoMatch();
 }
 
@@ -1095,8 +1075,6 @@ static bool IsAnchorEnd(Regexp** pre, int depth) {
 
 void Compiler::Setup(Regexp::ParseFlags flags, int64_t max_mem,
                      RE2::Anchor anchor) {
-  prog_->set_flags(flags);
-
   if (flags & Regexp::Latin1)
     encoding_ = kEncodingLatin1;
   max_mem_ = max_mem;
@@ -1117,14 +1095,11 @@ void Compiler::Setup(Regexp::ParseFlags flags, int64_t max_mem,
     // on the program.)
     if (m >= 1<<24)
       m = 1<<24;
-
     // Inst imposes its own limit (currently bigger than 2^24 but be safe).
     if (m > Prog::Inst::kMaxInst)
       m = Prog::Inst::kMaxInst;
-
     max_ninst_ = static_cast<int>(m);
   }
-
   anchor_ = anchor;
 }
 
@@ -1178,10 +1153,10 @@ Prog* Compiler::Compile(Regexp* re, bool reversed, int64_t max_mem) {
   c.prog_->set_start_unanchored(all.begin);
 
   // Hand ownership of prog_ to caller.
-  return c.Finish();
+  return c.Finish(re);
 }
 
-Prog* Compiler::Finish() {
+Prog* Compiler::Finish(Regexp* re) {
   if (failed_)
     return NULL;
 
@@ -1198,6 +1173,13 @@ Prog* Compiler::Finish() {
   prog_->Flatten();
   prog_->ComputeByteMap();
 
+  if (!prog_->reversed()) {
+    std::string prefix;
+    bool prefix_foldcase;
+    if (re->RequiredPrefixForAccel(&prefix, &prefix_foldcase))
+      prog_->ConfigurePrefixAccel(prefix, prefix_foldcase);
+  }
+
   // Record remaining memory for DFA.
   if (max_mem_ <= 0) {
     prog_->set_dfa_mem(1<<20);
@@ -1254,7 +1236,7 @@ Prog* Compiler::CompileSet(Regexp* re, RE2::Anchor anchor, int64_t max_mem) {
   c.prog_->set_start(all.begin);
   c.prog_->set_start_unanchored(all.begin);
 
-  Prog* prog = c.Finish();
+  Prog* prog = c.Finish(re);
   if (prog == NULL)
     return NULL;
 
@@ -1276,4 +1258,4 @@ Prog* Prog::CompileSet(Regexp* re, RE2::Anchor anchor, int64_t max_mem) {
   return Compiler::CompileSet(re, anchor, max_mem);
 }
 
-}  // namespace duckdb_re2
+}  // namespace re2