duckdb 0.10.2-dev0.0 → 0.10.2-dev5.0

package/src/duckdb/third_party/re2/re2/prog.cc

@@ -7,6 +7,12 @@
 
 #include "re2/prog.h"
 
+#if defined(__AVX2__)
+#include <immintrin.h>
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+#endif
 #include <stdint.h>
 #include <string.h>
 #include <algorithm>
@@ -32,9 +38,9 @@ void Prog::Inst::InitAlt(uint32_t out, uint32_t out1) {
 void Prog::Inst::InitByteRange(int lo, int hi, int foldcase, uint32_t out) {
   DCHECK_EQ(out_opcode_, 0);
   set_out_opcode(out, kInstByteRange);
-    inst_byte_range_data_.lo_ = lo & 0xFF;
-    inst_byte_range_data_.hi_ = hi & 0xFF;
-    inst_byte_range_data_.hint_foldcase_ = foldcase&1;
+  lo_ = lo & 0xFF;
+  hi_ = hi & 0xFF;
+  hint_foldcase_ = foldcase&1;
 }
 
 void Prog::Inst::InitCapture(int cap, uint32_t out) {
@@ -79,7 +85,7 @@ std::string Prog::Inst::Dump() {
     case kInstByteRange:
       return StringPrintf("byte%s [%02x-%02x] %d -> %d",
                           foldcase() ? "/i" : "",
-                          inst_byte_range_data_.lo_, inst_byte_range_data_.hi_, hint(), out());
+                          lo_, hi_, hint(), out());
 
     case kInstCapture:
       return StringPrintf("capture %d -> %d", cap_, out());
@@ -109,9 +115,10 @@ Prog::Prog()
     start_unanchored_(0),
     size_(0),
     bytemap_range_(0),
-    first_byte_(-1),
-    flags_(0),
+    prefix_foldcase_(false),
+    prefix_size_(0),
     list_count_(0),
+    bit_state_text_max_size_(0),
     dfa_mem_(0),
     dfa_first_(NULL),
     dfa_longest_(NULL) {
@@ -120,6 +127,8 @@ Prog::Prog()
 Prog::~Prog() {
   DeleteDFA(dfa_longest_);
   DeleteDFA(dfa_first_);
+  if (prefix_foldcase_)
+    delete[] prefix_dfa_;
 }
 
 typedef SparseSet Workq;
@@ -134,7 +143,7 @@ static std::string ProgToString(Prog* prog, Workq* q) {
   for (Workq::iterator i = q->begin(); i != q->end(); ++i) {
     int id = *i;
     Prog::Inst* ip = prog->inst(id);
-    StringAppendF(&s, "%d. %s\n", id, ip->Dump().c_str());
+    s += StringPrintf("%d. %s\n", id, ip->Dump().c_str());
     AddToQueue(q, ip->out());
     if (ip->opcode() == kInstAlt || ip->opcode() == kInstAltMatch)
       AddToQueue(q, ip->out1());
@@ -147,9 +156,9 @@ static std::string FlattenedProgToString(Prog* prog, int start) {
   for (int id = start; id < prog->size(); id++) {
     Prog::Inst* ip = prog->inst(id);
     if (ip->last())
-      StringAppendF(&s, "%d. %s\n", id, ip->Dump().c_str());
+      s += StringPrintf("%d. %s\n", id, ip->Dump().c_str());
     else
-      StringAppendF(&s, "%d+ %s\n", id, ip->Dump().c_str());
+      s += StringPrintf("%d+ %s\n", id, ip->Dump().c_str());
   }
   return s;
 }
@@ -180,19 +189,36 @@ std::string Prog::DumpByteMap() {
     while (c < 256-1 && bytemap_[c+1] == b)
       c++;
     int hi = c;
-    StringAppendF(&map, "[%02x-%02x] -> %d\n", lo, hi, b);
+    map += StringPrintf("[%02x-%02x] -> %d\n", lo, hi, b);
   }
   return map;
 }
 
-int Prog::first_byte() {
-  std::call_once(first_byte_once_, [](Prog* prog) {
-    prog->first_byte_ = prog->ComputeFirstByte();
-  }, this);
-  return first_byte_;
-}
+// Is ip a guaranteed match at end of text, perhaps after some capturing?
+static bool IsMatch(Prog* prog, Prog::Inst* ip) {
+  for (;;) {
+    switch (ip->opcode()) {
+      default:
+        LOG(DFATAL) << "Unexpected opcode in IsMatch: " << ip->opcode();
+        return false;
+
+      case kInstAlt:
+      case kInstAltMatch:
+      case kInstByteRange:
+      case kInstFail:
+      case kInstEmptyWidth:
+        return false;
 
-static bool IsMatch(Prog*, Prog::Inst*);
+      case kInstCapture:
+      case kInstNop:
+        ip = prog->inst(ip->out());
+        break;
+
+      case kInstMatch:
+        return true;
+    }
+  }
+}
 
 // Peep-hole optimizer.
 void Prog::Optimize() {
@@ -258,54 +284,28 @@ void Prog::Optimize() {
   }
 }
 
-// Is ip a guaranteed match at end of text, perhaps after some capturing?
-static bool IsMatch(Prog* prog, Prog::Inst* ip) {
-  for (;;) {
-    switch (ip->opcode()) {
-      default:
-        LOG(DFATAL) << "Unexpected opcode in IsMatch: " << ip->opcode();
-        return false;
-
-      case kInstAlt:
-      case kInstAltMatch:
-      case kInstByteRange:
-      case kInstFail:
-      case kInstEmptyWidth:
-        return false;
-
-      case kInstCapture:
-      case kInstNop:
-        ip = prog->inst(ip->out());
-        break;
-
-      case kInstMatch:
-        return true;
-    }
-  }
-}
-
 uint32_t Prog::EmptyFlags(const StringPiece& text, const char* p) {
   int flags = 0;
 
   // ^ and \A
-  if (p == text.begin())
+  if (p == text.data())
     flags |= kEmptyBeginText | kEmptyBeginLine;
   else if (p[-1] == '\n')
     flags |= kEmptyBeginLine;
 
   // $ and \z
-  if (p == text.end())
+  if (p == text.data() + text.size())
     flags |= kEmptyEndText | kEmptyEndLine;
-  else if (p < text.end() && p[0] == '\n')
+  else if (p < text.data() + text.size() && p[0] == '\n')
     flags |= kEmptyEndLine;
 
   // \b and \B
-  if (p == text.begin() && p == text.end()) {
+  if (p == text.data() && p == text.data() + text.size()) {
     // no word boundary here
-  } else if (p == text.begin()) {
+  } else if (p == text.data()) {
     if (IsWordChar(p[0]))
       flags |= kEmptyWordBoundary;
-  } else if (p == text.end()) {
+  } else if (p == text.data() + text.size()) {
     if (IsWordChar(p[-1]))
       flags |= kEmptyWordBoundary;
   } else {
@@ -510,13 +510,13 @@ void Prog::ComputeByteMap() {
   }
 
   builder.Build(bytemap_, &bytemap_range_);
-}
 
-int TotalInstructions(int inst_count_[]) {
-  int total = 0;
-  for (int i = 0; i < kNumInst; i++)
-    total += inst_count_[i];
-  return total;
+  if ((0)) {  // For debugging, use trivial bytemap.
+    LOG(ERROR) << "Using trivial bytemap.";
+    for (int i = 0; i < 256; i++)
+      bytemap_[i] = static_cast<uint8_t>(i);
+    bytemap_range_ = 256;
+  }
 }
 
 // Prog::Flatten() implements a graph rewriting algorithm.
@@ -611,7 +611,13 @@ void Prog::Flatten() {
     inst_count_[ip->opcode()]++;
   }
 
-  DCHECK_EQ(TotalInstructions(inst_count_), static_cast<int>(flat.size()));
+#if !defined(NDEBUG)
+  // Address a `-Wunused-but-set-variable' warning from Clang 13.x.
+  size_t total = 0;
+  for (int i = 0; i < kNumInst; i++)
+    total += inst_count_[i];
+  CHECK_EQ(total, flat.size());
+#endif
 
   // Remap start_unanchored and start.
   if (start_unanchored() == 0) {
@@ -638,6 +644,11 @@ void Prog::Flatten() {
     for (int i = 0; i < list_count_; ++i)
       list_heads_[flatmap[i]] = i;
   }
+
+  // BitState allocates a bitmap of size list_count_ * (text.size()+1)
+  // for tracking pairs of possibilities that it has already explored.
+  const size_t kBitStateBitmapMaxSize = 256*1024;  // max size in bits
+  bit_state_text_max_size_ = kBitStateBitmapMaxSize / list_count_ - 1;
 }
 
 void Prog::MarkSuccessors(SparseArray<int>* rootmap,
@@ -910,9 +921,255 @@ void Prog::ComputeHints(std::vector<Inst>* flat, int begin, int end) {
 
     if (first != end) {
       uint16_t hint = static_cast<uint16_t>(std::min(first - id, 32767));
-      ip->inst_byte_range_data_.hint_foldcase_ |= hint<<1;
+      ip->hint_foldcase_ |= hint<<1;
     }
   }
 }
 
-}  // namespace duckdb_re2
+// The final state will always be this, which frees up a register for the hot
+// loop and thus avoids the spilling that can occur when building with Clang.
+static const size_t kShiftDFAFinal = 9;
+
+// This function takes the prefix as std::string (i.e. not const std::string&
+// as normal) because it's going to clobber it, so a temporary is convenient.
+static uint64_t* BuildShiftDFA(std::string prefix) {
+  // This constant is for convenience now and also for correctness later when
+  // we clobber the prefix, but still need to know how long it was initially.
+  const size_t size = prefix.size();
+
+  // Construct the NFA.
+  // The table is indexed by input byte; each element is a bitfield of states
+  // reachable by the input byte. Given a bitfield of the current states, the
+  // bitfield of states reachable from those is - for this specific purpose -
+  // always ((ncurr << 1) | 1). Intersecting the reachability bitfields gives
+  // the bitfield of the next states reached by stepping over the input byte.
+  // Credits for this technique: the Hyperscan paper by Geoff Langdale et al.
+  uint16_t nfa[256]{};
+  for (size_t i = 0; i < size; ++i) {
+    uint8_t b = prefix[i];
+    nfa[b] |= 1 << (i+1);
+  }
+  // This is the `\C*?` for unanchored search.
+  for (int b = 0; b < 256; ++b)
+    nfa[b] |= 1;
+
+  // This maps from DFA state to NFA states; the reverse mapping is used when
+  // recording transitions and gets implemented with plain old linear search.
+  // The "Shift DFA" technique limits this to ten states when using uint64_t;
+  // to allow for the initial state, we use at most nine bytes of the prefix.
+  // That same limit is also why uint16_t is sufficient for the NFA bitfield.
+  uint16_t states[kShiftDFAFinal+1]{};
+  states[0] = 1;
+  for (size_t dcurr = 0; dcurr < size; ++dcurr) {
+    uint8_t b = prefix[dcurr];
+    uint16_t ncurr = states[dcurr];
+    uint16_t nnext = nfa[b] & ((ncurr << 1) | 1);
+    size_t dnext = dcurr+1;
+    if (dnext == size)
+      dnext = kShiftDFAFinal;
+    states[dnext] = nnext;
+  }
+
+  // Sort and unique the bytes of the prefix to avoid repeating work while we
+  // record transitions. This clobbers the prefix, but it's no longer needed.
+  std::sort(prefix.begin(), prefix.end());
+  prefix.erase(std::unique(prefix.begin(), prefix.end()), prefix.end());
+
+  // Construct the DFA.
+  // The table is indexed by input byte; each element is effectively a packed
+  // array of uint6_t; each array value will be multiplied by six in order to
+  // avoid having to do so later in the hot loop as well as masking/shifting.
+  // Credits for this technique: "Shift-based DFAs" on GitHub by Per Vognsen.
+  uint64_t* dfa = new uint64_t[256]{};
+  // Record a transition from each state for each of the bytes of the prefix.
+  // Note that all other input bytes go back to the initial state by default.
+  for (size_t dcurr = 0; dcurr < size; ++dcurr) {
+    for (uint8_t b : prefix) {
+      uint16_t ncurr = states[dcurr];
+      uint16_t nnext = nfa[b] & ((ncurr << 1) | 1);
+      size_t dnext = 0;
+      while (states[dnext] != nnext)
+        ++dnext;
+      dfa[b] |= static_cast<uint64_t>(dnext * 6) << (dcurr * 6);
+      // Convert ASCII letters to uppercase and record the extra transitions.
+      // Note that ASCII letters are guaranteed to be lowercase at this point
+      // because that's how the parser normalises them. #FunFact: 'k' and 's'
+      // match U+212A and U+017F, respectively, so they won't occur here when
+      // using UTF-8 encoding because the parser will emit character classes.
+      if ('a' <= b && b <= 'z') {
+        b -= 'a' - 'A';
+        dfa[b] |= static_cast<uint64_t>(dnext * 6) << (dcurr * 6);
+      }
+    }
+  }
+  // This lets the final state "saturate", which will matter for performance:
+  // in the hot loop, we check for a match only at the end of each iteration,
+  // so we must keep signalling the match until we get around to checking it.
+  for (int b = 0; b < 256; ++b)
+    dfa[b] |= static_cast<uint64_t>(kShiftDFAFinal * 6) << (kShiftDFAFinal * 6);
+
+  return dfa;
+}
+
+void Prog::ConfigurePrefixAccel(const std::string& prefix,
+                                bool prefix_foldcase) {
+  prefix_foldcase_ = prefix_foldcase;
+  prefix_size_ = prefix.size();
+  if (prefix_foldcase_) {
+    // Use PrefixAccel_ShiftDFA().
+    // ... and no more than nine bytes of the prefix. (See above for details.)
+    prefix_size_ = std::min(prefix_size_, kShiftDFAFinal);
+    prefix_dfa_ = BuildShiftDFA(prefix.substr(0, prefix_size_));
+  } else if (prefix_size_ != 1) {
+    // Use PrefixAccel_FrontAndBack().
+    prefix_front_ = prefix.front();
+    prefix_back_ = prefix.back();
+  } else {
+    // Use memchr(3).
+    prefix_front_ = prefix.front();
+  }
+}
+
+const void* Prog::PrefixAccel_ShiftDFA(const void* data, size_t size) {
+  if (size < prefix_size_)
+    return NULL;
+
+  uint64_t curr = 0;
+
+  // At the time of writing, rough benchmarks on a Broadwell machine showed
+  // that this unroll factor (i.e. eight) achieves a speedup factor of two.
+  if (size >= 8) {
+    const uint8_t* p = reinterpret_cast<const uint8_t*>(data);
+    const uint8_t* endp = p + (size&~7);
+    do {
+      uint8_t b0 = p[0];
+      uint8_t b1 = p[1];
+      uint8_t b2 = p[2];
+      uint8_t b3 = p[3];
+      uint8_t b4 = p[4];
+      uint8_t b5 = p[5];
+      uint8_t b6 = p[6];
+      uint8_t b7 = p[7];
+
+      uint64_t next0 = prefix_dfa_[b0];
+      uint64_t next1 = prefix_dfa_[b1];
+      uint64_t next2 = prefix_dfa_[b2];
+      uint64_t next3 = prefix_dfa_[b3];
+      uint64_t next4 = prefix_dfa_[b4];
+      uint64_t next5 = prefix_dfa_[b5];
+      uint64_t next6 = prefix_dfa_[b6];
+      uint64_t next7 = prefix_dfa_[b7];
+
+      uint64_t curr0 = next0 >> (curr  & 63);
+      uint64_t curr1 = next1 >> (curr0 & 63);
+      uint64_t curr2 = next2 >> (curr1 & 63);
+      uint64_t curr3 = next3 >> (curr2 & 63);
+      uint64_t curr4 = next4 >> (curr3 & 63);
+      uint64_t curr5 = next5 >> (curr4 & 63);
+      uint64_t curr6 = next6 >> (curr5 & 63);
+      uint64_t curr7 = next7 >> (curr6 & 63);
+
+      if ((curr7 & 63) == kShiftDFAFinal * 6) {
+        // At the time of writing, using the same masking subexpressions from
+        // the preceding lines caused Clang to clutter the hot loop computing
+        // them - even though they aren't actually needed for shifting! Hence
+        // these rewritten conditions, which achieve a speedup factor of two.
+        if (((curr7-curr0) & 63) == 0) return p+1-prefix_size_;
+        if (((curr7-curr1) & 63) == 0) return p+2-prefix_size_;
+        if (((curr7-curr2) & 63) == 0) return p+3-prefix_size_;
+        if (((curr7-curr3) & 63) == 0) return p+4-prefix_size_;
+        if (((curr7-curr4) & 63) == 0) return p+5-prefix_size_;
+        if (((curr7-curr5) & 63) == 0) return p+6-prefix_size_;
+        if (((curr7-curr6) & 63) == 0) return p+7-prefix_size_;
+        if (((curr7-curr7) & 63) == 0) return p+8-prefix_size_;
+      }
+
+      curr = curr7;
+      p += 8;
+    } while (p != endp);
+    data = p;
+    size = size&7;
+  }
+
+  const uint8_t* p = reinterpret_cast<const uint8_t*>(data);
+  const uint8_t* endp = p + size;
+  while (p != endp) {
+    uint8_t b = *p++;
+    uint64_t next = prefix_dfa_[b];
+    curr = next >> (curr & 63);
+    if ((curr & 63) == kShiftDFAFinal * 6)
+      return p-prefix_size_;
+  }
+  return NULL;
+}
+
+#if defined(__AVX2__)
+// Finds the least significant non-zero bit in n.
+static int FindLSBSet(uint32_t n) {
+  DCHECK_NE(n, 0);
+#if defined(__GNUC__)
+  return __builtin_ctz(n);
+#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
+  unsigned long c;
+  _BitScanForward(&c, n);
+  return static_cast<int>(c);
+#else
+  int c = 31;
+  for (int shift = 1 << 4; shift != 0; shift >>= 1) {
+    uint32_t word = n << shift;
+    if (word != 0) {
+      n = word;
+      c -= shift;
+    }
+  }
+  return c;
+#endif
+}
+#endif
+
+const void* Prog::PrefixAccel_FrontAndBack(const void* data, size_t size) {
+  DCHECK_GE(prefix_size_, 2);
+  if (size < prefix_size_)
+    return NULL;
+  // Don't bother searching the last prefix_size_-1 bytes for prefix_front_.
+  // This also means that probing for prefix_back_ doesn't go out of bounds.
+  size -= prefix_size_-1;
+
+#if defined(__AVX2__)
+  // Use AVX2 to look for prefix_front_ and prefix_back_ 32 bytes at a time.
+  if (size >= sizeof(__m256i)) {
+    const __m256i* fp = reinterpret_cast<const __m256i*>(
+        reinterpret_cast<const char*>(data));
+    const __m256i* bp = reinterpret_cast<const __m256i*>(
+        reinterpret_cast<const char*>(data) + prefix_size_-1);
+    const __m256i* endfp = fp + size/sizeof(__m256i);
+    const __m256i f_set1 = _mm256_set1_epi8(prefix_front_);
+    const __m256i b_set1 = _mm256_set1_epi8(prefix_back_);
+    do {
+      const __m256i f_loadu = _mm256_loadu_si256(fp++);
+      const __m256i b_loadu = _mm256_loadu_si256(bp++);
+      const __m256i f_cmpeq = _mm256_cmpeq_epi8(f_set1, f_loadu);
+      const __m256i b_cmpeq = _mm256_cmpeq_epi8(b_set1, b_loadu);
+      const int fb_testz = _mm256_testz_si256(f_cmpeq, b_cmpeq);
+      if (fb_testz == 0) {  // ZF: 1 means zero, 0 means non-zero.
+        const __m256i fb_and = _mm256_and_si256(f_cmpeq, b_cmpeq);
+        const int fb_movemask = _mm256_movemask_epi8(fb_and);
+        const int fb_ctz = FindLSBSet(fb_movemask);
+        return reinterpret_cast<const char*>(fp-1) + fb_ctz;
+      }
+    } while (fp != endfp);
+    data = fp;
+    size = size%sizeof(__m256i);
+  }
+#endif
+
+  const char* p0 = reinterpret_cast<const char*>(data);
+  for (const char* p = p0;; p++) {
+    DCHECK_GE(size, static_cast<size_t>(p-p0));
+    p = reinterpret_cast<const char*>(memchr(p, prefix_front_, size - (p-p0)));
+    if (p == NULL || p[prefix_size_-1] == prefix_back_)
+      return p;
+  }
+}
+
+}  // namespace re2