chipper 0.4.2

data/ext/re2/simplify.cc

@@ -0,0 +1,393 @@
+// Copyright 2006 The RE2 Authors.  All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Rewrite POSIX and other features in re
+// to use simple extended regular expression features.
+// Also sort and simplify character classes.
+
+#include "util/util.h"
+#include "re2/regexp.h"
+#include "re2/walker-inl.h"
+
+namespace re2 {
+
+// Parses the regexp src and then simplifies it and sets *dst to the
+// string representation of the simplified form.  Returns true on success.
+// Returns false and sets *error (if error != NULL) on error.
+bool Regexp::SimplifyRegexp(const StringPiece& src, ParseFlags flags,
+                            string* dst,
+                            RegexpStatus* status) {
+  Regexp* re = Parse(src, flags, status);
+  if (re == NULL)
+    return false;
+  Regexp* sre = re->Simplify();
+  re->Decref();
+  if (sre == NULL) {
+    // Should not happen, since Simplify never fails.
+    LOG(ERROR) << "Simplify failed on " << src;
+    if (status) {
+      status->set_code(kRegexpInternalError);
+      status->set_error_arg(src);
+    }
+    return false;
+  }
+  *dst = sre->ToString();
+  sre->Decref();
+  return true;
+}
+
+// Assuming the simple_ flags on the children are accurate,
+// is this Regexp* simple?
+bool Regexp::ComputeSimple() {
+  Regexp** subs;
+  switch (op_) {
+    case kRegexpNoMatch:
+    case kRegexpEmptyMatch:
+    case kRegexpLiteral:
+    case kRegexpLiteralString:
+    case kRegexpBeginLine:
+    case kRegexpEndLine:
+    case kRegexpBeginText:
+    case kRegexpWordBoundary:
+    case kRegexpNoWordBoundary:
+    case kRegexpEndText:
+    case kRegexpAnyChar:
+    case kRegexpAnyByte:
+    case kRegexpHaveMatch:
+      return true;
+    case kRegexpConcat:
+    case kRegexpAlternate:
+      // These are simple as long as the subpieces are simple.
+      subs = sub();
+      for (int i = 0; i < nsub_; i++)
+        if (!subs[i]->simple_)
+          return false;
+      return true;
+    case kRegexpCharClass:
+      // Simple as long as the char class is not empty, not full.
+      if (ccb_ != NULL)
+        return !ccb_->empty() && !ccb_->full();
+      return !cc_->empty() && !cc_->full();
+    case kRegexpCapture:
+      subs = sub();
+      return subs[0]->simple_;
+    case kRegexpStar:
+    case kRegexpPlus:
+    case kRegexpQuest:
+      subs = sub();
+      if (!subs[0]->simple_)
+        return false;
+      switch (subs[0]->op_) {
+        case kRegexpStar:
+        case kRegexpPlus:
+        case kRegexpQuest:
+        case kRegexpEmptyMatch:
+        case kRegexpNoMatch:
+          return false;
+        default:
+          break;
+      }
+      return true;
+    case kRegexpRepeat:
+      return false;
+  }
+  LOG(DFATAL) << "Case not handled in ComputeSimple: " << op_;
+  return false;
+}
+
+// Walker subclass used by Simplify.
+// The simplify walk is purely post-recursive: given the simplified children,
+// PostVisit creates the simplified result.
+// The child_args are simplified Regexp*s.
+class SimplifyWalker : public Regexp::Walker<Regexp*> {
+ public:
+  SimplifyWalker() {}
+  virtual Regexp* PreVisit(Regexp* re, Regexp* parent_arg, bool* stop);
+  virtual Regexp* PostVisit(Regexp* re,
+                            Regexp* parent_arg,
+                            Regexp* pre_arg,
+                            Regexp** child_args, int nchild_args);
+  virtual Regexp* Copy(Regexp* re);
+  virtual Regexp* ShortVisit(Regexp* re, Regexp* parent_arg);
+
+ private:
+  // These functions are declared inside SimplifyWalker so that
+  // they can edit the private fields of the Regexps they construct.
+
+  // Creates a concatenation of two Regexp, consuming refs to re1 and re2.
+  // Caller must Decref return value when done with it.
+  static Regexp* Concat2(Regexp* re1, Regexp* re2, Regexp::ParseFlags flags);
+
+  // Simplifies the expression re{min,max} in terms of *, +, and ?.
+  // Returns a new regexp.  Does not edit re.  Does not consume reference to re.
+  // Caller must Decref return value when done with it.
+  static Regexp* SimplifyRepeat(Regexp* re, int min, int max,
+                                Regexp::ParseFlags parse_flags);
+
+  // Simplifies a character class by expanding any named classes
+  // into rune ranges.  Does not edit re.  Does not consume ref to re.
+  // Caller must Decref return value when done with it.
+  static Regexp* SimplifyCharClass(Regexp* re);
+
+  DISALLOW_EVIL_CONSTRUCTORS(SimplifyWalker);
+};
+
+// Simplifies a regular expression, returning a new regexp.
+// The new regexp uses traditional Unix egrep features only,
+// plus the Perl (?:) non-capturing parentheses.
+// Otherwise, no POSIX or Perl additions.  The new regexp
+// captures exactly the same subexpressions (with the same indices)
+// as the original.
+// Does not edit current object.
+// Caller must Decref() return value when done with it.
+
+Regexp* Regexp::Simplify() {
+  if (simple_)
+    return Incref();
+  SimplifyWalker w;
+  return w.Walk(this, NULL);
+}
+
+#define Simplify DontCallSimplify  // Avoid accidental recursion
+
+Regexp* SimplifyWalker::Copy(Regexp* re) {
+  return re->Incref();
+}
+
+Regexp* SimplifyWalker::ShortVisit(Regexp* re, Regexp* parent_arg) {
+  // This should never be called, since we use Walk and not
+  // WalkExponential.
+  LOG(DFATAL) << "SimplifyWalker::ShortVisit called";
+  return re->Incref();
+}
+
+Regexp* SimplifyWalker::PreVisit(Regexp* re, Regexp* parent_arg, bool* stop) {
+  if (re->simple_) {
+    *stop = true;
+    return re->Incref();
+  }
+  return NULL;
+}
+
+Regexp* SimplifyWalker::PostVisit(Regexp* re,
+                                  Regexp* parent_arg,
+                                  Regexp* pre_arg,
+                                  Regexp** child_args,
+                                  int nchild_args) {
+  switch (re->op()) {
+    case kRegexpNoMatch:
+    case kRegexpEmptyMatch:
+    case kRegexpLiteral:
+    case kRegexpLiteralString:
+    case kRegexpBeginLine:
+    case kRegexpEndLine:
+    case kRegexpBeginText:
+    case kRegexpWordBoundary:
+    case kRegexpNoWordBoundary:
+    case kRegexpEndText:
+    case kRegexpAnyChar:
+    case kRegexpAnyByte:
+    case kRegexpHaveMatch:
+      // All these are always simple.
+      re->simple_ = true;
+      return re->Incref();
+
+    case kRegexpConcat:
+    case kRegexpAlternate: {
+      // These are simple as long as the subpieces are simple.
+      // Two passes to avoid allocation in the common case.
+      bool changed = false;
+      Regexp** subs = re->sub();
+      for (int i = 0; i < re->nsub_; i++) {
+        Regexp* sub = subs[i];
+        Regexp* newsub = child_args[i];
+        if (newsub != sub) {
+          changed = true;
+          break;
+        }
+      }
+      if (!changed) {
+        for (int i = 0; i < re->nsub_; i++) {
+          Regexp* newsub = child_args[i];
+          newsub->Decref();
+        }
+        re->simple_ = true;
+        return re->Incref();
+      }
+      Regexp* nre = new Regexp(re->op(), re->parse_flags());
+      nre->AllocSub(re->nsub_);
+      Regexp** nre_subs = nre->sub();
+      for (int i = 0; i <re->nsub_; i++)
+        nre_subs[i] = child_args[i];
+      nre->simple_ = true;
+      return nre;
+    }
+
+    case kRegexpCapture: {
+      Regexp* newsub = child_args[0];
+      if (newsub == re->sub()[0]) {
+        newsub->Decref();
+        re->simple_ = true;
+        return re->Incref();
+      }
+      Regexp* nre = new Regexp(kRegexpCapture, re->parse_flags());
+      nre->AllocSub(1);
+      nre->sub()[0] = newsub;
+      nre->cap_ = re->cap_;
+      nre->simple_ = true;
+      return nre;
+    }
+
+    case kRegexpStar:
+    case kRegexpPlus:
+    case kRegexpQuest: {
+      Regexp* newsub = child_args[0];
+      // Special case: repeat the empty string as much as
+      // you want, but it's still the empty string.
+      if (newsub->op() == kRegexpEmptyMatch)
+        return newsub;
+
+      // These are simple as long as the subpiece is simple.
+      if (newsub == re->sub()[0]) {
+        newsub->Decref();
+        re->simple_ = true;
+        return re->Incref();
+      }
+
+      // These are also idempotent if flags are constant.
+      if (re->op() == newsub->op() &&
+          re->parse_flags() == newsub->parse_flags())
+        return newsub;
+
+      Regexp* nre = new Regexp(re->op(), re->parse_flags());
+      nre->AllocSub(1);
+      nre->sub()[0] = newsub;
+      nre->simple_ = true;
+      return nre;
+    }
+
+    case kRegexpRepeat: {
+      Regexp* newsub = child_args[0];
+      // Special case: repeat the empty string as much as
+      // you want, but it's still the empty string.
+      if (newsub->op() == kRegexpEmptyMatch)
+        return newsub;
+
+      Regexp* nre = SimplifyRepeat(newsub, re->min_, re->max_,
+                                   re->parse_flags());
+      newsub->Decref();
+      nre->simple_ = true;
+      return nre;
+    }
+
+    case kRegexpCharClass: {
+      Regexp* nre = SimplifyCharClass(re);
+      nre->simple_ = true;
+      return nre;
+    }
+  }
+
+  LOG(ERROR) << "Simplify case not handled: " << re->op();
+  return re->Incref();
+}
+
+// Creates a concatenation of two Regexp, consuming refs to re1 and re2.
+// Returns a new Regexp, handing the ref to the caller.
+Regexp* SimplifyWalker::Concat2(Regexp* re1, Regexp* re2,
+                                Regexp::ParseFlags parse_flags) {
+  Regexp* re = new Regexp(kRegexpConcat, parse_flags);
+  re->AllocSub(2);
+  Regexp** subs = re->sub();
+  subs[0] = re1;
+  subs[1] = re2;
+  return re;
+}
+
+// Simplifies the expression re{min,max} in terms of *, +, and ?.
+// Returns a new regexp.  Does not edit re.  Does not consume reference to re.
+// Caller must Decref return value when done with it.
+// The result will *not* necessarily have the right capturing parens
+// if you call ToString() and re-parse it: (x){2} becomes (x)(x),
+// but in the Regexp* representation, both (x) are marked as $1.
+Regexp* SimplifyWalker::SimplifyRepeat(Regexp* re, int min, int max,
+                                       Regexp::ParseFlags f) {
+  // x{n,} means at least n matches of x.
+  if (max == -1) {
+    // Special case: x{0,} is x*
+    if (min == 0)
+      return Regexp::Star(re->Incref(), f);
+
+    // Special case: x{1,} is x+
+    if (min == 1)
+      return Regexp::Plus(re->Incref(), f);
+
+    // General case: x{4,} is xxxx+
+    Regexp* nre = new Regexp(kRegexpConcat, f);
+    nre->AllocSub(min);
+    VLOG(1) << "Simplify " << min;
+    Regexp** nre_subs = nre->sub();
+    for (int i = 0; i < min-1; i++)
+      nre_subs[i] = re->Incref();
+    nre_subs[min-1] = Regexp::Plus(re->Incref(), f);
+    return nre;
+  }
+
+  // Special case: (x){0} matches only empty string.
+  if (min == 0 && max == 0)
+    return new Regexp(kRegexpEmptyMatch, f);
+
+  // Special case: x{1} is just x.
+  if (min == 1 && max == 1)
+    return re->Incref();
+
+  // General case: x{n,m} means n copies of x and m copies of x?.
+  // The machine will do less work if we nest the final m copies,
+  // so that x{2,5} = xx(x(x(x)?)?)?
+
+  // Build leading prefix: xx.  Capturing only on the last one.
+  Regexp* nre = NULL;
+  if (min > 0) {
+    nre = new Regexp(kRegexpConcat, f);
+    nre->AllocSub(min);
+    Regexp** nre_subs = nre->sub();
+    for (int i = 0; i < min; i++)
+      nre_subs[i] = re->Incref();
+  }
+
+  // Build and attach suffix: (x(x(x)?)?)?
+  if (max > min) {
+    Regexp* suf = Regexp::Quest(re->Incref(), f);
+    for (int i = min+1; i < max; i++)
+      suf = Regexp::Quest(Concat2(re->Incref(), suf, f), f);
+    if (nre == NULL)
+      nre = suf;
+    else
+      nre = Concat2(nre, suf, f);
+  }
+
+  if (nre == NULL) {
+    // Some degenerate case, like min > max, or min < max < 0.
+    // This shouldn't happen, because the parser rejects such regexps.
+    LOG(DFATAL) << "Malformed repeat " << re->ToString() << " " << min << " " << max;
+    return new Regexp(kRegexpNoMatch, f);
+  }
+
+  return nre;
+}
+
+// Simplifies a character class.
+// Caller must Decref return value when done with it.
+Regexp* SimplifyWalker::SimplifyCharClass(Regexp* re) {
+  CharClass* cc = re->cc();
+
+  // Special cases
+  if (cc->empty())
+    return new Regexp(kRegexpNoMatch, re->parse_flags());
+  if (cc->full())
+    return new Regexp(kRegexpAnyChar, re->parse_flags());
+
+  return re->Incref();
+}
+
+}  // namespace re2

data/ext/re2/stringpiece.cc

@@ -0,0 +1,87 @@
+// Copyright 2004 The RE2 Authors.  All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "re2/stringpiece.h"
+#include "util/util.h"
+
+using re2::StringPiece;
+
+std::ostream& operator<<(std::ostream& o, const StringPiece& piece) {
+  o.write(piece.data(), piece.size());
+  return o;
+}
+
+bool StringPiece::_equal(const StringPiece& x, const StringPiece& y) {
+  int len = x.size();
+  if (len != y.size()) {
+    return false;
+  }
+  const char* p = x.data();
+  const char* p2 = y.data();
+  // Test last byte in case strings share large common prefix
+  if ((len > 0) && (p[len-1] != p2[len-1])) return false;
+  const char* p_limit = p + len;
+  for (; p < p_limit; p++, p2++) {
+    if (*p != *p2)
+      return false;
+  }
+  return true;
+}
+
+void StringPiece::CopyToString(string* target) const {
+  target->assign(ptr_, length_);
+}
+
+int StringPiece::copy(char* buf, size_type n, size_type pos) const {
+  int ret = min(length_ - pos, n);
+  memcpy(buf, ptr_ + pos, ret);
+  return ret;
+}
+
+int StringPiece::find(const StringPiece& s, size_type pos) const {
+  if (length_ < 0 || pos > static_cast<size_type>(length_))
+    return npos;
+
+  const char* result = std::search(ptr_ + pos, ptr_ + length_,
+                                   s.ptr_, s.ptr_ + s.length_);
+  const size_type xpos = result - ptr_;
+  return xpos + s.length_ <= length_ ? xpos : npos;
+}
+
+int StringPiece::find(char c, size_type pos) const {
+  if (length_ <= 0 || pos >= static_cast<size_type>(length_)) {
+    return npos;
+  }
+  const char* result = std::find(ptr_ + pos, ptr_ + length_, c);
+  return result != ptr_ + length_ ? result - ptr_ : npos;
+}
+
+int StringPiece::rfind(const StringPiece& s, size_type pos) const {
+  if (length_ < s.length_) return npos;
+  const size_t ulen = length_;
+  if (s.length_ == 0) return min(ulen, pos);
+
+  const char* last = ptr_ + min(ulen - s.length_, pos) + s.length_;
+  const char* result = std::find_end(ptr_, last, s.ptr_, s.ptr_ + s.length_);
+  return result != last ? result - ptr_ : npos;
+}
+
+int StringPiece::rfind(char c, size_type pos) const {
+  if (length_ <= 0) return npos;
+  for (int i = min(pos, static_cast<size_type>(length_ - 1));
+       i >= 0; --i) {
+    if (ptr_[i] == c) {
+      return i;
+    }
+  }
+  return npos;
+}
+
+StringPiece StringPiece::substr(size_type pos, size_type n) const {
+  if (pos > length_) pos = length_;
+  if (n > length_ - pos) n = length_ - pos;
+  return StringPiece(ptr_ + pos, n);
+}
+
+const StringPiece::size_type StringPiece::npos = size_type(-1);

data/ext/re2/stringpiece.h

@@ -0,0 +1,182 @@
+// Copyright 2001-2010 The RE2 Authors.  All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// A string-like object that points to a sized piece of memory.
+//
+// Functions or methods may use const StringPiece& parameters to accept either
+// a "const char*" or a "string" value that will be implicitly converted to
+// a StringPiece.  The implicit conversion means that it is often appropriate
+// to include this .h file in other files rather than forward-declaring
+// StringPiece as would be appropriate for most other Google classes.
+//
+// Systematic usage of StringPiece is encouraged as it will reduce unnecessary
+// conversions from "const char*" to "string" and back again.
+//
+//
+// Arghh!  I wish C++ literals were "string".
+
+#ifndef STRINGS_STRINGPIECE_H__
+#define STRINGS_STRINGPIECE_H__
+
+#include <string.h>
+#include <cstddef>
+#include <iosfwd>
+#include <string>
+
+namespace re2 {
+
+class StringPiece {
+ private:
+  const char*   ptr_;
+  int           length_;
+
+ public:
+  // We provide non-explicit singleton constructors so users can pass
+  // in a "const char*" or a "string" wherever a "StringPiece" is
+  // expected.
+  StringPiece() : ptr_(NULL), length_(0) { }
+  StringPiece(const char* str)
+    : ptr_(str), length_((str == NULL) ? 0 : static_cast<int>(strlen(str))) { }
+  StringPiece(const std::string& str)
+    : ptr_(str.data()), length_(static_cast<int>(str.size())) { }
+  StringPiece(const char* offset, int len) : ptr_(offset), length_(len) { }
+
+  // data() may return a pointer to a buffer with embedded NULs, and the
+  // returned buffer may or may not be null terminated.  Therefore it is
+  // typically a mistake to pass data() to a routine that expects a NUL
+  // terminated string.
+  const char* data() const { return ptr_; }
+  int size() const { return length_; }
+  int length() const { return length_; }
+  bool empty() const { return length_ == 0; }
+
+  void clear() { ptr_ = NULL; length_ = 0; }
+  void set(const char* data, int len) { ptr_ = data; length_ = len; }
+  void set(const char* str) {
+    ptr_ = str;
+    if (str != NULL)
+      length_ = static_cast<int>(strlen(str));
+    else
+      length_ = 0;
+  }
+  void set(const void* data, int len) {
+    ptr_ = reinterpret_cast<const char*>(data);
+    length_ = len;
+  }
+
+  char operator[](int i) const { return ptr_[i]; }
+
+  void remove_prefix(int n) {
+    ptr_ += n;
+    length_ -= n;
+  }
+
+  void remove_suffix(int n) {
+    length_ -= n;
+  }
+
+  int compare(const StringPiece& x) const {
+    int r = memcmp(ptr_, x.ptr_, std::min(length_, x.length_));
+    if (r == 0) {
+      if (length_ < x.length_) r = -1;
+      else if (length_ > x.length_) r = +1;
+    }
+    return r;
+  }
+
+  std::string as_string() const {
+    return std::string(data(), size());
+  }
+  // We also define ToString() here, since many other string-like
+  // interfaces name the routine that converts to a C++ string
+  // "ToString", and it's confusing to have the method that does that
+  // for a StringPiece be called "as_string()".  We also leave the
+  // "as_string()" method defined here for existing code.
+  std::string ToString() const {
+    return std::string(data(), size());
+  }
+
+  void CopyToString(std::string* target) const;
+  void AppendToString(std::string* target) const;
+
+  // Does "this" start with "x"
+  bool starts_with(const StringPiece& x) const {
+    return ((length_ >= x.length_) &&
+            (memcmp(ptr_, x.ptr_, x.length_) == 0));
+  }
+
+  // Does "this" end with "x"
+  bool ends_with(const StringPiece& x) const {
+    return ((length_ >= x.length_) &&
+            (memcmp(ptr_ + (length_-x.length_), x.ptr_, x.length_) == 0));
+  }
+
+  // standard STL container boilerplate
+  typedef char value_type;
+  typedef const char* pointer;
+  typedef const char& reference;
+  typedef const char& const_reference;
+  typedef size_t size_type;
+  typedef ptrdiff_t difference_type;
+  static const size_type npos;
+  typedef const char* const_iterator;
+  typedef const char* iterator;
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+  iterator begin() const { return ptr_; }
+  iterator end() const { return ptr_ + length_; }
+  const_reverse_iterator rbegin() const {
+    return const_reverse_iterator(ptr_ + length_);
+  }
+  const_reverse_iterator rend() const {
+    return const_reverse_iterator(ptr_);
+  }
+  // STLS says return size_type, but Google says return int
+  int max_size() const { return length_; }
+  int capacity() const { return length_; }
+
+  int copy(char* buf, size_type n, size_type pos = 0) const;
+
+  int find(const StringPiece& s, size_type pos = 0) const;
+  int find(char c, size_type pos = 0) const;
+  int rfind(const StringPiece& s, size_type pos = npos) const;
+  int rfind(char c, size_type pos = npos) const;
+
+  StringPiece substr(size_type pos, size_type n = npos) const;
+  
+  static bool _equal(const StringPiece&, const StringPiece&);
+};
+
+inline bool operator==(const StringPiece& x, const StringPiece& y) {
+  return StringPiece::_equal(x, y);
+}
+
+inline bool operator!=(const StringPiece& x, const StringPiece& y) {
+  return !(x == y);
+}
+
+inline bool operator<(const StringPiece& x, const StringPiece& y) {
+  const int r = memcmp(x.data(), y.data(),
+                       std::min(x.size(), y.size()));
+  return ((r < 0) || ((r == 0) && (x.size() < y.size())));
+}
+
+inline bool operator>(const StringPiece& x, const StringPiece& y) {
+  return y < x;
+}
+
+inline bool operator<=(const StringPiece& x, const StringPiece& y) {
+  return !(x > y);
+}
+
+inline bool operator>=(const StringPiece& x, const StringPiece& y) {
+  return !(x < y);
+}
+
+}  // namespace re2
+
+// allow StringPiece to be logged
+extern std::ostream& operator<<(std::ostream& o, const re2::StringPiece& piece);
+
+#endif  // STRINGS_STRINGPIECE_H__