chipper 0.4.2

data/ext/re2/prefilter.h

@@ -0,0 +1,105 @@
+// Copyright 2009 The RE2 Authors.  All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Prefilter is the class used to extract string guards from regexps.
+// Rather than using Prefilter class directly, use FilteredRE2.
+// See filtered_re2.h
+
+#ifndef RE2_PREFILTER_H_
+#define RE2_PREFILTER_H_
+
+#include "util/util.h"
+
+namespace re2 {
+
+class RE2;
+
+class Regexp;
+
+class Prefilter {
+  // Instead of using Prefilter directly, use FilteredRE2; see filtered_re2.h
+ public:
+  enum Op {
+    ALL = 0,  // Everything matches
+    NONE,  // Nothing matches
+    ATOM,  // The string atom() must match
+    AND,   // All in subs() must match
+    OR,   // One of subs() must match
+  };
+
+  explicit Prefilter(Op op);
+  ~Prefilter();
+
+  Op op() { return op_; }
+  const string& atom() const { return atom_; }
+  void set_unique_id(int id) { unique_id_ = id; }
+  int unique_id() const { return unique_id_; }
+
+  // The children of the Prefilter node.
+  vector<Prefilter*>* subs() {
+    CHECK(op_ == AND || op_ == OR);
+    return subs_;
+  }
+
+  // Set the children vector. Prefilter takes ownership of subs and
+  // subs_ will be deleted when Prefilter is deleted.
+  void set_subs(vector<Prefilter*>* subs) { subs_ = subs; }
+
+  // Given a RE2, return a Prefilter. The caller takes ownership of
+  // the Prefilter and should deallocate it. Returns NULL if Prefilter
+  // cannot be formed.
+  static Prefilter* FromRE2(const RE2* re2);
+
+  // Returns a readable debug string of the prefilter.
+  string DebugString() const;
+
+ private:
+  class Info;
+
+  // Combines two prefilters together to create an AND. The passed
+  // Prefilters will be part of the returned Prefilter or deleted.
+  static Prefilter* And(Prefilter* a, Prefilter* b);
+
+  // Combines two prefilters together to create an OR. The passed
+  // Prefilters will be part of the returned Prefilter or deleted.
+  static Prefilter* Or(Prefilter* a, Prefilter* b);
+
+  // Generalized And/Or
+  static Prefilter* AndOr(Op op, Prefilter* a, Prefilter* b);
+
+  static Prefilter* FromRegexp(Regexp* a);
+
+  static Prefilter* FromString(const string& str);
+
+  static Prefilter* OrStrings(set<string>* ss);
+
+  static Info* BuildInfo(Regexp* re);
+
+  Prefilter* Simplify();
+
+  // Kind of Prefilter.
+  Op op_;
+
+  // Sub-matches for AND or OR Prefilter.
+  vector<Prefilter*>* subs_;
+
+  // Actual string to match in leaf node.
+  string atom_;
+
+  // If different prefilters have the same string atom, or if they are
+  // structurally the same (e.g., OR of same atom strings) they are
+  // considered the same unique nodes. This is the id for each unique
+  // node. This field is populated with a unique id for every node,
+  // and -1 for duplicate nodes.
+  int unique_id_;
+
+  // Used for debugging, helps in tracking memory leaks.
+  int alloc_id_;
+
+  DISALLOW_EVIL_CONSTRUCTORS(Prefilter);
+};
+
+}  // namespace re2
+
+#endif  // RE2_PREFILTER_H_

data/ext/re2/prefilter_tree.cc

@@ -0,0 +1,398 @@
+// Copyright 2009 The RE2 Authors.  All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "util/util.h"
+#include "util/flags.h"
+#include "re2/prefilter.h"
+#include "re2/prefilter_tree.h"
+#include "re2/re2.h"
+
+DEFINE_int32(filtered_re2_min_atom_len,
+             3,
+             "Strings less than this length are not stored as atoms");
+
+namespace re2 {
+
+PrefilterTree::PrefilterTree()
+    : compiled_(false) {
+}
+
+PrefilterTree::~PrefilterTree() {
+  for (int i = 0; i < prefilter_vec_.size(); i++)
+    delete prefilter_vec_[i];
+
+  for (int i = 0; i < entries_.size(); i++)
+    delete entries_[i].parents;
+}
+
+// Functions used for adding and Compiling prefilters to the
+// PrefilterTree.
+static bool KeepPart(Prefilter* prefilter, int level) {
+  if (prefilter == NULL)
+    return false;
+
+  switch (prefilter->op()) {
+    default:
+      LOG(DFATAL) << "Unexpected op in KeepPart: "
+                  << prefilter->op();
+      return false;
+
+    case Prefilter::ALL:
+      return false;
+
+    case Prefilter::ATOM:
+      return prefilter->atom().size() >=
+          FLAGS_filtered_re2_min_atom_len;
+
+    case Prefilter::AND: {
+      int j = 0;
+      vector<Prefilter*>* subs = prefilter->subs();
+      for (int i = 0; i < subs->size(); i++)
+        if (KeepPart((*subs)[i], level + 1))
+          (*subs)[j++] = (*subs)[i];
+        else
+          delete (*subs)[i];
+
+      subs->resize(j);
+      return j > 0;
+    }
+
+    case Prefilter::OR:
+      for (int i = 0; i < prefilter->subs()->size(); i++)
+        if (!KeepPart((*prefilter->subs())[i], level + 1))
+          return false;
+      return true;
+  }
+}
+
+void PrefilterTree::Add(Prefilter *f) {
+  if (compiled_) {
+    LOG(DFATAL) << "Add after Compile.";
+    return;
+  }
+  if (f != NULL && !KeepPart(f, 0)) {
+    delete f;
+    f = NULL;
+  }
+
+  prefilter_vec_.push_back(f);
+}
+
+void PrefilterTree::Compile(vector<string>* atom_vec) {
+  if (compiled_) {
+    LOG(DFATAL) << "Compile after Compile.";
+    return;
+  }
+
+  // We do this check to support some legacy uses of
+  // PrefilterTree that call Compile before adding any regexps,
+  // and expect Compile not to have effect.
+  if (prefilter_vec_.empty())
+    return;
+
+  compiled_ = true;
+
+  AssignUniqueIds(atom_vec);
+
+  // Identify nodes that are too common among prefilters and are
+  // triggering too many parents. Then get rid of them if possible.
+  // Note that getting rid of a prefilter node simply means they are
+  // no longer necessary for their parent to trigger; that is, we do
+  // not miss out on any regexps triggering by getting rid of a
+  // prefilter node.
+  for (int i = 0; i < entries_.size(); i++) {
+    IntMap* parents = entries_[i].parents;
+    if (parents->size() > 8) {
+      // This one triggers too many things. If all the parents are AND
+      // nodes and have other things guarding them, then get rid of
+      // this trigger. TODO(vsri): Adjust the threshold appropriately,
+      // make it a function of total number of nodes?
+      bool have_other_guard = true;
+      for (IntMap::iterator it = parents->begin(); it != parents->end(); ++it)
+        have_other_guard = have_other_guard &&
+            (entries_[it->index()].propagate_up_at_count > 1);
+
+      if (have_other_guard) {
+        for (IntMap::iterator it = parents->begin();
+             it != parents->end(); ++it)
+          entries_[it->index()].propagate_up_at_count -= 1;
+
+        parents->clear();  // Forget the parents
+      }
+    }
+  }
+
+  PrintDebugInfo();
+}
+
+Prefilter* PrefilterTree::CanonicalNode(Prefilter* node) {
+  string node_string = NodeString(node);
+  map<string, Prefilter*>::iterator iter = node_map_.find(node_string);
+  if (iter == node_map_.end())
+    return NULL;
+  return (*iter).second;
+}
+
+static string Itoa(int n) {
+  char buf[100];
+  snprintf(buf, sizeof buf, "%d", n);
+  return string(buf);
+}
+
+string PrefilterTree::NodeString(Prefilter* node) const {
+  // Adding the operation disambiguates AND/OR/atom nodes.
+  string s = Itoa(node->op()) + ":";
+  if (node->op() == Prefilter::ATOM) {
+    s += node->atom();
+  } else {
+    for (int i = 0; i < node->subs()->size() ; i++) {
+      if (i > 0)
+        s += ',';
+      s += Itoa((*node->subs())[i]->unique_id());
+    }
+  }
+  return s;
+}
+
+void PrefilterTree::AssignUniqueIds(vector<string>* atom_vec) {
+  atom_vec->clear();
+
+  // Build vector of all filter nodes, sorted topologically
+  // from top to bottom in v.
+  vector<Prefilter*> v;
+
+  // Add the top level nodes of each regexp prefilter.
+  for (int i = 0; i < prefilter_vec_.size(); i++) {
+    Prefilter* f = prefilter_vec_[i];
+    if (f == NULL)
+      unfiltered_.push_back(i);
+
+    // We push NULL also on to v, so that we maintain the
+    // mapping of index==regexpid for level=0 prefilter nodes.
+    v.push_back(f);
+  }
+
+  // Now add all the descendant nodes.
+  for (int i = 0; i < v.size(); i++) {
+    Prefilter* f = v[i];
+    if (f == NULL)
+      continue;
+    if (f->op() == Prefilter::AND || f->op() == Prefilter::OR) {
+      const vector<Prefilter*>& subs = *f->subs();
+      for (int j = 0; j < subs.size(); j++)
+        v.push_back(subs[j]);
+    }
+  }
+
+  // Identify unique nodes.
+  int unique_id = 0;
+  for (int i = v.size() - 1; i >= 0; i--) {
+    Prefilter *node = v[i];
+    if (node == NULL)
+      continue;
+    node->set_unique_id(-1);
+    Prefilter* canonical = CanonicalNode(node);
+    if (canonical == NULL) {
+      // Any further nodes that have the same node string
+      // will find this node as the canonical node.
+      node_map_[NodeString(node)] = node;
+      if (node->op() == Prefilter::ATOM) {
+        atom_vec->push_back(node->atom());
+        atom_index_to_id_.push_back(unique_id);
+      }
+      node->set_unique_id(unique_id++);
+    } else {
+      node->set_unique_id(canonical->unique_id());
+    }
+  }
+  entries_.resize(node_map_.size());
+
+  // Create parent IntMap for the entries.
+  for (int i = v.size()  - 1; i >= 0; i--) {
+    Prefilter* prefilter = v[i];
+    if (prefilter == NULL)
+      continue;
+
+    if (CanonicalNode(prefilter) != prefilter)
+      continue;
+
+    Entry* entry = &entries_[prefilter->unique_id()];
+    entry->parents = new IntMap(node_map_.size());
+  }
+
+  // Fill the entries.
+  for (int i = v.size()  - 1; i >= 0; i--) {
+    Prefilter* prefilter = v[i];
+    if (prefilter == NULL)
+      continue;
+
+    if (CanonicalNode(prefilter) != prefilter)
+      continue;
+
+    Entry* entry = &entries_[prefilter->unique_id()];
+
+    switch (prefilter->op()) {
+      default:
+      case Prefilter::ALL:
+        LOG(DFATAL) << "Unexpected op: " << prefilter->op();
+        return;
+
+      case Prefilter::ATOM:
+        entry->propagate_up_at_count = 1;
+        break;
+
+      case Prefilter::OR:
+      case Prefilter::AND: {
+        IntMap uniq_child(node_map_.size());
+        for (int j = 0; j < prefilter->subs()->size() ; j++) {
+          Prefilter* child = (*prefilter->subs())[j];
+          Prefilter* canonical = CanonicalNode(child);
+          if (canonical == NULL) {
+            LOG(DFATAL) << "Null canonical node";
+            return;
+          }
+          int child_id = canonical->unique_id();
+          if (!uniq_child.has_index(child_id))
+            uniq_child.set_new(child_id, 1);
+          // To the child, we want to add to parent indices.
+          Entry* child_entry = &entries_[child_id];
+          if (!child_entry->parents->has_index(prefilter->unique_id()))
+            child_entry->parents->set_new(prefilter->unique_id(), 1);
+        }
+        entry->propagate_up_at_count =
+            prefilter->op() == Prefilter::AND ? uniq_child.size() : 1;
+
+        break;
+      }
+    }
+  }
+
+  // For top level nodes, populate regexp id.
+  for (int i = 0; i < prefilter_vec_.size(); i++) {
+    if (prefilter_vec_[i] == NULL)
+      continue;
+    int id = CanonicalNode(prefilter_vec_[i])->unique_id();
+    DCHECK_LE(0, id);
+    Entry* entry = &entries_[id];
+    entry->regexps.push_back(i);
+  }
+}
+
+// Functions for triggering during search.
+void PrefilterTree::RegexpsGivenStrings(
+    const vector<int>& matched_atoms,
+    vector<int>* regexps) const {
+  regexps->clear();
+  if (!compiled_) {
+    LOG(WARNING) << "Compile() not called";
+    for (int i = 0; i < prefilter_vec_.size(); ++i)
+      regexps->push_back(i);
+  } else {
+    if (!prefilter_vec_.empty()) {
+      IntMap regexps_map(prefilter_vec_.size());
+      vector<int> matched_atom_ids;
+      for (int j = 0; j < matched_atoms.size(); j++) {
+        matched_atom_ids.push_back(atom_index_to_id_[matched_atoms[j]]);
+        VLOG(10) << "Atom id:" << atom_index_to_id_[matched_atoms[j]];
+      }
+      PropagateMatch(matched_atom_ids, &regexps_map);
+      for (IntMap::iterator it = regexps_map.begin();
+           it != regexps_map.end();
+           ++it)
+        regexps->push_back(it->index());
+
+      regexps->insert(regexps->end(), unfiltered_.begin(), unfiltered_.end());
+    }
+  }
+  sort(regexps->begin(), regexps->end());
+}
+
+void PrefilterTree::PropagateMatch(const vector<int>& atom_ids,
+                                   IntMap* regexps) const {
+  IntMap count(entries_.size());
+  IntMap work(entries_.size());
+  for (int i = 0; i < atom_ids.size(); i++)
+    work.set(atom_ids[i], 1);
+  for (IntMap::iterator it = work.begin(); it != work.end(); ++it) {
+    const Entry& entry = entries_[it->index()];
+    VLOG(10) << "Processing: " << it->index();
+    // Record regexps triggered.
+    for (int i = 0; i < entry.regexps.size(); i++) {
+      VLOG(10) << "Regexp triggered: " << entry.regexps[i];
+      regexps->set(entry.regexps[i], 1);
+    }
+    int c;
+    // Pass trigger up to parents.
+    for (IntMap::iterator it = entry.parents->begin();
+         it != entry.parents->end();
+         ++it) {
+      int j = it->index();
+      const Entry& parent = entries_[j];
+      VLOG(10) << " parent= " << j << " trig= " << parent.propagate_up_at_count;
+      // Delay until all the children have succeeded.
+      if (parent.propagate_up_at_count > 1) {
+        if (count.has_index(j)) {
+          c = count.get_existing(j) + 1;
+          count.set_existing(j, c);
+        } else {
+          c = 1;
+          count.set_new(j, c);
+        }
+        if (c < parent.propagate_up_at_count)
+          continue;
+      }
+      VLOG(10) << "Triggering: " << j;
+      // Trigger the parent.
+      work.set(j, 1);
+    }
+  }
+}
+
+// Debugging help.
+void PrefilterTree::PrintPrefilter(int regexpid) {
+  LOG(INFO) << DebugNodeString(prefilter_vec_[regexpid]);
+}
+
+void PrefilterTree::PrintDebugInfo() {
+  VLOG(10) << "#Unique Atoms: " << atom_index_to_id_.size();
+  VLOG(10) << "#Unique Nodes: " << entries_.size();
+
+  for (int i = 0; i < entries_.size(); ++i) {
+    IntMap* parents = entries_[i].parents;
+    const vector<int>& regexps = entries_[i].regexps;
+    VLOG(10) << "EntryId: " << i
+            << " N: " << parents->size() << " R: " << regexps.size();
+    for (IntMap::iterator it = parents->begin(); it != parents->end(); ++it)
+      VLOG(10) << it->index();
+  }
+  VLOG(10) << "Map:";
+  for (map<string, Prefilter*>::const_iterator iter = node_map_.begin();
+       iter != node_map_.end(); ++iter)
+    VLOG(10) << "NodeId: " << (*iter).second->unique_id()
+            << " Str: " << (*iter).first;
+}
+
+string PrefilterTree::DebugNodeString(Prefilter* node) const {
+  string node_string = "";
+
+  if (node->op() == Prefilter::ATOM) {
+    DCHECK(!node->atom().empty());
+    node_string += node->atom();
+  } else {
+    // Adding the operation disambiguates AND and OR nodes.
+    node_string +=  node->op() == Prefilter::AND ? "AND" : "OR";
+    node_string += "(";
+    for (int i = 0; i < node->subs()->size() ; i++) {
+      if (i > 0)
+        node_string += ',';
+      node_string += Itoa((*node->subs())[i]->unique_id());
+      node_string += ":";
+      node_string += DebugNodeString((*node->subs())[i]);
+    }
+    node_string += ")";
+  }
+  return node_string;
+}
+
+}  // namespace re2

data/ext/re2/prefilter_tree.h

@@ -0,0 +1,130 @@
+// Copyright 2009 The RE2 Authors.  All Rights Reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The PrefilterTree class is used to form an AND-OR tree of strings
+// that would trigger each regexp. The 'prefilter' of each regexp is
+// added tp PrefilterTree, and then PrefilterTree is used to find all
+// the unique strings across the prefilters. During search, by using
+// matches from a string matching engine, PrefilterTree deduces the
+// set of regexps that are to be triggered. The 'string matching
+// engine' itself is outside of this class, and the caller can use any
+// favorite engine. PrefilterTree provides a set of strings (called
+// atoms) that the user of this class should use to do the string
+// matching.
+//
+#ifndef RE2_PREFILTER_TREE_H_
+#define RE2_PREFILTER_TREE_H_
+
+#include "util/util.h"
+#include "util/sparse_array.h"
+
+namespace re2 {
+
+typedef SparseArray<int> IntMap;
+
+class Prefilter;
+
+class PrefilterTree {
+ public:
+  PrefilterTree();
+  ~PrefilterTree();
+
+  // Adds the prefilter for the next regexp. Note that we assume that
+  // Add called sequentially for all regexps. All Add calls
+  // must precede Compile.
+  void Add(Prefilter* prefilter);
+
+  // The Compile returns a vector of string in atom_vec.
+  // Call this after all the prefilters are added through Add.
+  // No calls to Add after Compile are allowed.
+  // The caller should use the returned set of strings to do string matching.
+  // Each time a string matches, the corresponding index then has to be
+  // and passed to RegexpsGivenStrings below.
+  void Compile(vector<string>* atom_vec);
+
+  // Given the indices of the atoms that matched, returns the indexes
+  // of regexps that should be searched.  The matched_atoms should
+  // contain all the ids of string atoms that were found to match the
+  // content. The caller can use any string match engine to perform
+  // this function. This function is thread safe.
+  void RegexpsGivenStrings(const vector<int>& matched_atoms,
+                           vector<int>* regexps) const;
+
+  // Print debug prefilter. Also prints unique ids associated with
+  // nodes of the prefilter of the regexp.
+  void PrintPrefilter(int regexpid);
+
+
+  // Each unique node has a corresponding Entry that helps in
+  // passing the matching trigger information along the tree.
+  struct Entry {
+   public:
+    // How many children should match before this node triggers the
+    // parent. For an atom and an OR node, this is 1 and for an AND
+    // node, it is the number of unique children.
+    int propagate_up_at_count;
+
+    // When this node is ready to trigger the parent, what are the indices
+    // of the parent nodes to trigger. The reason there may be more than
+    // one is because of sharing. For example (abc | def) and (xyz | def)
+    // are two different nodes, but they share the atom 'def'. So when
+    // 'def' matches, it triggers two parents, corresponding to the two
+    // different OR nodes.
+    IntMap* parents;
+
+    // When this node is ready to trigger the parent, what are the
+    // regexps that are triggered.
+    vector<int> regexps;
+  };
+
+ private:
+  // This function assigns unique ids to various parts of the
+  // prefilter, by looking at if these nodes are already in the
+  // PrefilterTree.
+  void AssignUniqueIds(vector<string>* atom_vec);
+
+  // Given the matching atoms, find the regexps to be triggered.
+  void PropagateMatch(const vector<int>& atom_ids,
+                      IntMap* regexps) const;
+
+  // Returns the prefilter node that has the same NodeString as this
+  // node. For the canonical node, returns node.
+  Prefilter* CanonicalNode(Prefilter* node);
+
+  // A string that uniquely identifies the node. Assumes that the
+  // children of node has already been assigned unique ids.
+  string NodeString(Prefilter* node) const;
+
+  // Recursively constructs a readable prefilter string.
+  string DebugNodeString(Prefilter* node) const;
+
+  // Used for debugging.
+  void PrintDebugInfo();
+
+  // These are all the nodes formed by Compile. Essentially, there is
+  // one node for each unique atom and each unique AND/OR node.
+  vector<Entry> entries_;
+
+  // Map node string to canonical Prefilter node.
+  map<string, Prefilter*> node_map_;
+
+  // indices of regexps that always pass through the filter (since we
+  // found no required literals in these regexps).
+  vector<int> unfiltered_;
+
+  // vector of Prefilter for all regexps.
+  vector<Prefilter*> prefilter_vec_;
+
+  // Atom index in returned strings to entry id mapping.
+  vector<int> atom_index_to_id_;
+
+  // Has the prefilter tree been compiled.
+  bool compiled_;
+
+  DISALLOW_EVIL_CONSTRUCTORS(PrefilterTree);
+};
+
+}  // namespace
+
+#endif  // RE2_PREFILTER_TREE_H_