ferret 0.9.1 → 0.9.2

data/ext/r_store.c

@@ -6,6 +6,10 @@ VALUE cDirectory;
 VALUE cRAMDirectory;
 VALUE cFSDirectory;
 
+
+static ID id_mkdir_p;
+static ID id_is_directory;
+
 /****************************************************************************
  *
  * Lock Methods
@@ -87,11 +91,10 @@ frt_lock_release(VALUE self)
  ****************************************************************************/
 
 void
-frt_dir_free(void *p)
+frt_dir_free(Store *store)
 {
-  Store *store = (Store *)p;
-  object_del(p);
-  store->close(store);
+  object_del(store);
+  store_deref(store);
 }
 
 #define GET_STORE Store *store; Data_Get_Struct(self, Store, store)
@@ -99,10 +102,11 @@ static VALUE
 frt_dir_close(VALUE self)
 {
   /*
+   * No need to do anything here. Leave it do the garbage collector
   GET_STORE;
   Frt_Unwrap_Struct(self);
   object_del(store);
-  store->close(store);
+  store_deref(store);
   */
   return Qnil;
 }
@@ -212,11 +216,23 @@ frt_fsdir_new(VALUE klass, VALUE rpath, VALUE rcreate)
   Store *store;
   bool create = RTEST(rcreate);
   rpath = rb_obj_as_string(rpath);
+  if (create) {
+    VALUE mFileUtils;
+    rb_require("fileutils");
+    mFileUtils = rb_define_module("FileUtils");
+    rb_funcall(mFileUtils, id_mkdir_p, 1, rpath);
+  }
+  if (!rb_funcall(rb_cFile, id_is_directory, 1, rpath)) {
+    rb_raise(rb_eIOError, "There is no directory: %s. Use create = true to "
+        "create one.", RSTRING(rpath)->ptr);
+  }
   store = open_fs_store(RSTRING(rpath)->ptr);
   if (create) store->clear_all(store);
   if ((self = object_get(store)) == Qnil) {
     self = Data_Wrap_Struct(klass, NULL, &frt_dir_free, store);
     object_add(store, self);
+  } else {
+    store_deref(store);
   }
   return self;
 }
@@ -227,19 +243,12 @@ frt_fsdir_new(VALUE klass, VALUE rpath, VALUE rcreate)
  *
  ****************************************************************************/
 
-#define DIR_METHODS(dir)\
-  rb_define_method(dir, "close", frt_dir_close, 0);\
-  rb_define_method(dir, "exists?", frt_dir_exists, 1);\
-  rb_define_method(dir, "touch", frt_dir_touch, 1);\
-  rb_define_method(dir, "delete", frt_dir_delete, 1);\
-  rb_define_method(dir, "file_count", frt_dir_file_count, 0);\
-  rb_define_method(dir, "refresh", frt_dir_refresh, 0);\
-  rb_define_method(dir, "rename", frt_dir_rename, 2);\
-  rb_define_method(dir, "make_lock", frt_dir_make_lock, 1);
-
 void
 Init_dir(void)
 {
+  id_mkdir_p = rb_intern("mkdir_p");
+  id_is_directory = rb_intern("directory?");
+
   cLock = rb_define_class_under(mStore, "Lock", rb_cObject);
   rb_define_method(cLock, "obtain", frt_lock_obtain, -1);
   rb_define_method(cLock, "while_locked", frt_lock_while_locked, -1);
@@ -248,16 +257,22 @@ Init_dir(void)
 
   cDirectory = rb_define_class_under(mStore, "Directory", rb_cObject);
   rb_define_const(cDirectory, "LOCK_PREFIX", rb_str_new2(LOCK_PREFIX));
+  rb_define_method(cDirectory, "close", frt_dir_close, 0);\
+  rb_define_method(cDirectory, "exists?", frt_dir_exists, 1);\
+  rb_define_method(cDirectory, "touch", frt_dir_touch, 1);\
+  rb_define_method(cDirectory, "delete", frt_dir_delete, 1);\
+  rb_define_method(cDirectory, "file_count", frt_dir_file_count, 0);\
+  rb_define_method(cDirectory, "refresh", frt_dir_refresh, 0);\
+  rb_define_method(cDirectory, "rename", frt_dir_rename, 2);\
+  rb_define_method(cDirectory, "make_lock", frt_dir_make_lock, 1);
 
   /* RAMDirectory */
   cRAMDirectory = rb_define_class_under(mStore, "RAMDirectory", cDirectory);
   rb_define_alloc_func(cRAMDirectory, frt_data_alloc);
   rb_define_method(cRAMDirectory, "initialize", frt_ramdir_init, -1);
-  DIR_METHODS(cRAMDirectory);
 
   /* FSDirectory */
   cFSDirectory = rb_define_class_under(mStore, "FSDirectory", cDirectory);
   rb_define_alloc_func(cFSDirectory, frt_data_alloc);
   rb_define_singleton_method(cFSDirectory, "new", frt_fsdir_new, 2);
-  DIR_METHODS(cFSDirectory);
 }

data/ext/ram_store.c

@@ -4,12 +4,14 @@
 static char * const RENAME_ERROR_MSG = "tried to rename a file that doesn't exist";
 static char * const MISSING_RAMFILE_ERROR_MSG ="Couldn't open the ram file to read";
 
+extern void store_destroy(Store *store);
+
 typedef struct RamFile {
   char *name;
   uchar **buffers;
   int bufcnt;
   int len;
-  int refcnt;
+  int ref_cnt;
   bool alive;
 } RamFile;
 
@@ -21,7 +23,7 @@ RamFile *rf_create(const char *name)
   rf->name = estrdup(name);
   rf->len = 0;
   rf->bufcnt = 1;
-  rf->refcnt = 0;
+  rf->ref_cnt = 0;
   rf->alive = true;
   return rf;
 }
@@ -38,7 +40,7 @@ void rf_close(void *p)
 {
   int i;
   RamFile *rf = (RamFile *)p;
-  if (rf->refcnt > 0 || rf->alive) return;
+  if (rf->ref_cnt > 0 || rf->alive) return;
   free(rf->name);
   for (i = 0; i < rf->bufcnt; i++) {
     free(rf->buffers[i]);
@@ -76,17 +78,21 @@ int ram_remove(Store *store, char *filename)
 int ram_rename(Store *store, char *from, char *to)
 {
   RamFile *rf = (RamFile *)h_rem(store->dir.ht, from, false);
-  if (rf == NULL)
+  RamFile *tmp;
+
+  if (rf == NULL) {
     RAISE(IO_ERROR, RENAME_ERROR_MSG);
+  }
 
   free(rf->name);
 
   rf->name = estrdup(to);
 
-  // clean up the file we are overwriting
-  RamFile *tmp = (RamFile *)h_get(store->dir.ht, to);
-  if (tmp != NULL)
+  /* clean up the file we are overwriting */
+  tmp = (RamFile *)h_get(store->dir.ht, to);
+  if (tmp != NULL) {
     tmp->alive = false;
+  }
 
   h_set(store->dir.ht, rf->name, rf);
   return true;
@@ -112,7 +118,7 @@ void ram_each(Store *store, void (*func)(char *fname, void *arg), void *arg)
   }
 }
 
-void ram_close(Store *store)
+void ram_close_i(Store *store)
 {
   HshTable *ht = store->dir.ht;
   RamFile *rf;
@@ -185,6 +191,7 @@ int ramo_length(OutStream *os)
 
 void ramo_flush_internal(OutStream *os, uchar *src, int len)
 {
+  uchar *buffer;
   RamFile *rf = (RamFile *)os->file;
   int buffer_number, buffer_offset, bytes_in_buffer, bytes_to_copy;
   int src_offset;
@@ -197,7 +204,7 @@ void ramo_flush_internal(OutStream *os, uchar *src, int len)
 
   rf_extend_if_necessary(rf, buffer_number);
 
-  uchar *buffer = rf->buffers[buffer_number];
+  buffer = rf->buffers[buffer_number];
   memcpy(buffer + buffer_offset, src, bytes_to_copy);
   
   if (bytes_to_copy < len) {
@@ -230,7 +237,7 @@ void ramo_reset(OutStream *os)
 void ramo_close_internal(OutStream *os)
 {
   RamFile *rf = (RamFile *)os->file;
-  rf->refcnt--;
+  rf->ref_cnt--;
   rf_close(rf);
 }
 
@@ -238,9 +245,12 @@ void ramo_write_to(OutStream *os, OutStream *other_o)
 {
   int i, len;
   RamFile *rf = (RamFile *)os->file;
+  int last_buffer_number;
+  int last_buffer_offset;
+
   os_flush(os);
-  int last_buffer_number = (int)(rf->len / BUFFER_SIZE);
-  int last_buffer_offset = rf->len % BUFFER_SIZE;
+  last_buffer_number = (int)(rf->len / BUFFER_SIZE);
+  last_buffer_offset = rf->len % BUFFER_SIZE;
   for (i = 0; i <= last_buffer_number; i++) {
     len = (i == last_buffer_number ? last_buffer_offset : BUFFER_SIZE);
     os_write_bytes(other_o, rf->buffers[i], len);
@@ -250,8 +260,9 @@ void ramo_write_to(OutStream *os, OutStream *other_o)
 OutStream *ram_create_buffer()
 {
   RamFile *rf = rf_create("");
-  rf->alive = false;
   OutStream *os = os_create();
+
+  rf->alive = false;
   os->file = rf;
   os->pointer = 0;
   os->flush_internal = &ramo_flush_internal;
@@ -269,12 +280,13 @@ void ram_destroy_buffer(OutStream *os)
 OutStream *ram_create_output(Store *store, const char *filename)
 {
   RamFile *rf = (RamFile *)h_get(store->dir.ht, filename);
+  OutStream *os = os_create();
+
   if (rf == NULL) {
     rf = rf_create(filename);
     h_set(store->dir.ht, rf->name, rf);
   }
-  rf->refcnt++;
-  OutStream *os = os_create();
+  rf->ref_cnt++;
   os->pointer = 0;
   os->file = rf;
   os->flush_internal = &ramo_flush_internal;
@@ -325,23 +337,24 @@ void rami_seek_internal(InStream *is, int pos)
 void rami_close_internal(InStream *is)
 {
   RamFile *rf = (RamFile *)is->file;
-  rf->refcnt--;
+  rf->ref_cnt--;
   rf_close(rf);
 }
 
 void rami_clone_internal(InStream *is, InStream *new_index_i)
 {
-  ((RamFile *)is->file)->refcnt++;
+  ((RamFile *)is->file)->ref_cnt++;
 }
 
 InStream *ram_open_input(Store *store, const char *filename)
 {
   RamFile *rf = (RamFile *)h_get(store->dir.ht, filename);
+  InStream *is = is_create();
+
   if (rf == NULL) {
     RAISE(IO_ERROR, MISSING_RAMFILE_ERROR_MSG);
   }
-  rf->refcnt++;
-  InStream *is = is_create();
+  rf->ref_cnt++;
   is->file = rf;
   is->d.pointer = 0;
   is->is_clone = false;
@@ -404,7 +417,6 @@ Store *open_ram_store()
   new_store->remove        = &ram_remove;
   new_store->rename        = &ram_rename;
   new_store->count         = &ram_count;
-  new_store->close         = &ram_close;
   new_store->clear         = &ram_clear;
   new_store->clear_all     = &ram_clear_all;
   new_store->clear_locks   = &ram_clear_locks;
@@ -414,6 +426,7 @@ Store *open_ram_store()
   new_store->open_input    = &ram_open_input;
   new_store->open_lock     = &ram_open_lock;
   new_store->close_lock    = &ram_close_lock;
+  new_store->close_i       = &ram_close_i;
   return new_store;
 }
 
@@ -427,11 +440,14 @@ static void copy_files(char *fname, void *arg)
   OutStream *os = cfa->to_store->create_output(cfa->to_store, fname);
   InStream *is = cfa->from_store->open_input(cfa->from_store, fname);
   int len = is_length(is);
-  uchar buffer[len+1];
+  uchar *buffer = ALLOC_N(uchar, len+1);
+  
   is_read_bytes(is, buffer, 0, len);
   os_write_bytes(os, buffer, len);
+
   is_close(is);
   os_close(os);
+  free(buffer);
 }
 
 Store *open_ram_store_and_copy(Store *from_store, bool close_dir)
@@ -444,7 +460,7 @@ Store *open_ram_store_and_copy(Store *from_store, bool close_dir)
   from_store->each(from_store, &copy_files, &cfa);
 
   if (close_dir)
-    from_store->close(from_store);
+    store_deref(from_store);
 
   return store;
 }

data/ext/search.c

@@ -135,30 +135,34 @@ Hit *hit_pq_pop(PriorityQueue *pq)
 
 inline void hit_pq_up(PriorityQueue *pq)
 {
-	int i,j;
-	i = pq->count;
-	j = i >> 1;
   Hit **heap = (Hit **)pq->heap;
-  Hit *node = heap[i];
+  Hit *node;
+  int i = pq->count;
+  int j = i >> 1;
+  node = heap[i];
 
-	while ((j > 0) && hit_lt(node, heap[j])) {
-		heap[i] = heap[j];
-		i = j;
-		j = j >> 1;
-	}
+  while ((j > 0) && hit_lt(node, heap[j])) {
+    heap[i] = heap[j];
+    i = j;
+    j = j >> 1;
+  }
   heap[i] = node;
 }
 
-
-void hit_pq_push(PriorityQueue *pq, void *elem) 
+void hit_pq_insert(PriorityQueue *pq, Hit *hit) 
 {
-  pq->count++;
-  pq->heap[pq->count] = elem;
-  hit_pq_up(pq);
+  if (pq->count < pq->size) {
+    Hit *new_hit = ALLOC(Hit);
+    memcpy(new_hit, hit, sizeof(Hit));
+    pq->count++;
+    pq->heap[pq->count] = new_hit;
+    hit_pq_up(pq);
+  } else if (pq->count > 0 && hit_lt((Hit *)pq->heap[1], hit)) {
+    memcpy(pq->heap[1], hit, sizeof(Hit));
+    hit_pq_down(pq);
+  }
 }
 
-
-
 /***************************************************************************
  *
  * TopDocs
@@ -174,9 +178,8 @@ TopDocs *td_create(int total_hits, int size, Hit **hits)
   return td;
 }
 
-void td_destroy(void *p)
+void td_destroy(TopDocs *td)
 {
-  TopDocs *td = (TopDocs *)p;
   int i;
   for (i = 0; i < td->size; i++) {
     free(td->hits[i]);
@@ -226,54 +229,78 @@ void w_normalize(Weight *self, float normalization_factor)
   self->value = self->qweight * self->idf; // idf for document
 }
 
+void w_destroy(Weight *self)
+{
+  q_deref(self->query);
+  free(self);
+}
+
+Weight *w_create(Query *query)
+{
+  Weight *self = ALLOC_AND_ZERO_N(Weight, 1);
+  ref(query);
+  self->query = query;
+
+  self->get_query = &w_get_query;
+  self->get_value = &w_get_value;
+  self->normalize = &w_normalize;
+  self->destroy   = &w_destroy;
+  return self;
+}
+
 /***************************************************************************
  *
  * Query
  *
  ***************************************************************************/
 
-Similarity *q_get_similarity(Query *self, Searcher *searcher)
+Similarity *q_get_similarity_i(Query *self, Searcher *searcher)
 {
   return searcher->get_similarity(searcher);
 }
 
 Query *q_rewrite(Query *self, IndexReader *ir)
 {
+  self->ref_cnt++;
   return self;
 }
 
 Weight *q_weight(Query *self, Searcher *searcher)
 {
-  if (self->weight) {
-    self->weight->destroy(self->weight);
-  }
   Query  *query   = searcher->rewrite(searcher, self);
-  Weight *weight  = query->create_weight(query, searcher);
+  Weight *weight  = query->create_weight_i(query, searcher);
   float   sum     = weight->sum_of_squared_weights(weight);
   Similarity *sim = query->get_similarity(query, searcher);
   float   norm    = sim_query_norm(sim, sum);
+  q_deref(query);
 
   weight->normalize(weight, norm);
   return self->weight = weight;
 }
 
-void q_destroy(Query *self)
+Weight *q_create_weight_unsup(Query *self, Searcher *searcher)
+{
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+  return NULL;
+}
+
+void q_destroy_i(Query *self)
 {
-  if (self->rewritten) {
-    self->rewritten->destroy(self->rewritten);
-    self->rewritten = NULL;
-  }
-  if (self->weight) {
-    self->weight->destroy(self->weight);
-  }
   free(self);
 }
 
-void q_extract_terms(Query *self, Array *terms)
+void q_extract_terms(Query *self, HashSet *terms)
 {
   /* do nothing by default */
 }
 
+void q_deref(Query *self)
+{  
+  if (--self->ref_cnt == 0) {
+    self->destroy_i(self);
+  }
+}
+
 Query *q_create()
 {
   Query *self = ALLOC(Query);
@@ -281,30 +308,92 @@ Query *q_create()
   self->destroy_all = true;
   self->boost = 1.0;
   self->rewrite = &q_rewrite;
-  self->get_similarity = &q_get_similarity;
+  self->get_similarity = &q_get_similarity_i;
   self->extract_terms = &q_extract_terms;
   self->weight = NULL;
-  self->rewritten = NULL;
+  self->ref_cnt = 1;
   return self;
 }
 
+uint q_hash(Query *self)
+{
+  return (self->hash(self) << 4) | self->type;
+}
+
+int q_eq(Query *self, Query *o)
+{
+  return (self == o) || ((self->type == o->type) &&
+                         (self->boost == o->boost) &&
+                         self->eq(self, o));
+}
+
+Query *q_combine(Query **queries, int q_cnt)
+{
+  int i;
+  Query *q, *ret_q;
+  HashSet *uniques =
+    hs_create((hash_ft)&q_hash, (eq_ft)&q_eq, NULL);
+
+  for (i = 0; i < q_cnt; i++) {
+    q = queries[i];
+    if (q->type == BOOLEAN_QUERY) {
+      int j;
+      bool splittable = true;
+      BooleanQuery *bq = (BooleanQuery *)q->data;
+      if (bq->coord_disabled == false) {
+        splittable = false;
+      } else {
+        for (j = 0; j < bq->clause_cnt; j++) {
+          if (bq->clauses[j]->occur != BC_SHOULD) {
+            splittable = false;
+            break;
+          }
+        }
+      }
+      if (splittable) {
+        for (j = 0; j < bq->clause_cnt; j++) {
+          q = bq->clauses[j]->query;
+          hs_add(uniques, q);
+        }
+      } else {
+        hs_add(uniques, q);
+      }
+    } else {
+      hs_add(uniques, q);
+    }
+  }
+  if (uniques->size == 1) {
+    ret_q = (Query *)uniques->elems[0]; 
+    ref(ret_q);
+  } else {
+    ret_q = bq_create(true);
+    for (i = 0; i < uniques->size; i++) {
+      q = (Query *)uniques->elems[i];
+      ref(q);
+      bq_add_query(ret_q, q, BC_SHOULD);
+    }
+  }
+  hs_destroy(uniques);
+
+  return ret_q;
+}
+
 /***************************************************************************
  *
  * Scorer
  *
  ***************************************************************************/
 
-void scorer_destroy(void *p)
+void scorer_destroy_i(Scorer *self)
 {
-  Scorer *scorer = (Scorer *)p;
-  free(scorer->data);
-  free(scorer);
+  free(self->data);
+  free(self);
 }
 
 Scorer *scorer_create(Similarity *similarity)
 {
   Scorer *self = ALLOC(Scorer);
-  self->destroy = &scorer_destroy;
+  self->destroy = &scorer_destroy_i;
   self->data = NULL;
   self->similarity = similarity;
   return self;
@@ -326,43 +415,58 @@ int scorer_doc_cmp(const void *p1, const void *p2)
 {
   return (*(Scorer **)p1)->doc - (*(Scorer **)p2)->doc;
 }
+
 /***************************************************************************
  *
  * Searcher
  *
  ***************************************************************************/
 
-int sea_doc_freq(Searcher *self, Term *term)
+static int s_doc_freq(Searcher *self, Term *term)
 {
   return self->ir->doc_freq(self->ir, term);
 }
 
-int *sea_doc_freqs(Searcher *self, Term **terms, int tcnt)
+static int *s_doc_freqs(Searcher *self, Term **terms, int tcnt)
 {
-  int *freqs = ALLOC_N(int, tcnt);
   int i;
+  int *freqs = ALLOC_N(int, tcnt);
+
   for (i = 0; i < tcnt; i++) {
     freqs[i] = self->ir->doc_freq(self->ir, terms[i]);
   }
   return freqs;
 }
 
-Document *sea_get_doc(Searcher *self, int doc_num)
+static int *ss_doc_freqs(Searcher *self, Term **terms, int tcnt)
+{
+  int i;
+  int *freqs = ALLOC_N(int, tcnt);
+
+  for (i = 0; i < tcnt; i++) {
+    freqs[i] = self->doc_freq(self, terms[i]);
+  }
+
+  return freqs;
+}
+
+
+static Document *s_get_doc(Searcher *self, int doc_num)
 {
   return self->ir->get_doc(self->ir, doc_num);
 }
 
-int sea_max_doc(Searcher *self)
+static int s_max_doc(Searcher *self)
 {
   return self->ir->max_doc(self->ir);
 }
 
-Weight *sea_create_weight(Searcher *self, Query *query)
+static Weight *s_create_weight(Searcher *self, Query *query)
 {
   return q_weight(query, self);
 }
 
-TopDocs *sea_search(Searcher *self, Query *query, int first_doc,
+static TopDocs *s_search(Searcher *self, Query *query, int first_doc,
     int num_docs, Filter *filter, Sort *sort)
 {
   int max_size = first_doc + num_docs;
@@ -370,14 +474,13 @@ TopDocs *sea_search(Searcher *self, Query *query, int first_doc,
   Weight *weight;
   Scorer *scorer;
   Hit **score_docs = NULL;
-  Hit *hit;
+  Hit hit;
   int total_hits = 0;
-  float min_score = 0.0, score;
+  float score;
   BitVector *bits = (filter ? filter->get_bv(filter, self->ir) : NULL);
   Hit *(*hq_pop)(PriorityQueue *pq);
-  void (*hq_down)(PriorityQueue *pq);
-  void (*hq_push)(PriorityQueue *pq, void *elem);
-  void (*hq_destroy)(void *p);
+  void (*hq_insert)(PriorityQueue *pq, Hit *hit);
+  void (*hq_destroy)(PriorityQueue *self);
   PriorityQueue *hq;
 
 
@@ -391,20 +494,19 @@ TopDocs *sea_search(Searcher *self, Query *query, int first_doc,
   scorer = weight->scorer(weight, self->ir);
   if (!scorer) {
     if (bits) bv_destroy(bits);
+    weight->destroy(weight);
     return td_create(0, 0, NULL);
   }
 
   if (sort) {
     hq = fshq_pq_create(max_size, sort, self->ir);
     hq_pop = &fshq_pq_pop;
-    hq_down = &fshq_pq_down;
-    hq_push = &fshq_pq_push;
+    hq_insert = &fshq_pq_insert;
     hq_destroy = &fshq_pq_destroy;
   } else {
     hq = pq_create(max_size, &hit_less_than);
     hq_pop = &hit_pq_pop;
-    hq_down = &hit_pq_down;
-    hq_push = &hit_pq_push;
+    hq_insert = &hit_pq_insert;
     hq_destroy = &pq_destroy;
   }
 
@@ -412,19 +514,11 @@ TopDocs *sea_search(Searcher *self, Query *query, int first_doc,
     if (bits && !bv_get(bits, scorer->doc)) continue;
     total_hits++;
     score = scorer->score(scorer);
-    if (hq->count < max_size) {
-      hit = ALLOC(Hit);
-      hit->doc = scorer->doc; hit->score = score;
-      hq_push(hq, hit);
-      min_score = ((Hit *)pq_top(hq))->score; // maintain min_score
-    } else if (score > min_score) {
-      hit = pq_top(hq);
-      hit->doc = scorer->doc; hit->score = score;
-      hq_down(hq);
-      min_score = ((Hit *)pq_top(hq))->score; // maintain min_score
-    }
+    hit.doc = scorer->doc; hit.score = score;
+    hq_insert(hq, &hit);
   }
   scorer->destroy(scorer);
+  weight->destroy(weight);
 
   if (hq->count > first_doc) {
     if ((hq->count - first_doc) < num_docs) {
@@ -446,14 +540,12 @@ TopDocs *sea_search(Searcher *self, Query *query, int first_doc,
   return td_create(total_hits, num_docs, score_docs);
 }
 
-void sea_search_each(Searcher *self, Query *query, Filter *filter,
-    void (*fn)(Searcher *self, int doc_num, void *arg), void *arg)
+static void s_search_each_w(Searcher *self, Weight *weight, Filter *filter,
+    void (*fn)(Searcher *, int, float, void *), void *arg)
 {
-  Weight *weight;
   Scorer *scorer;
   BitVector *bits = (filter ? filter->get_bv(filter, self->ir) : NULL);
 
-  weight = q_weight(query, self);
   scorer = weight->scorer(weight, self->ir);
   if (!scorer) {
     if (bits) bv_destroy(bits);
@@ -462,34 +554,53 @@ void sea_search_each(Searcher *self, Query *query, Filter *filter,
 
   while (scorer->next(scorer)) {
     if (bits && !bv_get(bits, scorer->doc)) continue;
-    fn(self, scorer->doc, arg);
+    fn(self, scorer->doc, scorer->score(scorer), arg);
   }
   scorer->destroy(scorer);
 }
 
-Query *sea_rewrite(Searcher *self, Query *original)
+static void s_search_each(Searcher *self, Query *query, Filter *filter,
+    void (*fn)(Searcher *, int, float, void *), void *arg)
 {
+  Weight *weight;
+  weight = q_weight(query, self);
+  s_search_each_w(self, weight, filter, fn, arg);
+  weight->destroy(weight);
+}
+
+static Query *s_rewrite(Searcher *self, Query *original)
+{
+  int q_is_destroyed = false;
   Query *query = original;
   Query *rewritten_query = query->rewrite(query, self->ir);
-  while (query != rewritten_query) {
+  while (q_is_destroyed || (query != rewritten_query)) {
     query = rewritten_query;
     rewritten_query = query->rewrite(query, self->ir);
+    q_is_destroyed = (query->ref_cnt <= 1);
+    q_deref(query); /* destroy intermediate queries */
   }
   return query;
 }
 
-Explanation *sea_explain(Searcher *self, Query *query, int doc_num)
+static Explanation *s_explain(Searcher *self, Query *query, int doc_num)
 {
   Weight *weight = q_weight(query, self);
-  return weight->explain(weight, self->ir, doc_num);
+  Explanation *e =  weight->explain(weight, self->ir, doc_num);
+  weight->destroy(weight);
+  return e;
+}
+
+static Explanation *s_explain_w(Searcher *self, Weight *w, int doc_num)
+{
+  return w->explain(w, self->ir, doc_num);
 }
 
-Similarity *sea_get_similarity(Searcher *self)
+static Similarity *s_get_similarity(Searcher *self)
 {
   return self->similarity;
 }
 
-void sea_close(Searcher *self)
+static void s_close(Searcher *self)
 {
   if (self->ir && self->close_ir)
     ir_close(self->ir);
@@ -502,17 +613,436 @@ Searcher *sea_create(IndexReader *ir)
   self->ir = ir;
   self->close_ir = true;
   self->similarity = sim_create_default();
-  self->doc_freq = &sea_doc_freq;
-  self->doc_freqs = &sea_doc_freqs;
-  self->get_doc = &sea_get_doc;
-  self->max_doc = &sea_max_doc;
-  self->create_weight = &sea_create_weight;
-  self->search = &sea_search;
-  self->rewrite = &sea_rewrite;
-  self->explain = &sea_explain;
-  self->get_similarity = &sea_get_similarity;
-  self->close = &sea_close;
+  self->doc_freq = &s_doc_freq;
+  self->doc_freqs = &s_doc_freqs;
+  self->get_doc = &s_get_doc;
+  self->max_doc = &s_max_doc;
+  self->create_weight = &s_create_weight;
+  self->search = &s_search;
+  self->search_each = &s_search_each;
+  self->search_each_w = &s_search_each_w;
+  self->rewrite = &s_rewrite;
+  self->explain = &s_explain;
+  self->explain_w = &s_explain_w;
+  self->get_similarity = &s_get_similarity;
+  self->close = &s_close;
+  return self;
+}
+
+/***************************************************************************
+ *
+ * CachedDFSearcher
+ *
+ ***************************************************************************/
+
+typedef struct CachedDFSearcher {
+  HshTable *df_map;
+  int max_doc;
+} CachedDFSearcher;
+
+static int cdfsea_doc_freq(Searcher *self, Term *term)
+{
+  CachedDFSearcher *cdfsea = (CachedDFSearcher *)self->data;
+  return (int)h_get(cdfsea->df_map, term);
+}
+
+static Document *cdfsea_get_doc(Searcher *self, int doc_num)
+{
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+  return NULL;
+}
+
+static int cdfsea_max_doc(Searcher *self)
+{
+  return ((CachedDFSearcher *)self->data)->max_doc;
+}
+
+static Weight *cdfsea_create_weight(Searcher *self, Query *query)
+{
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+  return NULL;
+}
+
+static TopDocs *cdfsea_search(Searcher *self, Query *query, int first_doc,
+    int num_docs, Filter *filter, Sort *sort)
+{
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+  return NULL;
+}
+
+static void cdfsea_search_each(Searcher *self, Query *query, Filter *filter,
+    void (*fn)(Searcher *, int, float, void *), void *arg)
+{
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+}
+
+static void cdfsea_search_each_w(Searcher *self, Weight *w, Filter *filter,
+    void (*fn)(Searcher *, int, float, void *), void *arg)
+{
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+}
+
+static Query *cdfsea_rewrite(Searcher *self, Query *original)
+{
+  original->ref_cnt++;
+  return original;
+}
+
+static Explanation *cdfsea_explain(Searcher *self, Query *query, int doc_num)
+{
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+  return NULL;
+}
+
+static Explanation *cdfsea_explain_w(Searcher *self, Weight *w, int doc_num)
+{
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+  return NULL;
+}
+
+static Similarity *cdfsea_get_similarity(Searcher *self)
+{
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+  return NULL;
+}
+
+static void cdfsea_close(Searcher *self)
+{
+  CachedDFSearcher *cdfsea = (CachedDFSearcher *)self->data;
+  h_destroy(cdfsea->df_map);
+  free(cdfsea);
+  free(self);
+}
+
+Searcher *cdfsea_create(HshTable *df_map, int max_doc)
+{
+  Searcher *self = ALLOC(Searcher);
+
+  CachedDFSearcher *cdfsea = ALLOC(CachedDFSearcher);
+
+  cdfsea->df_map = df_map;
+  cdfsea->max_doc = max_doc;
+  self->data = cdfsea;
+
+  self->doc_freq = &cdfsea_doc_freq;
+  self->doc_freqs = &ss_doc_freqs;
+  self->get_doc = &cdfsea_get_doc;
+  self->max_doc = &cdfsea_max_doc;
+  self->create_weight = &cdfsea_create_weight;
+  self->search = &cdfsea_search;
+  self->search_each = &cdfsea_search_each;
+  self->search_each_w = &cdfsea_search_each_w;
+  self->rewrite = &cdfsea_rewrite;
+  self->explain = &cdfsea_explain;
+  self->explain_w = &cdfsea_explain_w;
+  self->get_similarity = &cdfsea_get_similarity;
+  self->close = &cdfsea_close;
   return self;
 }
 
+/***************************************************************************
+ *
+ * MultiSearcher
+ *
+ ***************************************************************************/
+
+static inline int msea_get_searcher_index(Searcher *self, int n)
+{
+  MultiSearcher *msea = (MultiSearcher *)self->data;
+  int lo = 0;               /* search starts array */
+  int hi = msea->s_cnt - 1; /* for 1st element < n, return its index */
+  int mid, mid_val;
+
+  while (hi >= lo) {
+    mid = (lo + hi) >> 1;
+    mid_val = msea->starts[mid];
+    if (n < mid_val) {
+      hi = mid - 1;
+    } else if (n > mid_val) {
+      lo = mid + 1;
+    } else {                  /* found a match */
+      while (((mid+1) < msea->s_cnt) && (msea->starts[mid+1] == mid_val)) {
+        mid++;                /* scan to last match */
+      }
+      return mid;
+    }
+  }
+  return hi;
+}
+
+static int msea_doc_freq(Searcher *self, Term *term)
+{
+  int i;
+  int doc_freq = 0;
+  Searcher *s;
+  MultiSearcher *msea = (MultiSearcher *)self->data;
+  for (i = 0; i < msea->s_cnt; i++) {
+    s = msea->searchers[i];
+    doc_freq += s->doc_freq(s, term);
+  }
+
+  return doc_freq;
+}
+
+static Document *msea_get_doc(Searcher *self, int doc_num)
+{
+  MultiSearcher *msea = (MultiSearcher *)self->data;
+  int i = msea_get_searcher_index(self, doc_num);
+  Searcher *s = msea->searchers[i];
+  return s->get_doc(s, doc_num - msea->starts[i]);
+}
+
+static int msea_max_doc(Searcher *self)
+{
+  return ((MultiSearcher *)self->data)->max_doc;
+}
+
+static Weight *msea_create_weight(Searcher *self, Query *query)
+{
+  int i, *dfs;
+  Searcher *cdfsea;
+  Weight *w;
+  HshTable *df_map = h_new((hash_ft)&term_hash, (eq_ft)&term_eq,
+                          (free_ft)NULL, (free_ft)NULL);
+  Query *rq = self->rewrite(self, query);
+  HashSet *terms = term_set_create();
+  rq->extract_terms(rq, terms);
+  dfs = self->doc_freqs(self, (Term **)terms->elems, terms->size);
+
+  for (i = 0; i < terms->size; i++) {
+    h_set(df_map, terms->elems[i], (void *)dfs[i]); 
+  }
+  /* don't destroy the individual terms, only the HashSet */
+  hs_destroy(terms);
+  free(dfs);
+
+  cdfsea = cdfsea_create(df_map, ((MultiSearcher *)self->data)->max_doc);
+
+  w = q_weight(rq, cdfsea);
+  q_deref(rq);
+  cdfsea->close(cdfsea);
+
+  return w;
+}
+
+struct MultiSearchEachArg {
+  int start;
+  void *arg;
+  void (*fn)(Searcher *, int, float, void *);
+};
+
+void msea_search_each_i(Searcher *self, int doc_num, float score, void *arg)
+{
+  struct MultiSearchEachArg *mse_arg = (struct MultiSearchEachArg *)arg;
+
+  mse_arg->fn(self, doc_num + mse_arg->start, score, mse_arg->arg);
+}
+
+static void msea_search_each_w(Searcher *self, Weight *w, Filter *filter,
+    void (*fn)(Searcher *, int, float, void *), void *arg)
+{
+  int i;
+  struct MultiSearchEachArg mse_arg;
+  MultiSearcher *msea = (MultiSearcher *)self->data;
+  Searcher *s;
+
+  mse_arg.fn = fn;
+  mse_arg.arg = arg;
+  for (i = 0; i < msea->s_cnt; i++) {
+    s = msea->searchers[i];
+    mse_arg.start = msea->starts[i];
+    s->search_each_w(s, w, filter, &msea_search_each_i, &mse_arg);
+  }
+}
 
+static void msea_search_each(Searcher *self, Query *query, Filter *filter,
+    void (*fn)(Searcher *, int, float, void *), void *arg)
+{
+  Weight *w = q_weight(query, self);
+  msea_search_each_w(self, w, filter, fn, arg);
+  w->destroy(w);
+}
+
+struct MultiSearchArg {
+  int total_hits, max_size;
+  float min_score;
+  PriorityQueue *hq;
+  void (*hq_insert)(PriorityQueue *pq, Hit *hit);
+};
+
+void msea_search_i(Searcher *self, int doc_num, float score, void *arg)
+{
+  struct MultiSearchArg *ms_arg = (struct MultiSearchArg *)arg;
+  Hit hit;
+
+  ms_arg->total_hits++;
+  hit.doc = doc_num;
+  hit.score = score;
+  ms_arg->hq_insert(ms_arg->hq, &hit);
+}
+
+static TopDocs *msea_search(Searcher *self, Query *query, int first_doc,
+    int num_docs, Filter *filter, Sort *sort)
+{
+  int max_size = first_doc + num_docs;
+  int i;
+  Weight *weight;
+  Hit **score_docs = NULL;
+  BitVector *bits = (filter ? filter->get_bv(filter, self->ir) : NULL);
+  Hit *(*hq_pop)(PriorityQueue *pq);
+  void (*hq_insert)(PriorityQueue *pq, Hit *hit);
+  void (*hq_destroy)(PriorityQueue *self);
+  PriorityQueue *hq;
+  struct MultiSearchArg ms_arg;
+
+
+  if (num_docs <= 0)
+    RAISE(ARG_ERROR, NUM_DOCS_ARG_ERROR_MSG);
+
+  if (first_doc < 0)
+    RAISE(ARG_ERROR, FIRST_DOC_ARG_ERROR_MSG);
+
+  weight = q_weight(query, self);
+  if (sort) {
+    hq = fshq_pq_create(max_size, sort, self->ir);
+    hq_pop = &fshq_pq_pop;
+    hq_insert = &fshq_pq_insert;
+    hq_destroy = &fshq_pq_destroy;
+  } else {
+    hq = pq_create(max_size, &hit_less_than);
+    hq_pop = &hit_pq_pop;
+    hq_insert = &hit_pq_insert;
+    hq_destroy = &pq_destroy;
+  }
+
+
+  ms_arg.hq = hq;
+  ms_arg.total_hits = 0;
+  ms_arg.max_size = max_size;
+  ms_arg.min_score = 0.0;
+  ms_arg.hq_insert = hq_insert;
+
+  msea_search_each_w(self, weight, filter, msea_search_i, &ms_arg);
+
+  weight->destroy(weight);
+
+  if (hq->count > first_doc) {
+    if ((hq->count - first_doc) < num_docs) {
+      num_docs = hq->count - first_doc;
+    }
+    score_docs = ALLOC_N(Hit *, num_docs);
+    for (i = num_docs - 1; i >= 0; i--) {
+      score_docs[i] = hq_pop(hq);
+      //hit = score_docs[i] = pq_pop(hq);
+      //printf("hit = %d-->%f\n", hit->doc, hit->score);
+    }
+  } else {
+    num_docs = 0;
+  }
+  pq_clear(hq);
+  hq_destroy(hq);
+
+  if (bits) bv_destroy(bits);
+  return td_create(ms_arg.total_hits, num_docs, score_docs);
+}
+
+static Query *msea_rewrite(Searcher *self, Query *original)
+{
+  int i;
+  Searcher *s;
+  MultiSearcher *msea = (MultiSearcher *)self->data;
+  Query **queries = ALLOC_N(Query *, msea->s_cnt), *rewritten;
+
+  for (i = 0; i < msea->s_cnt; i++) {
+    s = msea->searchers[i];
+    queries[i] = s->rewrite(s, original);
+  }
+  rewritten = q_combine(queries, msea->s_cnt);
+
+  for (i = 0; i < msea->s_cnt; i++) {
+    q_deref(queries[i]);
+  }
+  free(queries);
+  return rewritten;
+}
+
+static Explanation *msea_explain(Searcher *self, Query *query, int doc_num)
+{
+  MultiSearcher *msea = (MultiSearcher *)self->data;
+  int i = msea_get_searcher_index(self, doc_num);
+  Weight *w = q_weight(query, self);
+  Searcher *s = msea->searchers[i];
+  Explanation *e = s->explain_w(s, w, doc_num - msea->starts[i]);
+  w->destroy(w);
+  return e;
+}
+
+static Explanation *msea_explain_w(Searcher *self, Weight *w, int doc_num)
+{
+  MultiSearcher *msea = (MultiSearcher *)self->data;
+  int i = msea_get_searcher_index(self, doc_num);
+  Searcher *s = msea->searchers[i];
+  Explanation *e = s->explain_w(s, w, doc_num - msea->starts[i]);
+  return e;
+}
+
+static Similarity *msea_get_similarity(Searcher *self)
+{
+  return self->similarity;
+}
+
+static void msea_close(Searcher *self)
+{
+  int i;
+  Searcher *s;
+  MultiSearcher *msea = (MultiSearcher *)self->data;
+  if (msea->close_subs) {
+    for (i = 0; i < msea->s_cnt; i++) {
+      s = msea->searchers[i];
+      s->close(s);
+    }
+    free(msea->searchers);
+  }
+  free(msea->starts);
+  free(msea);
+  free(self);
+}
+
+Searcher *msea_create(Searcher **searchers, int s_cnt, bool close_subs)
+{
+  int i, max_doc = 0, *starts;
+  Searcher *self = ALLOC(Searcher);
+
+  MultiSearcher *msea = ALLOC(MultiSearcher);
+
+  starts = ALLOC_N(int, s_cnt + 1);
+  for (i = 0; i < s_cnt; i++) {
+    starts[i] = max_doc;
+    max_doc += searchers[i]->max_doc(searchers[i]);
+  }
+  starts[i] = max_doc;
+
+  msea->s_cnt = s_cnt;
+  msea->searchers = searchers;
+  msea->starts = starts;
+  msea->max_doc = max_doc;
+  msea->close_subs = close_subs;
+  self->data = msea;
+
+  self->ir = (IndexReader *)NULL;
+  self->similarity = sim_create_default();
+  self->doc_freq = &msea_doc_freq;
+  self->doc_freqs = &ss_doc_freqs;
+  self->get_doc = &msea_get_doc;
+  self->max_doc = &msea_max_doc;
+  self->create_weight = &msea_create_weight;
+  self->search = &msea_search;
+  self->search_each = &msea_search_each;
+  self->search_each_w = &msea_search_each_w;
+  self->rewrite = &msea_rewrite;
+  self->explain = &msea_explain;
+  self->explain_w = &msea_explain_w;
+  self->get_similarity = &msea_get_similarity;
+  self->close = &msea_close;
+  return self;
+}