ferret 0.9.6 → 0.10.0

data/ext/search.c

@@ -1,8 +1,6 @@
 #include <string.h>
 #include "search.h"
-
-static char * const NUM_DOCS_ARG_ERROR_MSG = "num_docs must be > 0 to run a search";
-static char * const FIRST_DOC_ARG_ERROR_MSG = "first_doc must be >= 0 to run a search";
+#include "array.h"
 
 /***************************************************************************
  *
@@ -10,67 +8,65 @@ static char * const FIRST_DOC_ARG_ERROR_MSG = "first_doc must be >= 0 to run a s
  *
  ***************************************************************************/
 
-Explanation *expl_create(float value, char *description)
+Explanation *expl_new(float value, const char *description, ...)
 {
-  Explanation *self = ALLOC(Explanation);
-  self->value = value;
-  self->description = description;
-  self->dcnt = 0;
-  self->dcapa = EXPLANATION_DETAILS_START_SIZE;
-  self->details = ALLOC_N(Explanation *, EXPLANATION_DETAILS_START_SIZE);
-  return self;
+    Explanation *expl = ALLOC(Explanation);
+
+    va_list args;
+    va_start(args, description);
+    expl->description = vstrfmt(description, args);
+    va_end(args);
+
+    expl->value = value;
+    expl->details = ary_new_type_capa(Explanation *,
+                                      EXPLANATION_DETAILS_START_SIZE);
+    return expl;
 }
 
-void expl_destoy(void *p)
+void expl_destroy(Explanation *expl)
 {
-  Explanation *expl = (Explanation *)p;
-  int i;
-  for (i = 0; i < expl->dcnt; i++) {
-    expl_destoy(expl->details[i]);
-  }
-  free(expl->details);
-  free(expl->description);
-  free(expl);
+    ary_destroy((void **)expl->details, (free_ft)expl_destroy);
+    free(expl->description);
+    free(expl);
 }
 
-Explanation *expl_add_detail(Explanation *self, Explanation *detail)
+Explanation *expl_add_detail(Explanation *expl, Explanation *detail)
 {
-  if (self->dcnt >= self->dcapa) {
-    self->dcapa *= 2;
-    REALLOC_N(self->details, Explanation *, self->dcapa);
-  }
-  self->details[self->dcnt] = detail;
-  self->dcnt++;
-  return self;
+    ary_push(expl->details, detail);
+    return expl;
 }
 
-char *expl_to_s(Explanation *self, int depth)
+char *expl_to_s_depth(Explanation *expl, int depth)
 {
-  int i;
-  char *buffer = ALLOC_N(char, depth * 2 + 1);
-  memset(buffer, ' ', sizeof(char) * depth * 2);
-  buffer[depth*2] = 0;
+    int i;
+    char *buffer = ALLOC_N(char, depth * 2 + 1);
+    const int num_details = ary_size(expl->details);
+
+    memset(buffer, ' ', sizeof(char) * depth * 2);
+    buffer[depth*2] = 0;
 
-  buffer = estrcat(buffer, strfmt("%f = %s\n", self->value, self->description));
-  for (i = 0; i < self->dcnt; i++) {
-    buffer = estrcat(buffer, expl_to_s(self->details[i], depth + 1));
-  }
+    buffer = estrcat(buffer, strfmt("%f = %s\n", expl->value, expl->description));
+    for (i = 0; i < num_details; i++) {
+        buffer = estrcat(buffer, expl_to_s_depth(expl->details[i], depth + 1));
+    }
 
-  return buffer;
+    return buffer;
 }
 
-char *expl_to_html(Explanation *self)
+char *expl_to_html(Explanation *expl)
 {
-  int i;
-  char *buffer;
-  buffer = strfmt("<ul>\n<li>%f = %s</li>\n", self->value, self->description);
+    int i;
+    char *buffer;
+    const int num_details = ary_size(expl->details);
+
+    buffer = strfmt("<ul>\n<li>%f = %s</li>\n", expl->value, expl->description);
 
-  for (i = 0; i < self->dcnt; i++) {
-    estrcat(buffer, expl_to_html(self->details[i]));
-  }
+    for (i = 0; i < num_details; i++) {
+        estrcat(buffer, expl_to_html(expl->details[i]));
+    }
 
-  REALLOC_N(buffer, char, strlen(buffer) + 10);
-  return strcat(buffer, "</ul>\n");
+    REALLOC_N(buffer, char, strlen(buffer) + 10);
+    return strcat(buffer, "</ul>\n");
 }
 
 /***************************************************************************
@@ -79,88 +75,104 @@ char *expl_to_html(Explanation *self)
  *
  ***************************************************************************/
 
-bool hit_less_than(void *hit1, void *hit2)
+static bool hit_less_than(const Hit *hit1, const Hit *hit2)
 {
-  if (((Hit *)hit1)->score == ((Hit *)hit2)->score) {
-    return ((Hit *)hit1)->doc > ((Hit *)hit2)->doc;
-  } else {
-    return ((Hit *)hit1)->score < ((Hit *)hit2)->score;
-  }
+    if (hit1->score == hit2->score) {
+        return hit1->doc > hit2->doc;
+    }
+    else {
+        return hit1->score < hit1->score;
+    }
 }
 
-inline bool hit_lt(Hit *hit1, Hit *hit2)
+static bool hit_lt(Hit *hit1, Hit *hit2)
 {
-  if (hit1->score == hit2->score) {
-    return hit1->doc > hit2->doc;
-  } else {
-    return hit1->score < hit2->score;
-  }
+    if (hit1->score == hit2->score) {
+        return hit1->doc > hit2->doc;
+    }
+    else {
+        return hit1->score < hit2->score;
+    }
 }
 
-void hit_pq_down(PriorityQueue *pq)
+static void hit_pq_down(PriorityQueue *pq)
 {
-  register int i = 1;
-  register int j = 2; //i << 1;
-  register int k = 3; //j + 1;
-  Hit **heap = (Hit **)pq->heap;
-  Hit *node = heap[i]; // save top node
+    register int i = 1;
+    register int j = 2;     /* i << 1; */
+    register int k = 3;     /* j + 1;  */
+    Hit **heap = (Hit **)pq->heap;
+    Hit *node = heap[i];    /* save top node */
 
-  if ((k <= pq->count) && hit_lt(heap[k], heap[j]))
-    j = k;
+    if ((k <= pq->size) && hit_lt(heap[k], heap[j])) {
+        j = k;
+    }
 
-  while ((j <= pq->count) && hit_lt(heap[j], node)) {
-    heap[i] = heap[j];      // shift up child
-    i = j;
-    j = i << 1;
-    k = j + 1;
-    if ((k <= pq->count) && hit_lt(heap[k], heap[j]))
-      j = k;
-  }
-  heap[i] = node;
+    while ((j <= pq->size) && hit_lt(heap[j], node)) {
+        heap[i] = heap[j];  /* shift up child */
+        i = j;
+        j = i << 1;
+        k = j + 1;
+        if ((k <= pq->size) && hit_lt(heap[k], heap[j])) {
+            j = k;
+        }
+    }
+    heap[i] = node;
 }
 
-Hit *hit_pq_pop(PriorityQueue *pq)
+static Hit *hit_pq_pop(PriorityQueue *pq)
 {
-  if (pq->count > 0) {
-    Hit *result = (Hit *)pq->heap[1];  // save first value
-    pq->heap[1] = pq->heap[pq->count]; // move last to first
-    pq->heap[pq->count] = NULL;
-    pq->count--;
-    hit_pq_down(pq);                   // adjust heap
-    return result;
-  } else {
-    return NULL;
-  }
+    if (pq->size > 0) {
+        Hit *result = (Hit *)pq->heap[1]; /* save first value */
+        pq->heap[1] = pq->heap[pq->size]; /* move last to first */
+        pq->heap[pq->size] = NULL;
+        pq->size--;
+        hit_pq_down(pq);                  /* adjust heap */
+        return result;
+    }
+    else {
+        return NULL;
+    }
 }
 
-inline void hit_pq_up(PriorityQueue *pq)
+static void hit_pq_up(PriorityQueue *pq)
 {
-  Hit **heap = (Hit **)pq->heap;
-  Hit *node;
-  int i = pq->count;
-  int j = i >> 1;
-  node = heap[i];
+    Hit **heap = (Hit **)pq->heap;
+    Hit *node;
+    int i = pq->size;
+    int j = i >> 1;
+    node = heap[i];
+
+    while ((j > 0) && hit_lt(node, heap[j])) {
+        heap[i] = heap[j];
+        i = j;
+        j = j >> 1;
+    }
+    heap[i] = node;
+}
 
-  while ((j > 0) && hit_lt(node, heap[j])) {
-    heap[i] = heap[j];
-    i = j;
-    j = j >> 1;
-  }
-  heap[i] = node;
+static void hit_pq_insert(PriorityQueue *pq, Hit *hit) 
+{
+    if (pq->size < pq->capa) {
+        Hit *new_hit = ALLOC(Hit);
+        memcpy(new_hit, hit, sizeof(Hit));
+        pq->size++;
+        if (pq->size >= pq->mem_capa) {
+            pq->mem_capa <<= 1;
+            REALLOC_N(pq->heap, void *, pq->mem_capa);
+        }
+        pq->heap[pq->size] = new_hit;
+        hit_pq_up(pq);
+    }
+    else if (pq->size > 0 && hit_lt((Hit *)pq->heap[1], hit)) {
+        memcpy(pq->heap[1], hit, sizeof(Hit));
+        hit_pq_down(pq);
+    }
 }
 
-void hit_pq_insert(PriorityQueue *pq, Hit *hit) 
+static void hit_pq_multi_insert(PriorityQueue *pq, Hit *hit) 
 {
-  if (pq->count < pq->size) {
-    Hit *new_hit = ALLOC(Hit);
-    memcpy(new_hit, hit, sizeof(Hit));
-    pq->count++;
-    pq->heap[pq->count] = new_hit;
-    hit_pq_up(pq);
-  } else if (pq->count > 0 && hit_lt((Hit *)pq->heap[1], hit)) {
-    memcpy(pq->heap[1], hit, sizeof(Hit));
-    hit_pq_down(pq);
-  }
+    hit_pq_insert(pq, hit);
+    free(hit);
 }
 
 /***************************************************************************
@@ -169,35 +181,38 @@ void hit_pq_insert(PriorityQueue *pq, Hit *hit)
  *
  ***************************************************************************/
 
-TopDocs *td_create(int total_hits, int size, Hit **hits)
+TopDocs *td_new(int total_hits, int size, Hit **hits, float max_score)
 {
-  TopDocs *td = ALLOC(TopDocs);
-  td->total_hits = total_hits;
-  td->size = size;
-  td->hits = hits;
-  return td;
+    TopDocs *td = ALLOC(TopDocs);
+    td->total_hits = total_hits;
+    td->size = size;
+    td->hits = hits;
+    td->max_score = max_score;
+    return td;
 }
 
 void td_destroy(TopDocs *td)
 {
-  int i;
-  for (i = 0; i < td->size; i++) {
-    free(td->hits[i]);
-  }
-  free(td->hits);
-  free(td);
+    int i;
+
+    for (i = 0; i < td->size; i++) {
+        free(td->hits[i]);
+    }
+    free(td->hits);
+    free(td);
 }
 
 char *td_to_s(TopDocs *td)
 {
-  int i;
-  Hit *hit;
-  char *buffer = strfmt("%d hits sorted by <score, doc_num>\n", td->total_hits);
-  for (i = 0; i < td->size; i++) {
-    hit = td->hits[i];
-    estrcat(buffer, strfmt("\t%d:%f\n", hit->doc, hit->score));
-  }
-  return buffer;
+    int i;
+    Hit *hit;
+    char *buffer = strfmt("%d hits sorted by <score, doc_num>\n",
+                          td->total_hits);
+    for (i = 0; i < td->size; i++) {
+        hit = td->hits[i];
+        estrcat(buffer, strfmt("\t%d:%f\n", hit->doc, hit->score));
+    }
+    return buffer;
 }
 
 /***************************************************************************
@@ -208,44 +223,50 @@ char *td_to_s(TopDocs *td)
 
 Query *w_get_query(Weight *self)
 {
-  return self->query;
+    return self->query;
 }
 
 float w_get_value(Weight *self)
 {
-  return self->value;
+    return self->value;
 }
 
 float w_sum_of_squared_weights(Weight *self)
 {
-  self->qweight = self->idf * self->query->boost;
-  return self->qweight * self->qweight; // square it
+    self->qweight = self->idf * self->query->boost;
+    return self->qweight * self->qweight;   /* square it */
 }
 
 void w_normalize(Weight *self, float normalization_factor)
 {
-  self->qnorm = normalization_factor;
-  self->qweight *= normalization_factor; // normalize query weight
-  self->value = self->qweight * self->idf; // idf for document
+    self->qnorm = normalization_factor;
+    self->qweight *= normalization_factor;  /* normalize query weight */
+    self->value = self->qweight * self->idf;/* idf for document */
 }
 
 void w_destroy(Weight *self)
 {
-  q_deref(self->query);
-  free(self);
+    q_deref(self->query);
+    free(self);
 }
 
-Weight *w_create(Query *query)
+Weight *w_create(size_t size, Query *query)
 {
-  Weight *self = ALLOC_AND_ZERO_N(Weight, 1);
-  ref(query);
-  self->query = query;
-
-  self->get_query = &w_get_query;
-  self->get_value = &w_get_value;
-  self->normalize = &w_normalize;
-  self->destroy   = &w_destroy;
-  return self;
+    Weight *self                    = (Weight *)ecalloc(size);
+#ifdef DEBUG
+    if (size < sizeof(Weight)) {
+        RAISE(ERROR, "size of weight <%d> should be at least <%d>",
+              (int)size, (int)sizeof(Weight));
+    }
+#endif
+    REF(query);
+    self->query                     = query;
+    self->get_query                 = &w_get_query;
+    self->get_value                 = &w_get_value;
+    self->normalize                 = &w_normalize;
+    self->destroy                   = &w_destroy;
+    self->sum_of_squared_weights    = &w_sum_of_squared_weights;
+    return self;
 }
 
 /***************************************************************************
@@ -254,128 +275,181 @@ Weight *w_create(Query *query)
  *
  ***************************************************************************/
 
-Similarity *q_get_similarity_i(Query *self, Searcher *searcher)
-{
-  return searcher->get_similarity(searcher);
-}
+static const char *QUERY_NAMES[] = {
+    "TermQuery",
+    "MultiTermQuery",
+    "BooleanQuery",
+    "PhraseQuery",
+    "MultiPhraseQuery",
+    "ConstantScoreQuery",
+    "FilteredQuery",
+    "MatchAllQuery",
+    "RangeQuery",
+    "WildCardQuery",
+    "FuzzyQuery",
+    "PrefixQuery",
+    "SpanTermQuery",
+    "SpanFirstQuery",
+    "SpanOrQuery",
+    "SpanNotQuery",
+    "SpanNearQuery"
+};
 
-Query *q_rewrite(Query *self, IndexReader *ir)
-{
-  self->ref_cnt++;
-  return self;
+static const char *UNKNOWN_QUERY_NAME = "UnkownQuery";
+    
+const char *q_get_query_name(enum QUERY_TYPE type) {
+    if (type >= NELEMS(QUERY_NAMES)) {
+        return UNKNOWN_QUERY_NAME;
+    }
+    else {
+        return QUERY_NAMES[type];
+    }
 }
 
-Weight *q_weight(Query *self, Searcher *searcher)
+static Query *q_rewrite(Query *self, IndexReader *ir)
 {
-  Query  *query   = searcher->rewrite(searcher, self);
-  Weight *weight  = query->create_weight_i(query, searcher);
-  float   sum     = weight->sum_of_squared_weights(weight);
-  Similarity *sim = query->get_similarity(query, searcher);
-  float   norm    = sim_query_norm(sim, sum);
-  q_deref(query);
-
-  weight->normalize(weight, norm);
-  return self->weight = weight;
+    (void)ir;
+    self->ref_cnt++;
+    return self;
 }
 
-Weight *q_create_weight_unsup(Query *self, Searcher *searcher)
+static void q_extract_terms(Query *self, HashSet *terms)
 {
-  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
-  return NULL;
+    /* do nothing by default */
+    (void)self;
+    (void)terms;
 }
 
-void q_destroy_i(Query *self)
+Similarity *q_get_similarity_i(Query *self, Searcher *searcher)
 {
-  free(self);
+    (void)self;
+    return searcher->get_similarity(searcher);
 }
 
-void q_extract_terms(Query *self, HashSet *terms)
+void q_destroy_i(Query *self)
 {
-  /* do nothing by default */
+    free(self);
 }
 
 void q_deref(Query *self)
 {  
-  if (--self->ref_cnt == 0) {
-    self->destroy_i(self);
-  }
+    if (--(self->ref_cnt) == 0) {
+        self->destroy_i(self);
+    }
 }
 
-Query *q_create()
+Weight *q_create_weight_unsup(Query *self, Searcher *searcher)
 {
-  Query *self = ALLOC(Query);
-  ZEROSET(self, Query, 1);
-  self->destroy_all = true;
-  self->boost = 1.0;
-  self->rewrite = &q_rewrite;
-  self->get_similarity = &q_get_similarity_i;
-  self->extract_terms = &q_extract_terms;
-  self->weight = NULL;
-  self->ref_cnt = 1;
-  return self;
+    (void)self;
+    (void)searcher;
+    RAISE(UNSUPPORTED_ERROR,
+          "Create weight is unsupported for this type of query");
+    return NULL;
 }
 
-uint q_hash(Query *self)
+Weight *q_weight(Query *self, Searcher *searcher)
 {
-  return (self->hash(self) << 4) | self->type;
-}
+    Query      *query   = searcher->rewrite(searcher, self);
+    Weight     *weight  = query->create_weight_i(query, searcher);
+    float       sum     = weight->sum_of_squared_weights(weight);
+    Similarity *sim     = query->get_similarity(query, searcher);
+    float       norm    = sim_query_norm(sim, sum);
+    q_deref(query);
 
-int q_eq(Query *self, Query *o)
-{
-  return (self == o) || ((self->type == o->type) &&
-                         (self->boost == o->boost) &&
-                         self->eq(self, o));
+    weight->normalize(weight, norm);
+    return self->weight = weight;
 }
 
+#define BQ(query) ((BooleanQuery *)(query))
 Query *q_combine(Query **queries, int q_cnt)
 {
-  int i;
-  Query *q, *ret_q;
-  HashSet *uniques =
-    hs_create((hash_ft)&q_hash, (eq_ft)&q_eq, NULL);
-
-  for (i = 0; i < q_cnt; i++) {
-    q = queries[i];
-    if (q->type == BOOLEAN_QUERY) {
-      int j;
-      bool splittable = true;
-      BooleanQuery *bq = (BooleanQuery *)q->data;
-      if (bq->coord_disabled == false) {
-        splittable = false;
-      } else {
-        for (j = 0; j < bq->clause_cnt; j++) {
-          if (bq->clauses[j]->occur != BC_SHOULD) {
-            splittable = false;
-            break;
-          }
+    int i;
+    Query *q, *ret_q;
+    HashSet *uniques = hs_new((hash_ft)&q_hash, (eq_ft)&q_eq, NULL);
+
+    for (i = 0; i < q_cnt; i++) {
+        q = queries[i];
+        if (q->type == BOOLEAN_QUERY) {
+            int j;
+            bool splittable = true;
+            if (BQ(q)->coord_disabled == false) {
+                splittable = false;
+            }
+            else {
+                for (j = 0; j < BQ(q)->clause_cnt; j++) {
+                    if (BQ(q)->clauses[j]->occur != BC_SHOULD) {
+                        splittable = false;
+                        break;
+                    }
+                }
+            }
+            if (splittable) {
+                for (j = 0; j < BQ(q)->clause_cnt; j++) {
+                    Query *sub_q = BQ(q)->clauses[j]->query;
+                    hs_add(uniques, sub_q);
+                }
+            }
+            else {
+                hs_add(uniques, q);
+            }
         }
-      }
-      if (splittable) {
-        for (j = 0; j < bq->clause_cnt; j++) {
-          q = bq->clauses[j]->query;
-          hs_add(uniques, q);
+        else {
+            hs_add(uniques, q);
         }
-      } else {
-        hs_add(uniques, q);
-      }
-    } else {
-      hs_add(uniques, q);
     }
-  }
-  if (uniques->size == 1) {
-    ret_q = (Query *)uniques->elems[0]; 
-    ref(ret_q);
-  } else {
-    ret_q = bq_create(true);
-    for (i = 0; i < uniques->size; i++) {
-      q = (Query *)uniques->elems[i];
-      ref(q);
-      bq_add_query(ret_q, q, BC_SHOULD);
+    if (uniques->size == 1) {
+        ret_q = (Query *)uniques->elems[0]; 
+        REF(ret_q);
+    }
+    else {
+        ret_q = bq_new(true);
+        for (i = 0; i < uniques->size; i++) {
+            q = (Query *)uniques->elems[i];
+            bq_add_query(ret_q, q, BC_SHOULD);
+        }
     }
-  }
-  hs_destroy(uniques);
+    hs_destroy(uniques);
 
-  return ret_q;
+    return ret_q;
+}
+
+ulong q_hash(Query *self)
+{
+    return (self->hash(self) << 5) | self->type;
+}
+
+int q_eq(Query *self, Query *o)
+{
+    return (self == o)
+        || ((self->type == o->type)
+            && (self->boost == o->boost)
+            && self->eq(self, o));
+}
+
+static MatchVector *q_get_matchv_i(Query *self, MatchVector *mv, TermVector *tv)
+{
+    /* be default we don't add any matches */
+    (void)self; (void)tv;
+    return mv;
+}
+
+Query *q_create(size_t size)
+{
+    Query *self = (Query *)ecalloc(size);
+#ifdef DEBUG
+    if (size < sizeof(Query)) {
+        RAISE(ERROR, "Size of a query <%d> should never be smaller than the "
+              "size of a Query struct <%d>", (int)size, (int)sizeof(Query));
+    }
+#endif
+    self->boost             = 1.0;
+    self->rewrite           = &q_rewrite;
+    self->get_similarity    = &q_get_similarity_i;
+    self->extract_terms     = &q_extract_terms;
+    self->get_matchv_i      = &q_get_matchv_i;
+    self->weight            = NULL;
+    self->ref_cnt           = 1;
+    return self;
 }
 
 /***************************************************************************
@@ -384,36 +458,154 @@ Query *q_combine(Query **queries, int q_cnt)
  *
  ***************************************************************************/
 
-void scorer_destroy_i(Scorer *self)
+void scorer_destroy_i(Scorer *scorer)
 {
-  free(self->data);
-  free(self);
+    free(scorer);
 }
 
-Scorer *scorer_create(Similarity *similarity)
+Scorer *scorer_create(size_t size, Similarity *similarity)
 {
-  Scorer *self = ALLOC(Scorer);
-  self->destroy = &scorer_destroy_i;
-  self->data = NULL;
-  self->similarity = similarity;
-  return self;
+    Scorer *self        = (Scorer *)ecalloc(size);
+#ifdef DEBUG
+    if (size < sizeof(Scorer)) {
+        RAISE(ERROR, "size of scorer <%d> should be at least <%d>",
+              (int)size, (int)sizeof(Scorer));
+    }
+#endif
+    self->destroy       = &scorer_destroy_i;
+    self->similarity    = similarity;
+    return self;
 }
 
 bool scorer_less_than(void *p1, void *p2)
 {
-  Scorer *s1 = (Scorer *)p1;
-  Scorer *s2 = (Scorer *)p2;
-  return s1->score(s1) < s2->score(s2);
+    Scorer *s1 = (Scorer *)p1;
+    Scorer *s2 = (Scorer *)p2;
+    return s1->score(s1) < s2->score(s2);
 }
 
-bool scorer_doc_less_than(void *p1, void *p2)
+bool scorer_doc_less_than(const Scorer *s1, const Scorer *s2)
 {
-  return ((Scorer *)p1)->doc < ((Scorer *)p2)->doc;
+    return s1->doc < s2->doc;
 }
 
 int scorer_doc_cmp(const void *p1, const void *p2)
 {
-  return (*(Scorer **)p1)->doc - (*(Scorer **)p2)->doc;
+    return (*(Scorer **)p1)->doc - (*(Scorer **)p2)->doc;
+}
+
+/***************************************************************************
+ *
+ * Highlighter
+ *
+ ***************************************************************************/
+
+/* ** MatchRange ** */
+static int match_range_cmp(const void *p1, const void *p2)
+{
+    int diff = ((MatchRange *)p1)->start - ((MatchRange *)p2)->start;
+    if (diff != 0) {
+        return diff;
+    }
+    else {
+        return ((MatchRange *)p2)->end - ((MatchRange *)p1)->end;
+    }
+}
+
+
+
+/* ** MatchVector ** */
+MatchVector *matchv_new()
+{
+    MatchVector *matchv = ALLOC(MatchVector);
+
+    matchv->size = 0;
+    matchv->capa = MATCH_VECTOR_INIT_CAPA;
+    matchv->matches = ALLOC_N(MatchRange, MATCH_VECTOR_INIT_CAPA);
+
+    return matchv;
+}
+
+MatchVector *matchv_add(MatchVector *self, int start, int end)
+{
+    if (self->size >= self->capa) {
+        self->capa <<= 1;
+        REALLOC_N(self->matches, MatchRange, self->capa);
+    }
+    self->matches[self->size].start = start;
+    self->matches[self->size].end = end;
+    self->matches[self->size++].score = 1.0;
+    return self;
+}
+
+MatchVector *matchv_sort(MatchVector *self)
+{
+    qsort(self->matches, self->size, sizeof(MatchRange), &match_range_cmp);
+    return self;
+}
+
+MatchVector *matchv_compact(MatchVector *self)
+{
+    int left, right;
+    matchv_sort(self);
+    for (right = left = 0; right < self->size; right++) {
+        /* Note the end + 1. This compacts a range 3:5 and 6:8 inleft 3:8 */
+        if (self->matches[right].start > self->matches[left].end + 1) {
+            left++;
+            self->matches[left].start = self->matches[right].start;
+            self->matches[left].end = self->matches[right].end;
+            self->matches[left].score = self->matches[right].score;
+        }
+        else if (self->matches[right].end > self->matches[left].end) {
+            self->matches[left].end = self->matches[right].end;
+        }
+        else {
+            self->matches[left].score += self->matches[right].score;
+        }
+    }
+    self->size = left + 1;
+    return self;
+}
+
+MatchVector *matchv_compact_with_breaks(MatchVector *self)
+{
+    int left, right;
+    matchv_sort(self);
+    for (right = left = 0; right < self->size; right++) {
+        /* Note: no end + 1. Unlike above won't compact ranges 3:5 and 6:8 */
+        if (self->matches[right].start > self->matches[left].end) {
+            left++;
+            self->matches[left].start = self->matches[right].start;
+            self->matches[left].end = self->matches[right].end;
+            self->matches[left].score = self->matches[right].score;
+        }
+        else if (self->matches[right].end > self->matches[left].end) {
+            self->matches[left].end = self->matches[right].end;
+            self->matches[left].score += self->matches[right].score;
+        }
+        else if (right > left) {
+            self->matches[left].score += self->matches[right].score;
+        }
+    }
+    self->size = left + 1;
+    return self;
+}
+
+
+static MatchVector *matchv_set_offsets(MatchVector *mv, Offset *offsets)
+{
+    int i;
+    for (i = 0; i < mv->size; i++) {
+        mv->matches[i].start_offset = offsets[mv->matches[i].start].start;
+        mv->matches[i].end_offset = offsets[mv->matches[i].end].end;
+    }
+    return mv;
+}
+
+void matchv_destroy(MatchVector *self)
+{
+    free(self->matches);
+    free(self);
 }
 
 /***************************************************************************
@@ -422,211 +614,541 @@ int scorer_doc_cmp(const void *p1, const void *p2)
  *
  ***************************************************************************/
 
-static int s_doc_freq(Searcher *self, Term *term)
+MatchVector *searcher_get_match_vector(Searcher *self,
+                                       Query *query,
+                                       const int doc_num,
+                                       const char *field)
+{
+    MatchVector *mv = matchv_new();
+    Query *rewritten_query = self->rewrite(self, query);
+    TermVector *tv = self->get_term_vector(self, doc_num, field);
+    if (tv && tv->term_cnt > 0 && tv->terms[0].positions != NULL) {
+        mv = rewritten_query->get_matchv_i(rewritten_query, mv, tv);
+        tv_destroy(tv);
+    }
+    q_deref(rewritten_query);
+    return mv;
+}
+
+typedef struct Excerpt
 {
-  return self->ir->doc_freq(self->ir, term);
+    int start;
+    int end;
+    int start_pos;
+    int end_pos;
+    int start_offset;
+    int end_offset;
+    double score;
+} Excerpt;
+
+/*
+static int excerpt_cmp(const void *p1, const void *p2)
+{
+    double score1 = (*((Excerpt **)p1))->score;
+    double score2 = (*((Excerpt **)p2))->score;
+    if (score1 > score2) return 1;
+    if (score1 < score2) return -1;
+    return 0;
 }
+*/
 
-static int *s_doc_freqs(Searcher *self, Term **terms, int tcnt)
+static int excerpt_start_cmp(const void *p1, const void *p2)
 {
-  int i;
-  int *freqs = ALLOC_N(int, tcnt);
+    return (*((Excerpt **)p1))->start - (*((Excerpt **)p2))->start;
+}
 
-  for (i = 0; i < tcnt; i++) {
-    freqs[i] = self->ir->doc_freq(self->ir, terms[i]);
-  }
-  return freqs;
+static int excerpt_lt(Excerpt *e1, Excerpt *e2)
+{
+    return e1->score > e2->score; /* want the highest score at top */
 }
 
-static int *ss_doc_freqs(Searcher *self, Term **terms, int tcnt)
+static Excerpt *excerpt_new(int start, int end, double score)
 {
-  int i;
-  int *freqs = ALLOC_N(int, tcnt);
+    Excerpt *excerpt = ALLOC_AND_ZERO(Excerpt);
+    excerpt->start = start;
+    excerpt->end = end;
+    excerpt->score = score;
+    return excerpt;
+}
 
-  for (i = 0; i < tcnt; i++) {
-    freqs[i] = self->doc_freq(self, terms[i]);
-  }
+static Excerpt *excerpt_recalc_score(Excerpt *e, MatchVector *mv)
+{
+    int i;
+    double score = 0.0;
+    for (i = e->start; i <= e->end; i++) {
+        score += mv->matches[i].score;
+    }
+    e->score = score;
+    return e;
+}
 
-  return freqs;
+/* expand an excerpt to it's largest possible size */
+static Excerpt *excerpt_expand(Excerpt *e, const int len, TermVector *tv)
+{
+    Offset *offsets = tv->offsets;
+    int offset_cnt = tv->offset_cnt;
+    bool did_expansion = true;
+    int i;
+    /* fill in skipped offsets */
+    for (i = 1; i < offset_cnt; i++) {
+        if (offsets[i].start == 0) {
+            offsets[i].start = offsets[i-1].start;
+        }
+        if (offsets[i].end == 0) {
+            offsets[i].end = offsets[i-1].end;
+        }
+    }
+    
+    while (did_expansion) {
+        did_expansion = false;
+        if (e->start_pos > 0
+            && (e->end_offset - offsets[e->start_pos - 1].start) < len) {
+            e->start_pos--;
+            e->start_offset = offsets[e->start_pos].start;
+            did_expansion = true;
+        }
+        if (e->end_pos < (offset_cnt - 1)
+            && (offsets[e->end_pos + 1].end - e->start_offset) < len) {
+            e->end_pos++;
+            e->end_offset = offsets[e->end_pos].end;
+            did_expansion = true;
+        }
+    }
+    return e;
+}
+
+static char *excerpt_get_str(Excerpt *e, MatchVector *mv,
+                             LazyDocField *lazy_df,
+                             const char *pre_tag, 
+                             const char *post_tag,
+                             const char *ellipsis)
+{
+    int i, len;
+    int last_offset = e->start_offset;
+    const int num_matches = e->end - e->start + 1;
+    const int pre_tag_len = (int)strlen(pre_tag);
+    const int post_tag_len = (int)strlen(post_tag);
+    const int ellipsis_len = (int)strlen(ellipsis);
+    char *excerpt_str = ALLOC_N(char,
+                                10 + e->end_offset - e->start_offset
+                                + (num_matches * (pre_tag_len + post_tag_len))
+                                + (2 * ellipsis_len));
+    char *e_ptr = excerpt_str;
+    if (e->start_offset > 0) {
+        memcpy(e_ptr, ellipsis, ellipsis_len);
+        e_ptr += ellipsis_len;
+    }
+    for (i = e->start; i <= e->end; i++) {
+        MatchRange *mr = mv->matches + i;
+        len = mr->start_offset - last_offset;
+        lazy_df_get_bytes(lazy_df, e_ptr, last_offset, len);
+        e_ptr += len;
+        memcpy(e_ptr, pre_tag, pre_tag_len);
+        e_ptr += pre_tag_len;
+        len = mr->end_offset - mr->start_offset;
+        lazy_df_get_bytes(lazy_df, e_ptr, mr->start_offset, len);
+        e_ptr += len;
+        memcpy(e_ptr, post_tag, post_tag_len);
+        e_ptr += post_tag_len;
+        last_offset = mr->end_offset;
+    }
+    len = e->end_offset - last_offset;
+    lazy_df_get_bytes(lazy_df, e_ptr, last_offset, len);
+    e_ptr += len;
+    if (e->end_offset < lazy_df->len) {
+        memcpy(e_ptr, ellipsis, ellipsis_len);
+        e_ptr += ellipsis_len;
+    }
+    *e_ptr = '\0';
+    return excerpt_str;
+}
+
+char **searcher_highlight(Searcher *self,
+                          Query *query,
+                          const int doc_num,
+                          const char *field,
+                          const int excerpt_len,
+                          const int num_excerpts,
+                          const char *pre_tag,
+                          const char *post_tag,
+                          const char *ellipsis)
+{
+    char **excerpt_strs = NULL;
+    TermVector *tv = self->get_term_vector(self, doc_num, field);
+    LazyDoc *lazy_doc = self->get_lazy_doc(self, doc_num);
+    LazyDocField *lazy_df = NULL;
+    if (lazy_doc) {
+        lazy_df = h_get(lazy_doc->field_dict, field);
+    }
+    if (tv && lazy_df && tv->term_cnt > 0 && tv->terms[0].positions != NULL
+        && tv->offsets != NULL) {
+        MatchVector *mv;
+        query = self->rewrite(self, query);
+        mv = query->get_matchv_i(query, matchv_new(), tv);
+        if (mv->size > 0) {
+            Excerpt **excerpts = ALLOC_AND_ZERO_N(Excerpt *, num_excerpts);
+            int e_start, e_end, i, j;
+            MatchRange *matches = mv->matches;
+            double running_score = 0.0;
+            Offset *offsets = tv->offsets;
+            PriorityQueue *excerpt_pq;
+
+            matchv_compact_with_breaks(mv);
+            matchv_set_offsets(mv, offsets);
+            excerpt_pq = pq_new(mv->size, (lt_ft)&excerpt_lt, &free);
+            /* add all possible excerpts to the priority queue */
+            
+            for (e_start = 0, e_end = 1; e_start < mv->size; e_start++) {
+                const int start_offset = matches[e_start].start_offset;
+                if (e_start >= e_end) {
+                    e_end = e_start + 1;
+                }
+                running_score += matches[e_start].score;
+                while (e_end < mv->size && (matches[e_end].end_offset
+                                             <= start_offset + excerpt_len)) {
+                    running_score += matches[e_end].score;
+                    e_end++;
+                }
+                pq_push(excerpt_pq,
+                        excerpt_new(e_start, e_end - 1, running_score));
+                /* - 0.1 so that earlier matches take priority */
+                running_score -= matches[e_start].score;
+            }
+
+            for (i = 0; i < num_excerpts && excerpt_pq->size > 0; i++) {
+                excerpts[i] = pq_pop(excerpt_pq);
+                if (i < num_excerpts - 1) {
+                    /* set match ranges alread included to 0 */
+                    Excerpt *e = excerpts[i];
+                    for (j = e->start; j <= e->end; j++) {
+                        matches[j].score = 0.0;
+                    }
+                    e = NULL;
+                    while (e != (Excerpt *)pq_top(excerpt_pq)) {
+                        e = pq_top(excerpt_pq);
+                        excerpt_recalc_score(e, mv);
+                        pq_down(excerpt_pq);
+                    }
+                }
+            }
+
+            qsort(excerpts, i, sizeof(Excerpt *), &excerpt_start_cmp);
+            for (j = 0; j < i; j++) {
+                Excerpt *e = excerpts[j];
+                e->start_pos = matches[e->start].start;
+                e->end_pos = matches[e->end].end;
+                e->start_offset = offsets[e->start_pos].start;
+                e->end_offset = offsets[e->end_pos].end;
+            }
+
+            if (i < num_excerpts) {
+                const int diff = num_excerpts - i;
+                memmove(excerpts + (diff), excerpts,
+                        i * sizeof(Excerpt *));
+                for (j = 0; j < diff; j++) {
+                    /* these new excerpts will grow into one long excerpt at
+                     * the start */
+                    excerpts[j] = ALLOC_AND_ZERO(Excerpt);
+                    excerpts[j]->end = -1;
+                }
+            }
+
+            excerpt_strs = ary_new_type_capa(char *, num_excerpts);
+            /* merge excerpts where possible */
+            for (i = 0; i < num_excerpts;) {
+                Excerpt *ei = excerpts[i];
+                int merged = 1; /* 1 means a single excerpt, ie no merges */
+                for (j = i + 1; j < num_excerpts; j++) {
+                    Excerpt *ej = excerpts[j];
+                    if ((ej->end_offset - ei->start_offset)
+                        < (j - i + 1) * excerpt_len) {
+                        ei->end = ej->end;
+                        ei->end_pos = ej->end_pos;
+                        ei->end_offset = ej->end_offset;
+                        merged = j - i + 1;
+                    }
+                }
+                excerpt_expand(ei, merged * excerpt_len, tv);
+                ary_push(excerpt_strs,
+                         excerpt_get_str(ei, mv, lazy_df,
+                                         pre_tag, post_tag, ellipsis));
+                i += merged;
+            }
+            for (i = 0; i < num_excerpts; i++) {
+                free(excerpts[i]);
+            }
+            free(excerpts);
+            pq_destroy(excerpt_pq);
+            matchv_destroy(mv);
+        }
+        q_deref(query);
+    }
+    if (tv) tv_destroy(tv);
+    if (lazy_doc) lazy_doc_close(lazy_doc);
+    return excerpt_strs;
 }
 
+static Weight *sea_create_weight(Searcher *self, Query *query)
+{
+    return q_weight(query, self);
+}
 
-static Document *s_get_doc(Searcher *self, int doc_num)
+static void sea_check_args(int num_docs, int first_doc)
 {
-  return self->ir->get_doc(self->ir, doc_num);
+    if (num_docs <= 0) {
+        RAISE(ARG_ERROR, ":num_docs was set to %d but should be greater "
+              "than 0 : %d <= 0", num_docs, num_docs);
+    }
+
+    if (first_doc < 0) {
+        RAISE(ARG_ERROR, ":first_doc was set to %d but should be greater "
+              "than or equal to 0 : %d < 0", first_doc, first_doc);
+    }
 }
 
-static int s_max_doc(Searcher *self)
+static Similarity *sea_get_similarity(Searcher *self)
 {
-  return self->ir->max_doc(self->ir);
+    return self->similarity;
 }
 
-static Weight *s_create_weight(Searcher *self, Query *query)
+/***************************************************************************
+ *
+ * IndexSearcher
+ *
+ ***************************************************************************/
+
+#define ISEA(searcher) ((IndexSearcher *)(searcher))
+
+int isea_doc_freq(Searcher *self, const char *field, const char *term)
 {
-  return q_weight(query, self);
+    return ir_doc_freq(ISEA(self)->ir, field, term);
 }
 
-static TopDocs *s_search(Searcher *self, Query *query, int first_doc,
-    int num_docs, Filter *filter, Sort *sort)
+static Document *isea_get_doc(Searcher *self, int doc_num)
 {
-  int max_size = first_doc + num_docs;
-  int i;
-  Weight *weight;
-  Scorer *scorer;
-  Hit **score_docs = NULL;
-  Hit hit;
-  int total_hits = 0;
-  float score;
-  BitVector *bits = (filter ? filter->get_bv(filter, self->ir) : NULL);
-  Hit *(*hq_pop)(PriorityQueue *pq);
-  void (*hq_insert)(PriorityQueue *pq, Hit *hit);
-  void (*hq_destroy)(PriorityQueue *self);
-  PriorityQueue *hq;
+    IndexReader *ir = ISEA(self)->ir;
+    return ir->get_doc(ir, doc_num);
+}
 
+static LazyDoc *isea_get_lazy_doc(Searcher *self, int doc_num)
+{
+    IndexReader *ir = ISEA(self)->ir;
+    return ir->get_lazy_doc(ir, doc_num);
+}
 
-  if (num_docs <= 0)
-    RAISE(ARG_ERROR, NUM_DOCS_ARG_ERROR_MSG);
+static int isea_max_doc(Searcher *self)
+{
+    IndexReader *ir = ISEA(self)->ir;
+    return ir->max_doc(ir);
+}
 
-  if (first_doc < 0)
-    RAISE(ARG_ERROR, FIRST_DOC_ARG_ERROR_MSG);
+#define IS_FILTERED(bits, filter_func, scorer, searcher) \
+((bits && !bv_get(bits, scorer->doc))\
+ || (filter_func \
+     && !filter_func(scorer->doc, scorer->score(scorer), searcher)))
 
-  weight = q_weight(query, self);
-  scorer = weight->scorer(weight, self->ir);
-  if (!scorer) {
-    if (bits) bv_destroy(bits);
-    weight->destroy(weight);
-    return td_create(0, 0, NULL);
-  }
+static TopDocs *isea_search_w(Searcher *self,
+                              Weight *weight,
+                              int first_doc,
+                              int num_docs,
+                              Filter *filter,
+                              Sort *sort,
+                              filter_ft filter_func,
+                              bool load_fields)
+{
+    int max_size = first_doc + num_docs;
+    int i;
+    Scorer *scorer;
+    Hit **score_docs = NULL;
+    Hit hit;
+    int total_hits = 0;
+    float score, max_score = 0.0;
+    BitVector *bits = (filter
+                       ? filt_get_bv(filter, ISEA(self)->ir)
+                       : NULL);
+    Hit *(*hq_pop)(PriorityQueue *pq);
+    void (*hq_insert)(PriorityQueue *pq, Hit *hit);
+    void (*hq_destroy)(PriorityQueue *self);
+    PriorityQueue *hq;
 
-  if (sort) {
-    hq = fshq_pq_create(max_size, sort, self->ir);
-    hq_pop = &fshq_pq_pop;
-    hq_insert = &fshq_pq_insert;
-    hq_destroy = &fshq_pq_destroy;
-  } else {
-    hq = pq_create(max_size, &hit_less_than);
-    hq_pop = &hit_pq_pop;
-    hq_insert = &hit_pq_insert;
-    hq_destroy = &pq_destroy;
-  }
+    sea_check_args(num_docs, first_doc);
 
-  while (scorer->next(scorer)) {
-    if (bits && !bv_get(bits, scorer->doc)) continue;
-    total_hits++;
-    score = scorer->score(scorer);
-    hit.doc = scorer->doc; hit.score = score;
-    hq_insert(hq, &hit);
-  }
-  scorer->destroy(scorer);
-  weight->destroy(weight);
+    scorer = weight->scorer(weight, ISEA(self)->ir);
+    if (!scorer) {
+        return td_new(0, 0, NULL, 0.0);
+    }
 
-  if (hq->count > first_doc) {
-    if ((hq->count - first_doc) < num_docs) {
-      num_docs = hq->count - first_doc;
+    if (sort) {
+        hq = fshq_pq_new(max_size, sort, ISEA(self)->ir);
+        hq_insert = &fshq_pq_insert;
+        hq_destroy = &fshq_pq_destroy;
+        if (load_fields) {
+            hq_pop = &fshq_pq_pop_fd;
+        }
+        else {
+            hq_pop = &fshq_pq_pop;
+        }
+    }
+    else {
+        hq = pq_new(max_size, (lt_ft)&hit_less_than, &free);
+        hq_pop = &hit_pq_pop;
+        hq_insert = &hit_pq_insert;
+        hq_destroy = &pq_destroy;
     }
-    score_docs = ALLOC_N(Hit *, num_docs);
-    for (i = num_docs - 1; i >= 0; i--) {
-      score_docs[i] = hq_pop(hq);
-      //hit = score_docs[i] = pq_pop(hq);
-      //printf("hit = %d-->%f\n", hit->doc, hit->score);
+
+    while (scorer->next(scorer)) {
+        if (IS_FILTERED(bits, filter_func, scorer, self)) {
+            continue;
+        }
+        total_hits++;
+        score = scorer->score(scorer);
+        if (score > max_score) max_score = score;
+        hit.doc = scorer->doc; hit.score = score;
+        hq_insert(hq, &hit);
     }
-  } else {
-    num_docs = 0;
-  }
-  pq_clear(hq);
-  hq_destroy(hq);
+    scorer->destroy(scorer);
 
-  if (bits) bv_destroy(bits);
-  return td_create(total_hits, num_docs, score_docs);
+    if (hq->size > first_doc) {
+        if ((hq->size - first_doc) < num_docs) {
+            num_docs = hq->size - first_doc;
+        }
+        score_docs = ALLOC_N(Hit *, num_docs);
+        for (i = num_docs - 1; i >= 0; i--) {
+            score_docs[i] = hq_pop(hq);
+            /*
+            hit = score_docs[i] = pq_pop(hq);
+            printf("hit = %d-->%f\n", hit->doc, hit->score);
+            */
+        }
+    }
+    else {
+        num_docs = 0;
+    }
+    pq_clear(hq);
+    hq_destroy(hq);
+
+    return td_new(total_hits, num_docs, score_docs, max_score);
+}
+
+static TopDocs *isea_search(Searcher *self,
+                            Query *query,
+                            int first_doc,
+                            int num_docs,
+                            Filter *filter,
+                            Sort *sort,
+                            filter_ft filter_func,
+                            bool load_fields)
+{
+    TopDocs *td;
+    Weight *weight = q_weight(query, self);
+    td = isea_search_w(self, weight, first_doc, num_docs, filter,
+                         sort, filter_func, load_fields);
+    weight->destroy(weight);
+    return td;
 }
 
-static void s_search_each_w(Searcher *self, Weight *weight, Filter *filter,
-    void (*fn)(Searcher *, int, float, void *), void *arg)
+static void isea_search_each_w(Searcher *self, Weight *weight, Filter *filter,
+                               filter_ft filter_func,
+                               void (*fn)(Searcher *, int, float, void *),
+                               void *arg)
 {
-  Scorer *scorer;
-  BitVector *bits = (filter ? filter->get_bv(filter, self->ir) : NULL);
+    Scorer *scorer;
+    BitVector *bits = (filter
+                       ? filt_get_bv(filter, ISEA(self)->ir)
+                       : NULL);
 
-  scorer = weight->scorer(weight, self->ir);
-  if (!scorer) {
-    if (bits) bv_destroy(bits);
-    return;
-  }
+    scorer = weight->scorer(weight, ISEA(self)->ir);
+    if (!scorer) {
+        return;
+    }
 
-  while (scorer->next(scorer)) {
-    if (bits && !bv_get(bits, scorer->doc)) continue;
-    fn(self, scorer->doc, scorer->score(scorer), arg);
-  }
-  scorer->destroy(scorer);
+    while (scorer->next(scorer)) {
+        if (IS_FILTERED(bits, filter_func, scorer, self)) {
+            continue;
+        }
+        fn(self, scorer->doc, scorer->score(scorer), arg);
+    }
+    scorer->destroy(scorer);
 }
 
-static void s_search_each(Searcher *self, Query *query, Filter *filter,
-    void (*fn)(Searcher *, int, float, void *), void *arg)
+static void isea_search_each(Searcher *self, Query *query, Filter *filter,
+                             filter_ft filter_func,
+                             void (*fn)(Searcher *, int, float, void *),
+                             void *arg)
 {
-  Weight *weight = q_weight(query, self);
-  s_search_each_w(self, weight, filter, fn, arg);
-  weight->destroy(weight);
+    Weight *weight = q_weight(query, self);
+    isea_search_each_w(self, weight, filter, filter_func, fn, arg);
+    weight->destroy(weight);
 }
 
-static Query *s_rewrite(Searcher *self, Query *original)
+static Query *isea_rewrite(Searcher *self, Query *original)
 {
-  int q_is_destroyed = false;
-  Query *query = original;
-  Query *rewritten_query = query->rewrite(query, self->ir);
-  while (q_is_destroyed || (query != rewritten_query)) {
-    query = rewritten_query;
-    rewritten_query = query->rewrite(query, self->ir);
-    q_is_destroyed = (query->ref_cnt <= 1);
-    q_deref(query); /* destroy intermediate queries */
-  }
-  return query;
+    int q_is_destroyed = false;
+    Query *query = original;
+    Query *rewritten_query = query->rewrite(query, ISEA(self)->ir);
+    while (q_is_destroyed || (query != rewritten_query)) {
+        query = rewritten_query;
+        rewritten_query = query->rewrite(query, ISEA(self)->ir);
+        q_is_destroyed = (query->ref_cnt <= 1);
+        q_deref(query); /* destroy intermediate queries */
+    }
+    return query;
 }
 
-static Explanation *s_explain(Searcher *self, Query *query, int doc_num)
+static Explanation *isea_explain(Searcher *self, Query *query, int doc_num)
 {
-  Weight *weight = q_weight(query, self);
-  Explanation *e =  weight->explain(weight, self->ir, doc_num);
-  weight->destroy(weight);
-  return e;
+    Weight *weight = q_weight(query, self);
+    Explanation *e =  weight->explain(weight, ISEA(self)->ir, doc_num);
+    weight->destroy(weight);
+    return e;
 }
 
-static Explanation *s_explain_w(Searcher *self, Weight *w, int doc_num)
+static Explanation *isea_explain_w(Searcher *self, Weight *w, int doc_num)
 {
-  return w->explain(w, self->ir, doc_num);
+    return w->explain(w, ISEA(self)->ir, doc_num);
 }
 
-static Similarity *s_get_similarity(Searcher *self)
+static TermVector *isea_get_term_vector(Searcher *self,
+                                          const int doc_num,
+                                          const char *field)
 {
-  return self->similarity;
+    IndexReader *ir = ISEA(self)->ir;
+    return ir->term_vector(ir, doc_num, field);
 }
 
-static void s_close(Searcher *self)
+static void isea_close(Searcher *self)
 {
-  if (self->ir && self->close_ir) {
-    ir_close(self->ir);
-  }
-  free(self);
+    if (ISEA(self)->ir && ISEA(self)->close_ir) {
+        ir_close(ISEA(self)->ir);
+    }
+    free(self);
 }
 
-Searcher *sea_create(IndexReader *ir)
+Searcher *isea_new(IndexReader *ir)
 {
-  Searcher *self = ALLOC(Searcher);
-  self->ir = ir;
-  self->close_ir = true;
-  self->similarity = sim_create_default();
-  self->doc_freq = &s_doc_freq;
-  self->doc_freqs = &s_doc_freqs;
-  self->get_doc = &s_get_doc;
-  self->max_doc = &s_max_doc;
-  self->create_weight = &s_create_weight;
-  self->search = &s_search;
-  self->search_each = &s_search_each;
-  self->search_each_w = &s_search_each_w;
-  self->rewrite = &s_rewrite;
-  self->explain = &s_explain;
-  self->explain_w = &s_explain_w;
-  self->get_similarity = &s_get_similarity;
-  self->close = &s_close;
-  return self;
+    Searcher *self          = (Searcher *)ecalloc(sizeof(IndexSearcher));
+
+    ISEA(self)->ir          = ir;
+    ISEA(self)->close_ir    = true;
+
+    self->similarity        = sim_create_default();
+    self->doc_freq          = &isea_doc_freq;
+    self->get_doc           = &isea_get_doc;
+    self->get_lazy_doc      = &isea_get_lazy_doc;
+    self->max_doc           = &isea_max_doc;
+    self->create_weight     = &sea_create_weight;
+    self->search            = &isea_search;
+    self->search_w          = &isea_search_w;
+    self->search_each       = &isea_search_each;
+    self->search_each_w     = &isea_search_each_w;
+    self->rewrite           = &isea_rewrite;
+    self->explain           = &isea_explain;
+    self->explain_w         = &isea_explain_w;
+    self->get_term_vector   = &isea_get_term_vector;
+    self->get_similarity    = &sea_get_similarity;
+    self->close             = &isea_close;
+
+    return self;
 }
 
 /***************************************************************************
@@ -635,109 +1157,144 @@ Searcher *sea_create(IndexReader *ir)
  *
  ***************************************************************************/
 
-typedef struct CachedDFSearcher {
-  HshTable *df_map;
-  int max_doc;
+#define CDFSEA(searcher) ((CachedDFSearcher *)(searcher))
+typedef struct CachedDFSearcher
+{
+    Searcher    super;
+    HashTable  *df_map;
+    int         max_doc;
 } CachedDFSearcher;
 
-static int cdfsea_doc_freq(Searcher *self, Term *term)
+static int cdfsea_doc_freq(Searcher *self, const char *field, const char *text)
 {
-  CachedDFSearcher *cdfsea = (CachedDFSearcher *)self->data;
-  return (int)h_get(cdfsea->df_map, term);
+    Term term;
+    int *df;
+    term.field = (char *)field;
+    term.text = (char *)text;
+    df = (int *)h_get(CDFSEA(self)->df_map, &term);
+    return df ? *df : 0;
 }
 
 static Document *cdfsea_get_doc(Searcher *self, int doc_num)
 {
-  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
-  return NULL;
+    (void)self; (void)doc_num;
+    RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+    return NULL;
 }
 
 static int cdfsea_max_doc(Searcher *self)
 {
-  return ((CachedDFSearcher *)self->data)->max_doc;
+    (void)self;
+    return CDFSEA(self)->max_doc;
 }
 
 static Weight *cdfsea_create_weight(Searcher *self, Query *query)
 {
-  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
-  return NULL;
+    (void)self; (void)query;
+    RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+    return NULL;
+}
+
+static TopDocs *cdfsea_search_w(Searcher *self, Weight *w, int fd, int nd,
+                                Filter *f, Sort *s, filter_ft ff, bool load)
+{
+    (void)self; (void)w; (void)fd; (void)nd;
+    (void)f; (void)s; (void)ff, (void)load;
+    RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+    return NULL;
 }
 
-static TopDocs *cdfsea_search(Searcher *self, Query *query, int first_doc,
-    int num_docs, Filter *filter, Sort *sort)
+static TopDocs *cdfsea_search(Searcher *self, Query *q, int fd, int nd,
+                              Filter *f, Sort *s, filter_ft ff, bool load)
 {
-  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
-  return NULL;
+    (void)self; (void)q; (void)fd; (void)nd;
+    (void)f; (void)s; (void)ff, (void)load;
+    RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+    return NULL;
 }
 
 static void cdfsea_search_each(Searcher *self, Query *query, Filter *filter,
-    void (*fn)(Searcher *, int, float, void *), void *arg)
+                               filter_ft ff, 
+                               void (*fn)(Searcher *, int, float, void *),
+                               void *arg)
 {
-  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+    (void)self; (void)query; (void)filter; (void)ff; (void)fn; (void)arg;
+    RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
 }
 
 static void cdfsea_search_each_w(Searcher *self, Weight *w, Filter *filter,
-    void (*fn)(Searcher *, int, float, void *), void *arg)
+                                 filter_ft ff, 
+                                 void (*fn)(Searcher *, int, float, void *),
+                                 void *arg)
 {
-  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+    (void)self; (void)w; (void)filter; (void)ff; (void)fn; (void)arg;
+    RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
 }
 
 static Query *cdfsea_rewrite(Searcher *self, Query *original)
 {
-  original->ref_cnt++;
-  return original;
+    (void)self;
+    original->ref_cnt++;
+    return original;
 }
 
 static Explanation *cdfsea_explain(Searcher *self, Query *query, int doc_num)
 {
-  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
-  return NULL;
+    (void)self; (void)query; (void)doc_num;
+    RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+    return NULL;
 }
 
 static Explanation *cdfsea_explain_w(Searcher *self, Weight *w, int doc_num)
 {
-  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
-  return NULL;
+    (void)self; (void)w; (void)doc_num;
+    RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+    return NULL;
+}
+
+static TermVector *cdfsea_get_term_vector(Searcher *self, const int doc_num,
+                                          const char *field)
+{
+    (void)self; (void)doc_num; (void)field;
+    RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+    return NULL;
 }
 
 static Similarity *cdfsea_get_similarity(Searcher *self)
 {
-  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
-  return NULL;
+    (void)self;
+    RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
+    return NULL;
 }
 
 static void cdfsea_close(Searcher *self)
 {
-  CachedDFSearcher *cdfsea = (CachedDFSearcher *)self->data;
-  h_destroy(cdfsea->df_map);
-  free(cdfsea);
-  free(self);
+    h_destroy(CDFSEA(self)->df_map);
+    free(self);
 }
 
-Searcher *cdfsea_create(HshTable *df_map, int max_doc)
+static Searcher *cdfsea_new(HashTable *df_map, int max_doc)
 {
-  Searcher *self = ALLOC(Searcher);
+    Searcher *self          = (Searcher *)ecalloc(sizeof(CachedDFSearcher));
 
-  CachedDFSearcher *cdfsea = ALLOC(CachedDFSearcher);
+    CDFSEA(self)->df_map    = df_map;
+    CDFSEA(self)->max_doc   = max_doc;
 
-  cdfsea->df_map = df_map;
-  cdfsea->max_doc = max_doc;
-  self->data = cdfsea;
-
-  self->doc_freq = &cdfsea_doc_freq;
-  self->doc_freqs = &ss_doc_freqs;
-  self->get_doc = &cdfsea_get_doc;
-  self->max_doc = &cdfsea_max_doc;
-  self->create_weight = &cdfsea_create_weight;
-  self->search = &cdfsea_search;
-  self->search_each = &cdfsea_search_each;
-  self->search_each_w = &cdfsea_search_each_w;
-  self->rewrite = &cdfsea_rewrite;
-  self->explain = &cdfsea_explain;
-  self->explain_w = &cdfsea_explain_w;
-  self->get_similarity = &cdfsea_get_similarity;
-  self->close = &cdfsea_close;
-  return self;
+    self->doc_freq          = &cdfsea_doc_freq;
+    self->get_doc           = &cdfsea_get_doc;
+    self->max_doc           = &cdfsea_max_doc;
+    self->create_weight     = &cdfsea_create_weight;
+    self->search            = &cdfsea_search;
+    self->search_w          = &cdfsea_search_w;
+    self->search_each       = &cdfsea_search_each;
+    self->search_each_w     = &cdfsea_search_each_w;
+    self->rewrite           = &cdfsea_rewrite;
+    self->explain           = &cdfsea_explain;
+    self->explain_w         = &cdfsea_explain_w;
+    self->get_term_vector   = &cdfsea_get_term_vector;
+    self->get_similarity    = &cdfsea_get_similarity;
+    self->close             = &cdfsea_close;
+    return self;
 }
 
 /***************************************************************************
@@ -746,301 +1303,367 @@ Searcher *cdfsea_create(HshTable *df_map, int max_doc)
  *
  ***************************************************************************/
 
+#define MSEA(searcher) ((MultiSearcher *)(searcher))
 static inline int msea_get_searcher_index(Searcher *self, int n)
 {
-  MultiSearcher *msea = (MultiSearcher *)self->data;
-  int lo = 0;               /* search starts array */
-  int hi = msea->s_cnt - 1; /* for 1st element < n, return its index */
-  int mid, mid_val;
+    MultiSearcher *msea = MSEA(self);
+    int lo = 0;                 /* search starts array */
+    int hi = msea->s_cnt - 1;   /* for 1st element < n, return its index */
+    int mid, mid_val;
 
-  while (hi >= lo) {
-    mid = (lo + hi) >> 1;
-    mid_val = msea->starts[mid];
-    if (n < mid_val) {
-      hi = mid - 1;
-    } else if (n > mid_val) {
-      lo = mid + 1;
-    } else {                  /* found a match */
-      while (((mid+1) < msea->s_cnt) && (msea->starts[mid+1] == mid_val)) {
-        mid++;                /* scan to last match */
-      }
-      return mid;
+    while (hi >= lo) {
+        mid = (lo + hi) >> 1;
+        mid_val = msea->starts[mid];
+        if (n < mid_val) {
+            hi = mid - 1;
+        }
+        else if (n > mid_val) {
+            lo = mid + 1;
+        }
+        else {                  /* found a match */
+            while (((mid+1) < msea->s_cnt)
+                   && (msea->starts[mid+1] == mid_val)) {
+                mid++;          /* scan to last match */
+            }
+            return mid;
+        }
     }
-  }
-  return hi;
+    return hi;
 }
 
-static int msea_doc_freq(Searcher *self, Term *term)
+static int msea_doc_freq(Searcher *self, const char *field, const char *term)
 {
-  int i;
-  int doc_freq = 0;
-  Searcher *s;
-  MultiSearcher *msea = (MultiSearcher *)self->data;
-  for (i = 0; i < msea->s_cnt; i++) {
-    s = msea->searchers[i];
-    doc_freq += s->doc_freq(s, term);
-  }
+    int i;
+    int doc_freq = 0;
+    MultiSearcher *msea = MSEA(self);
+    for (i = 0; i < msea->s_cnt; i++) {
+        Searcher *s = msea->searchers[i];
+        doc_freq += s->doc_freq(s, field, term);
+    }
 
-  return doc_freq;
+    return doc_freq;
 }
 
 static Document *msea_get_doc(Searcher *self, int doc_num)
 {
-  MultiSearcher *msea = (MultiSearcher *)self->data;
-  int i = msea_get_searcher_index(self, doc_num);
-  Searcher *s = msea->searchers[i];
-  return s->get_doc(s, doc_num - msea->starts[i]);
+    MultiSearcher *msea = MSEA(self);
+    int i = msea_get_searcher_index(self, doc_num);
+    Searcher *s = msea->searchers[i];
+    return s->get_doc(s, doc_num - msea->starts[i]);
+}
+
+static LazyDoc *msea_get_lazy_doc(Searcher *self, int doc_num)
+{
+    MultiSearcher *msea = MSEA(self);
+    int i = msea_get_searcher_index(self, doc_num);
+    Searcher *s = msea->searchers[i];
+    return s->get_lazy_doc(s, doc_num - msea->starts[i]);
 }
 
 static int msea_max_doc(Searcher *self)
 {
-  return ((MultiSearcher *)self->data)->max_doc;
+    return MSEA(self)->max_doc;
+}
+
+static int *msea_get_doc_freqs(Searcher *self, HashSet *terms)
+{
+    int i;
+    const int num_terms = terms->size;
+    int *doc_freqs = ALLOC_N(int, num_terms);
+    for (i = 0; i < num_terms; i++) {
+        Term *t = (Term *)terms->elems[i];
+        doc_freqs[i] = msea_doc_freq(self, t->field, t->text);
+    }
+    return doc_freqs;
 }
 
 static Weight *msea_create_weight(Searcher *self, Query *query)
 {
-  int i, *dfs;
-  Searcher *cdfsea;
-  Weight *w;
-  HshTable *df_map = h_new((hash_ft)&term_hash, (eq_ft)&term_eq,
-                          (free_ft)NULL, (free_ft)NULL);
-  Query *rq = self->rewrite(self, query);
-  HashSet *terms = term_set_create();
-  rq->extract_terms(rq, terms);
-  dfs = self->doc_freqs(self, (Term **)terms->elems, terms->size);
+    int i, *doc_freqs;
+    Searcher *cdfsea;
+    Weight *w;
+    HashTable *df_map = h_new((hash_ft)&term_hash, (eq_ft)&term_eq,
+                             (free_ft)NULL, free);
+    Query *rewritten_query = self->rewrite(self, query);
+    HashSet *terms = term_set_new();
 
-  for (i = 0; i < terms->size; i++) {
-    h_set(df_map, terms->elems[i], (void *)dfs[i]); 
-  }
-  /* don't destroy the individual terms, only the HashSet */
-  hs_destroy(terms);
-  free(dfs);
+    rewritten_query->extract_terms(rewritten_query, terms);
+    doc_freqs = msea_get_doc_freqs(self, terms);
 
-  cdfsea = cdfsea_create(df_map, ((MultiSearcher *)self->data)->max_doc);
+    for (i = 0; i < terms->size; i++) {
+        h_set(df_map, terms->elems[i], imalloc(doc_freqs[i])); 
+    }
+    hs_destroy(terms);
+    free(doc_freqs);
 
-  w = q_weight(rq, cdfsea);
-  q_deref(rq);
-  cdfsea->close(cdfsea);
+    cdfsea = cdfsea_new(df_map, MSEA(self)->max_doc);
 
-  return w;
+    w = q_weight(rewritten_query, cdfsea);
+    q_deref(rewritten_query);
+    cdfsea->close(cdfsea);
+
+    return w;
 }
 
 struct MultiSearchEachArg {
-  int start;
-  void *arg;
-  void (*fn)(Searcher *, int, float, void *);
+    int start;
+    void *arg;
+    void (*fn)(Searcher *, int, float, void *);
 };
 
 void msea_search_each_i(Searcher *self, int doc_num, float score, void *arg)
 {
-  struct MultiSearchEachArg *mse_arg = (struct MultiSearchEachArg *)arg;
+    struct MultiSearchEachArg *mse_arg = (struct MultiSearchEachArg *)arg;
 
-  mse_arg->fn(self, doc_num + mse_arg->start, score, mse_arg->arg);
+    mse_arg->fn(self, doc_num + mse_arg->start, score, mse_arg->arg);
 }
 
 static void msea_search_each_w(Searcher *self, Weight *w, Filter *filter,
-    void (*fn)(Searcher *, int, float, void *), void *arg)
+                               filter_ft filter_func,
+                               void (*fn)(Searcher *, int, float, void *),
+                               void *arg)
 {
-  int i;
-  struct MultiSearchEachArg mse_arg;
-  MultiSearcher *msea = (MultiSearcher *)self->data;
-  Searcher *s;
+    int i;
+    struct MultiSearchEachArg mse_arg;
+    MultiSearcher *msea = MSEA(self);
+    Searcher *s;
 
-  mse_arg.fn = fn;
-  mse_arg.arg = arg;
-  for (i = 0; i < msea->s_cnt; i++) {
-    s = msea->searchers[i];
-    mse_arg.start = msea->starts[i];
-    s->search_each_w(s, w, filter, &msea_search_each_i, &mse_arg);
-  }
+    mse_arg.fn = fn;
+    mse_arg.arg = arg;
+    for (i = 0; i < msea->s_cnt; i++) {
+        s = msea->searchers[i];
+        mse_arg.start = msea->starts[i];
+        s->search_each_w(s, w, filter, filter_func,
+                         &msea_search_each_i, &mse_arg);
+    }
 }
 
 static void msea_search_each(Searcher *self, Query *query, Filter *filter,
-    void (*fn)(Searcher *, int, float, void *), void *arg)
+                             filter_ft filter_func,
+                             void (*fn)(Searcher *, int, float, void *), void *arg)
 {
-  Weight *w = q_weight(query, self);
-  msea_search_each_w(self, w, filter, fn, arg);
-  w->destroy(w);
+    Weight *w = q_weight(query, self);
+    msea_search_each_w(self, w, filter, filter_func, fn, arg);
+    w->destroy(w);
 }
 
 struct MultiSearchArg {
-  int total_hits, max_size;
-  PriorityQueue *hq;
-  void (*hq_insert)(PriorityQueue *pq, Hit *hit);
+    int total_hits, max_size;
+    PriorityQueue *hq;
+    void (*hq_insert)(PriorityQueue *pq, Hit *hit);
 };
 
 void msea_search_i(Searcher *self, int doc_num, float score, void *arg)
 {
-  struct MultiSearchArg *ms_arg = (struct MultiSearchArg *)arg;
-  Hit hit;
-
-  ms_arg->total_hits++;
-  hit.doc = doc_num;
-  hit.score = score;
-  ms_arg->hq_insert(ms_arg->hq, &hit);
-}
-
-static TopDocs *msea_search(Searcher *self, Query *query, int first_doc,
-    int num_docs, Filter *filter, Sort *sort)
-{
-  int max_size = first_doc + num_docs;
-  int i;
-  Weight *weight;
-  Hit **score_docs = NULL;
-  BitVector *bits = (filter ? filter->get_bv(filter, self->ir) : NULL);
-  Hit *(*hq_pop)(PriorityQueue *pq);
-  void (*hq_insert)(PriorityQueue *pq, Hit *hit);
-  void (*hq_destroy)(PriorityQueue *self);
-  PriorityQueue *hq;
-  struct MultiSearchArg ms_arg;
-
-
-  if (num_docs <= 0)
-    RAISE(ARG_ERROR, NUM_DOCS_ARG_ERROR_MSG);
-
-  if (first_doc < 0)
-    RAISE(ARG_ERROR, FIRST_DOC_ARG_ERROR_MSG);
-
-  weight = q_weight(query, self);
-  if (sort) {
-    hq = fshq_pq_create(max_size, sort, self->ir);
-    hq_pop = &fshq_pq_pop;
-    hq_insert = &fshq_pq_insert;
-    hq_destroy = &fshq_pq_destroy;
-  } else {
-    hq = pq_create(max_size, &hit_less_than);
-    hq_pop = &hit_pq_pop;
-    hq_insert = &hit_pq_insert;
-    hq_destroy = &pq_destroy;
-  }
-
-
-  ms_arg.hq = hq;
-  ms_arg.total_hits = 0;
-  ms_arg.max_size = max_size;
-  ms_arg.hq_insert = hq_insert;
-
-  msea_search_each_w(self, weight, filter, msea_search_i, &ms_arg);
+    struct MultiSearchArg *ms_arg = (struct MultiSearchArg *)arg;
+    Hit hit;
+    (void)self;
+
+    ms_arg->total_hits++;
+    hit.doc = doc_num;
+    hit.score = score;
+    ms_arg->hq_insert(ms_arg->hq, &hit);
+}
+
+static TopDocs *msea_search_w(Searcher *self,
+                              Weight *weight,
+                              int first_doc,
+                              int num_docs,
+                              Filter *filter,
+                              Sort *sort,
+                              filter_ft filter_func,
+                              bool load_fields)
+{
+    int max_size = first_doc + num_docs;
+    int i;
+    int total_hits = 0;
+    Hit **score_docs = NULL;
+    Hit *(*hq_pop)(PriorityQueue *pq);
+    void (*hq_insert)(PriorityQueue *pq, Hit *hit);
+    PriorityQueue *hq;
+    float max_score = 0.0;
+    (void)load_fields; /* does it automatically */
+
+    sea_check_args(num_docs, first_doc);
+
+    if (sort) {
+        hq = pq_new(max_size, (lt_ft)fdshq_lt, &free);
+        hq_insert = (void (*)(PriorityQueue *pq, Hit *hit))&pq_insert;
+        hq_pop = (Hit *(*)(PriorityQueue *pq))&pq_pop;
+    }
+    else {
+        hq = pq_new(max_size, (lt_ft)&hit_less_than, &free);
+        hq_insert = &hit_pq_multi_insert;
+        hq_pop = &hit_pq_pop;
+    }
 
-  weight->destroy(weight);
+    /*if (sort) printf("sort = %s\n", sort_to_s(sort)); */
+    for (i = 0; i < MSEA(self)->s_cnt; i++) {
+        Searcher *s = MSEA(self)->searchers[i];
+        TopDocs *td = s->search_w(s, weight, 0, max_size,
+                                  filter, sort, filter_func, true);
+        /*if (sort) printf("sort = %s\n", sort_to_s(sort)); */
+        if (td->size > 0) {
+            /*printf("td->size = %d %d\n", td->size, num_docs); */
+            int j;
+            int start = MSEA(self)->starts[i];
+            for (j = 0; j < td->size; j++) {
+                Hit *hit = td->hits[j];
+                hit->doc += start;
+                /*
+                printf("adding hit = %d:%f\n", hit->doc, hit->score);
+                */
+                hq_insert(hq, hit);
+            }
+            td->size = 0;
+            if (td->max_score > max_score) max_score = td->max_score;
+        }
+        total_hits += td->total_hits;
+        td_destroy(td);
+    }
 
-  if (hq->count > first_doc) {
-    if ((hq->count - first_doc) < num_docs) {
-      num_docs = hq->count - first_doc;
+    if (hq->size > first_doc) {
+        if ((hq->size - first_doc) < num_docs) {
+            num_docs = hq->size - first_doc;
+        }
+        score_docs = ALLOC_N(Hit *, num_docs);
+        for (i = num_docs - 1; i >= 0; i--) {
+            score_docs[i] = hq_pop(hq);
+            /*
+            Hit *hit = score_docs[i] = hq_pop(hq);
+            printf("popped hit = %d-->%f\n", hit->doc, hit->score);
+            */
+        }
     }
-    score_docs = ALLOC_N(Hit *, num_docs);
-    for (i = num_docs - 1; i >= 0; i--) {
-      score_docs[i] = hq_pop(hq);
-      //hit = score_docs[i] = pq_pop(hq);
-      //printf("hit = %d-->%f\n", hit->doc, hit->score);
+    else {
+        num_docs = 0;
     }
-  } else {
-    num_docs = 0;
-  }
-  pq_clear(hq);
-  hq_destroy(hq);
+    pq_clear(hq);
+    pq_destroy(hq);
+
+    return td_new(total_hits, num_docs, score_docs, max_score);
+}
 
-  if (bits) bv_destroy(bits);
-  return td_create(ms_arg.total_hits, num_docs, score_docs);
+static TopDocs *msea_search(Searcher *self,
+                            Query *query,
+                            int first_doc,
+                            int num_docs,
+                            Filter *filter,
+                            Sort *sort,
+                            filter_ft filter_func,
+                            bool load_fields)
+{
+    TopDocs *td;
+    Weight *weight = q_weight(query, self);
+    td = msea_search_w(self, weight, first_doc, num_docs, filter,
+                       sort, filter_func, load_fields);
+    weight->destroy(weight);
+    return td;
 }
 
 static Query *msea_rewrite(Searcher *self, Query *original)
 {
-  int i;
-  Searcher *s;
-  MultiSearcher *msea = (MultiSearcher *)self->data;
-  Query **queries = ALLOC_N(Query *, msea->s_cnt), *rewritten;
+    int i;
+    Searcher *s;
+    MultiSearcher *msea = MSEA(self);
+    Query **queries = ALLOC_N(Query *, msea->s_cnt), *rewritten;
 
-  for (i = 0; i < msea->s_cnt; i++) {
-    s = msea->searchers[i];
-    queries[i] = s->rewrite(s, original);
-  }
-  rewritten = q_combine(queries, msea->s_cnt);
+    for (i = 0; i < msea->s_cnt; i++) {
+        s = msea->searchers[i];
+        queries[i] = s->rewrite(s, original);
+    }
+    rewritten = q_combine(queries, msea->s_cnt);
 
-  for (i = 0; i < msea->s_cnt; i++) {
-    q_deref(queries[i]);
-  }
-  free(queries);
-  return rewritten;
+    for (i = 0; i < msea->s_cnt; i++) {
+        q_deref(queries[i]);
+    }
+    free(queries);
+    return rewritten;
 }
 
 static Explanation *msea_explain(Searcher *self, Query *query, int doc_num)
 {
-  MultiSearcher *msea = (MultiSearcher *)self->data;
-  int i = msea_get_searcher_index(self, doc_num);
-  Weight *w = q_weight(query, self);
-  Searcher *s = msea->searchers[i];
-  Explanation *e = s->explain_w(s, w, doc_num - msea->starts[i]);
-  w->destroy(w);
-  return e;
+    MultiSearcher *msea = MSEA(self);
+    int i = msea_get_searcher_index(self, doc_num);
+    Weight *w = q_weight(query, self);
+    Searcher *s = msea->searchers[i];
+    Explanation *e = s->explain_w(s, w, doc_num - msea->starts[i]);
+    w->destroy(w);
+    return e;
 }
 
 static Explanation *msea_explain_w(Searcher *self, Weight *w, int doc_num)
 {
-  MultiSearcher *msea = (MultiSearcher *)self->data;
-  int i = msea_get_searcher_index(self, doc_num);
-  Searcher *s = msea->searchers[i];
-  Explanation *e = s->explain_w(s, w, doc_num - msea->starts[i]);
-  return e;
+    MultiSearcher *msea = MSEA(self);
+    int i = msea_get_searcher_index(self, doc_num);
+    Searcher *s = msea->searchers[i];
+    Explanation *e = s->explain_w(s, w, doc_num - msea->starts[i]);
+    return e;
+}
+
+static TermVector *msea_get_term_vector(Searcher *self, const int doc_num,
+                                        const char *field)
+{
+    MultiSearcher *msea = MSEA(self);
+    int i = msea_get_searcher_index(self, doc_num);
+    Searcher *s = msea->searchers[i];
+    return s->get_term_vector(s, doc_num - msea->starts[i],
+                              field);
 }
 
 static Similarity *msea_get_similarity(Searcher *self)
 {
-  return self->similarity;
+    return self->similarity;
 }
 
 static void msea_close(Searcher *self)
 {
-  int i;
-  Searcher *s;
-  MultiSearcher *msea = (MultiSearcher *)self->data;
-  if (msea->close_subs) {
-    for (i = 0; i < msea->s_cnt; i++) {
-      s = msea->searchers[i];
-      s->close(s);
+    int i;
+    Searcher *s;
+    MultiSearcher *msea = MSEA(self);
+    if (msea->close_subs) {
+        for (i = 0; i < msea->s_cnt; i++) {
+            s = msea->searchers[i];
+            s->close(s);
+        }
+        free(msea->searchers);
     }
-    free(msea->searchers);
-  }
-  free(msea->starts);
-  free(msea);
-  free(self);
+    free(msea->starts);
+    free(self);
 }
 
-Searcher *msea_create(Searcher **searchers, int s_cnt, bool close_subs)
+Searcher *msea_new(Searcher **searchers, int s_cnt, bool close_subs)
 {
-  int i, max_doc = 0, *starts;
-  Searcher *self = ALLOC(Searcher);
-
-  MultiSearcher *msea = ALLOC(MultiSearcher);
-
-  starts = ALLOC_N(int, s_cnt + 1);
-  for (i = 0; i < s_cnt; i++) {
+    int i, max_doc = 0;
+    Searcher *self = (Searcher *)ecalloc(sizeof(MultiSearcher));
+    int *starts = ALLOC_N(int, s_cnt + 1);
+    for (i = 0; i < s_cnt; i++) {
+        starts[i] = max_doc;
+        max_doc += searchers[i]->max_doc(searchers[i]);
+    }
     starts[i] = max_doc;
-    max_doc += searchers[i]->max_doc(searchers[i]);
-  }
-  starts[i] = max_doc;
-
-  msea->s_cnt = s_cnt;
-  msea->searchers = searchers;
-  msea->starts = starts;
-  msea->max_doc = max_doc;
-  msea->close_subs = close_subs;
-  self->data = msea;
-
-  self->ir = (IndexReader *)NULL;
-  self->similarity = sim_create_default();
-  self->doc_freq = &msea_doc_freq;
-  self->doc_freqs = &ss_doc_freqs;
-  self->get_doc = &msea_get_doc;
-  self->max_doc = &msea_max_doc;
-  self->create_weight = &msea_create_weight;
-  self->search = &msea_search;
-  self->search_each = &msea_search_each;
-  self->search_each_w = &msea_search_each_w;
-  self->rewrite = &msea_rewrite;
-  self->explain = &msea_explain;
-  self->explain_w = &msea_explain_w;
-  self->get_similarity = &msea_get_similarity;
-  self->close = &msea_close;
-  return self;
+
+    MSEA(self)->s_cnt           = s_cnt;
+    MSEA(self)->searchers       = searchers;
+    MSEA(self)->starts          = starts;
+    MSEA(self)->max_doc         = max_doc;
+    MSEA(self)->close_subs      = close_subs;
+
+    self->similarity            = sim_create_default();
+    self->doc_freq              = &msea_doc_freq;
+    self->get_doc               = &msea_get_doc;
+    self->get_lazy_doc          = &msea_get_lazy_doc;
+    self->max_doc               = &msea_max_doc;
+    self->create_weight         = &msea_create_weight;
+    self->search                = &msea_search;
+    self->search_w              = &msea_search_w;
+    self->search_each           = &msea_search_each;
+    self->search_each_w         = &msea_search_each_w;
+    self->rewrite               = &msea_rewrite;
+    self->explain               = &msea_explain;
+    self->explain_w             = &msea_explain_w;
+    self->get_term_vector       = &msea_get_term_vector;
+    self->get_similarity        = &msea_get_similarity;
+    self->close                 = &msea_close;
+    return self;
 }