ferret 0.9.6 → 0.10.0

data/ext/similarity.c

@@ -1,172 +1,150 @@
-#include <search.h>
-#include <global.h>
+#include "similarity.h"
+#include "search.h"
+#include "array.h"
+#include "helper.h"
 #include <math.h>
+#include <stdlib.h>
+#include <string.h>
 
-static int low_bit = 0, low_mid_bit = 0, high_mid_bit = 0, high_bit = 0;
-static void
-setup_endian()
-{
-    static int init = 0;
-    char *p;
-
-    if (init) return;
-    init = 1;
-    p = (char*)&init;
-
-    if (p[0]) {
-      low_bit = 0;
-      low_mid_bit = 1;
-      high_mid_bit = 2;
-      high_bit = 3;
-    } else {
-      low_bit = 3;
-      low_mid_bit = 2;
-      high_mid_bit = 1;
-      high_bit = 0;
-    }
-}
+/****************************************************************************
+ *
+ * Term
+ *
+ ****************************************************************************/
 
-float byte_to_float(uchar b)
+Term *term_new(const char *field, const char *text)
 {
-  char flt[4];
-  if (b == 0) {
-    return 0.0;
-  } else {
-    int mantissa = b & 0x07;           // 0x07 =  7 = 0b00000111
-    int exponent = (b >> 3) & 0x1F;    // 0x1f = 31 = 0b00011111
-
-    if (!low_mid_bit) setup_endian();
-    flt[low_bit] = flt[low_mid_bit] = 0;
-    flt[high_mid_bit] = mantissa << 5;
-    flt[high_bit] = exponent + 48;
-    return *((float *)flt);
-  }
+    Term *t = ALLOC(Term);
+    t->field = estrdup(field);
+    t->text = estrdup(text);
+    return t;
 }
 
-uchar float_to_byte(float f)
+void term_destroy(Term *self)
 {
-  if (f <= 0.0) {
-    return 0;
-  } else {
-    char *bits = (char *)&f;
-    int mantissa = (bits[high_mid_bit] & 0xEf) >> 5;
-    int exponent = (bits[high_bit] - 48);
-
-    if (exponent > 0x1f) {
-      exponent = 0x1f;   // 0x1f = 31 = 0b00011111
-      mantissa = 0x07;   // 0x07 =  7 = 0b00000111
-    }
+    free(self->text);
+    free(self->field);
+    free(self);
+}
 
-    if (exponent < 0) {
-      exponent = 0;
-      mantissa = 1;
-    }
+int term_eq(const void *t1, const void *t2)
+{
+    return (strcmp(((Term *)t1)->text, ((Term *)t2)->text)) == 0 &&
+        (strcmp(((Term *)t1)->field, ((Term *)t2)->field) == 0);
+}
 
-    return ((exponent<<3) | mantissa);
-  }
+ulong term_hash(const void *t)
+{
+    return str_hash(((Term *)t)->text) * str_hash(((Term *)t)->field);
 }
 
-float simdef_length_norm(Similarity *s, char *field, int num_terms)
+/****************************************************************************
+ *
+ * Similarity
+ *
+ ****************************************************************************/
+
+float simdef_length_norm(Similarity *s, const char *field, int num_terms)
 {
-  return (float)(1.0 / sqrt(num_terms));
+    (void)s;
+    (void)field;
+    return (float)(1.0 / sqrt(num_terms));
 }
 
 float simdef_query_norm(struct Similarity *s, float sum_of_squared_weights)
 {
-  return (float)(1.0 / sqrt(sum_of_squared_weights));
+    (void)s;
+    return (float)(1.0 / sqrt(sum_of_squared_weights));
 }
 
 float simdef_tf(struct Similarity *s, float freq)
 {
-  return (float)sqrt(freq);
+    (void)s;
+    return (float)sqrt(freq);
 }
 
 float simdef_sloppy_freq(struct Similarity *s, int distance)
 {
-  return (float)(1.0 / (double)(distance + 1));
+    (void)s;
+    return (float)(1.0 / (double)(distance + 1));
 }
 
-float simdef_idf_term(struct Similarity *s, Term *term, Searcher *searcher)
+float simdef_idf_term(struct Similarity *s, const char *field, char *term,
+                      Searcher *searcher)
 {
-  return s->idf(s, searcher->doc_freq(searcher, term), searcher->max_doc(searcher));
+    return s->idf(s, searcher->doc_freq(searcher, field, term),
+                  searcher->max_doc(searcher));
 }
 
-float simdef_idf_phrase(struct Similarity *s, Term **terms, int tcnt, Searcher *searcher)
+float simdef_idf_phrase(struct Similarity *s, const char *field,
+                        PhrasePosition *positions,
+                        int pp_cnt, Searcher *searcher)
 {
-  float idf = 0.0;
-  int i;
-  for (i = 0; i < tcnt; i++) {
-    idf += s->idf_term(s, terms[i], searcher);
-  }
-  return idf;
+    float idf = 0.0;
+    int i, j;
+    for (i = 0; i < pp_cnt; i++) {
+        char **terms = positions[i].terms;
+        for (j = ary_size(terms) - 1; j >= 0; j--) {
+            idf += sim_idf_term(s, field, terms[j], searcher);
+        }
+    }
+    return idf;
 }
 
 float simdef_idf(struct Similarity *s, int doc_freq, int num_docs)
 {
-  return (float)(log((float)num_docs/(float)(doc_freq+1)) + 1.0);
+    (void)s;
+    return (float)(log((float)num_docs/(float)(doc_freq+1)) + 1.0);
 }
 
 float simdef_coord(struct Similarity *s, int overlap, int max_overlap)
 {
-  return (float)((double)overlap / (double)max_overlap);
+    (void)s;
+    return (float)((double)overlap / (double)max_overlap);
 }
 
 float simdef_decode_norm(struct Similarity *s, uchar b)
 {
-  return s->norm_table[b];
+    return s->norm_table[b];
 }
 
 uchar simdef_encode_norm(struct Similarity *s, float f)
 {
-  return float_to_byte(f);
+    (void)s;
+    return float2byte(f);
 }
 
 void simdef_destroy(Similarity *s)
 {
-  /* nothing to do here */
+    (void)s;
+    /* nothing to do here */
 }
 
-#ifdef WIN32
 static Similarity default_similarity = {
-  NULL,
-  {0},
-  &simdef_length_norm,
-  &simdef_query_norm,
-  &simdef_tf,
-  &simdef_sloppy_freq,
-  &simdef_idf_term,
-  &simdef_idf_phrase,
-  &simdef_idf,
-  &simdef_coord,
-  &simdef_decode_norm,
-  &simdef_encode_norm,
-  &simdef_destroy
+    NULL,
+    {0},
+    &simdef_length_norm,
+    &simdef_query_norm,
+    &simdef_tf,
+    &simdef_sloppy_freq,
+    &simdef_idf_term,
+    &simdef_idf_phrase,
+    &simdef_idf,
+    &simdef_coord,
+    &simdef_decode_norm,
+    &simdef_encode_norm,
+    &simdef_destroy
 };
-#else
-static Similarity default_similarity = {
-  data:NULL,
-  length_norm:&simdef_length_norm,
-  query_norm:&simdef_query_norm,
-  tf:&simdef_tf,
-  sloppy_freq:&simdef_sloppy_freq,
-  idf_term:&simdef_idf_term,
-  idf_phrase:&simdef_idf_phrase,
-  idf:&simdef_idf,
-  coord:&simdef_coord,
-  decode_norm:&simdef_decode_norm,
-  encode_norm:&simdef_encode_norm,
-  destroy:&simdef_destroy
-};
-#endif
 
 Similarity *sim_create_default()
 {
-  int i;
-  if (!default_similarity.data) {
-    for (i = 0; i < 256; i++)
-      default_similarity.norm_table[i] = byte_to_float(i);
+    int i;
+    if (!default_similarity.data) {
+        for (i = 0; i < 256; i++) {
+            default_similarity.norm_table[i] = byte2float((unsigned char)i);
+        }
 
-    default_similarity.data = &default_similarity;
-  }
-  return &default_similarity;
+        default_similarity.data = &default_similarity;
+    }
+    return &default_similarity;
 }

data/ext/similarity.h

@@ -9,18 +9,31 @@ typedef struct Searcher Searcher;
  *
  ****************************************************************************/
 
-typedef struct Term {
-  char *field;
-  char *text;
+#define term_set_new() \
+    hs_new((hash_ft)&term_hash, (eq_ft)&term_eq, (free_ft)&term_destroy)
+
+typedef struct Term
+{
+    char *field;
+    char *text;
 } Term;
 
-Term *term_clone(Term *term);
-Term *term_create(const char *field, char *text);
-void term_destroy(Term *self);
-int term_cmp(void *t1, void *t2);
-int term_eq(const void *t1, const void *t2);
-unsigned int term_hash(const void *t);
-char *term_to_s(Term *term);
+extern Term *term_new(const char *field, const char *text);
+extern void term_destroy(Term *self);
+extern int term_eq(const void *t1, const void *t2);
+extern unsigned long term_hash(const void *t);
+
+/***************************************************************************
+ *
+ * PhrasePosition
+ *
+ ***************************************************************************/
+
+typedef struct PhrasePosition
+{
+    int pos;
+    char **terms;
+} PhrasePosition;
 
 /***************************************************************************
  *
@@ -30,38 +43,40 @@ char *term_to_s(Term *term);
 
 typedef struct Similarity Similarity;
 
-struct Similarity {
-  void *data;
-  float norm_table[256];
-  float (*length_norm)(Similarity *self, char *field, int num_terms);
-  float (*query_norm)(Similarity *self, float sum_of_squared_weights);
-  float (*tf)(Similarity *self, float freq);
-  float (*sloppy_freq)(Similarity *self, int distance);
-  float (*idf_term)(Similarity *self, Term *term, Searcher *searcher);
-  float (*idf_phrase)(Similarity *self, Term **terms,
-      int tcnt, Searcher *searcher);
-  float (*idf)(Similarity *self, int doc_freq, int num_docs);
-  float (*coord)(Similarity *self, int overlap, int max_overlap);
-  float (*decode_norm)(Similarity *self, uchar b);
-  uchar (*encode_norm)(Similarity *self, float f);
-  void  (*destroy)(Similarity *self);
+struct Similarity
+{
+    void *data;
+    float norm_table[256];
+    float (*length_norm)(Similarity *self, const char *field, int num_terms);
+    float (*query_norm)(Similarity *self, float sum_of_squared_weights);
+    float (*tf)(Similarity *self, float freq);
+    float (*sloppy_freq)(Similarity *self, int distance);
+    float (*idf_term)(Similarity *self, const char *field, char *term,
+                      Searcher *searcher);
+    float (*idf_phrase)(Similarity *self, const char *field,
+                        PhrasePosition *positions,
+                        int pp_cnt, Searcher *searcher);
+    float (*idf)(Similarity *self, int doc_freq, int num_docs);
+    float (*coord)(Similarity *self, int overlap, int max_overlap);
+    float (*decode_norm)(Similarity *self, unsigned char b);
+    unsigned char (*encode_norm)(Similarity *self, float f);
+    void  (*destroy)(Similarity *self);
 };
 
 #define sim_length_norm(msim, field, num_terms) msim->length_norm(msim, field, num_terms)
 #define sim_query_norm(msim, sosw) msim->query_norm(msim, sosw)
 #define sim_tf(msim, freq) msim->tf(msim, freq)
 #define sim_sloppy_freq(msim, distance) msim->sloppy_freq(msim, distance)
-#define sim_idf_term(msim, term, searcher) msim->idf_term(msim, term, searcher)
-#define sim_idf_phrase(msim, terms, tcnt, searcher) msim->idf_phrase(msim, terms, tcnt, searcher)
+#define sim_idf_term(msim, field, term, searcher)\
+    msim->idf_term(msim, field, term, searcher)
+#define sim_idf_phrase(msim, field, positions, pos_cnt, searcher)\
+    msim->idf_phrase(msim, field, positions, pos_cnt, searcher)
 #define sim_idf(msim, doc_freq, num_docs) msim->idf(msim, doc_freq, num_docs)
 #define sim_coord(msim, overlap, max_overlap) msim->coord(msim, overlap, max_overlap)
 #define sim_decode_norm(msim, b) msim->decode_norm(msim, b)
 #define sim_encode_norm(msim, f) msim->encode_norm(msim, f)
 #define sim_destroy(msim) msim->destroy(msim)
 
-float byte_to_float(uchar b);
-uchar float_to_byte(float f);
-
 Similarity *sim_create_default();
 
 #endif

data/ext/sort.c

@@ -2,94 +2,96 @@
 #include "search.h"
 #include "index.h"
 
-static char * const NO_TERM_ERROR_MSG = "no terms in field to sort by";
-
 /***************************************************************************
  *
  * SortField
  *
  ***************************************************************************/
 
-unsigned int sort_field_hash(const void *p)
+ulong sort_field_hash(const void *p)
 {
-  SortField *self = (SortField *)p;
-  return str_hash(self->field) ^ (self->type*37);
+    SortField *self = (SortField *)p;
+    return str_hash(self->field) ^ (self->type*37);
 }
 
 int sort_field_eq(const void *p1, const void *p2)
 {
-  SortField *key1 = (SortField *)p1;
-  SortField *key2 = (SortField *)p2;
-  int equal = (strcmp(key1->field, key2->field) == 0) && key1->type == key2->type;
-  /* 
-   * TODO: The could probable be done more cleanly.
-   * If the sort field is an auto field then it was evaluated before it was
-   * entered into the cache so we need to pass the compare function back to
-   * the new sort field.
-   */
-  if (equal && (key1->type == SORT_TYPE_AUTO)) {
-    key2->compare = key1->compare;
-  }
-  return equal;
-}
-
-SortField *sort_field_clone(SortField *self)
-{
-  SortField *clone = ALLOC(SortField);
-  memcpy(clone, self, sizeof(SortField));
-  mutex_init(&clone->mutex, NULL);
-  clone->field = estrdup(self->field);
-  return clone;
-}
-
-SortField *sort_field_alloc(char *field, int type, bool reverse)
-{
-  SortField *self = ALLOC(SortField);
-  mutex_init(&self->mutex, NULL);
-  self->field = field ? estrdup(field) : NULL;
-  self->type = type;
-  self->reverse = reverse;
-  self->index = NULL;
-  self->destroy_index = &free;
-  self->compare = NULL;
-  return self;
-}
-
-SortField *sort_field_create(char *field, int type, bool reverse)
-{
-  SortField *sf = NULL;
-  switch (type) {
-    case SORT_TYPE_SCORE:
-      sf = sort_field_score_create(reverse);
-      break;
-    case SORT_TYPE_DOC:
-      sf = sort_field_doc_create(reverse);
-      break;
-    case SORT_TYPE_INTEGER:
-      sf = sort_field_int_create(field, reverse);
-      break;
-    case SORT_TYPE_FLOAT:
-      sf = sort_field_float_create(field, reverse);
-      break;
-    case SORT_TYPE_STRING:
-      sf = sort_field_string_create(field, reverse);
-      break;
-    case SORT_TYPE_AUTO:
-      sf = sort_field_auto_create(field, reverse);
-      break;
-  }
-  return sf;
+    SortField *key1 = (SortField *)p1;
+    SortField *key2 = (SortField *)p2;
+    return (strcmp(key1->field, key2->field) == 0)
+        && key1->type == key2->type;
+}
+
+static int sort_field_cache_eq(const void *p1, const void *p2)
+{
+    SortField *key1 = (SortField *)p1;
+    SortField *key2 = (SortField *)p2;
+    int equal = (strcmp(key1->field, key2->field) == 0)
+        && key1->type == key2->type;
+
+    return equal;
+}
+
+static SortField *sort_field_clone(SortField *self)
+{
+    SortField *clone = ALLOC(SortField);
+    memcpy(clone, self, sizeof(SortField));
+    mutex_init(&clone->mutex, NULL);
+    clone->field = estrdup(self->field);
+    return clone;
+}
+
+static SortField *sort_field_alloc(char *field, int type, bool reverse)
+{
+    SortField *self = ALLOC(SortField);
+    mutex_init(&self->mutex, NULL);
+    self->field         = field ? estrdup(field) : NULL;
+    self->type          = type;
+    self->reverse       = reverse;
+    self->index         = NULL;
+    self->destroy_index = &free;
+    self->compare       = NULL;
+    return self;
+}
+
+SortField *sort_field_new(char *field, enum SORT_TYPE type, bool reverse)
+{
+    SortField *sf = NULL;
+    switch (type) {
+        case SORT_TYPE_SCORE:
+            sf = sort_field_score_new(reverse);
+            break;
+        case SORT_TYPE_DOC:
+            sf = sort_field_doc_new(reverse);
+            break;
+        case SORT_TYPE_BYTE:
+            sf = sort_field_byte_new(field, reverse);
+            break;
+        case SORT_TYPE_INTEGER:
+            sf = sort_field_int_new(field, reverse);
+            break;
+        case SORT_TYPE_FLOAT:
+            sf = sort_field_float_new(field, reverse);
+            break;
+        case SORT_TYPE_STRING:
+            sf = sort_field_string_new(field, reverse);
+            break;
+        case SORT_TYPE_AUTO:
+            sf = sort_field_auto_new(field, reverse);
+            break;
+    }
+    return sf;
 }
 
 void sort_field_destroy(void *p)
 {
-  SortField *self = (SortField *)p;
-  if (self->index) {
-    self->destroy_index(self->index);
-  }
-  free(self->field);
-  mutex_destroy(&self->mutex);
-  free(p);
+    SortField *self = (SortField *)p;
+    if (self->index) {
+        self->destroy_index(self->index);
+    }
+    free(self->field);
+    mutex_destroy(&self->mutex);
+    free(p);
 }
 
 /*
@@ -97,210 +99,304 @@ void sort_field_destroy(void *p)
  */
 char *sort_field_to_s(SortField *self)
 {
-  char *str;
-  char *type = NULL;
-  switch (self->type) {
-    case SORT_TYPE_SCORE:
-      type = "<SCORE>";
-      break;
-    case SORT_TYPE_DOC:
-      type = "<DOC>";
-      break;
-    case SORT_TYPE_INTEGER:
-      type = "<integer>";
-      break;
-    case SORT_TYPE_FLOAT:
-      type = "<float>";
-      break;
-    case SORT_TYPE_STRING:
-      type = "<string>";
-      break;
-    case SORT_TYPE_AUTO:
-      type = "<auto>";
-      break;
-  }
-  if (self->field) {
-    str = ALLOC_N(char, 10 + strlen(self->field) + strlen(type));
-    sprintf(str, "%s:%s%s", self->field, type, (self->reverse ? "!" : ""));
-  } else {
-    str = ALLOC_N(char, 10 + strlen(type));
-    sprintf(str, "%s%s", type, (self->reverse ? "!" : ""));
-  }
-  return str;
+    char *str;
+    char *type = NULL;
+    switch (self->type) {
+        case SORT_TYPE_SCORE:
+            type = "<SCORE>";
+            break;
+        case SORT_TYPE_DOC:
+            type = "<DOC>";
+            break;
+        case SORT_TYPE_BYTE:
+            type = "<byte>";
+            break;
+        case SORT_TYPE_INTEGER:
+            type = "<integer>";
+            break;
+        case SORT_TYPE_FLOAT:
+            type = "<float>";
+            break;
+        case SORT_TYPE_STRING:
+            type = "<string>";
+            break;
+        case SORT_TYPE_AUTO:
+            type = "<auto>";
+            break;
+    }
+    if (self->field) {
+        str = ALLOC_N(char, 10 + strlen(self->field) + strlen(type));
+        sprintf(str, "%s:%s%s", self->field, type, (self->reverse ? "!" : ""));
+    } else {
+        str = ALLOC_N(char, 10 + strlen(type));
+        sprintf(str, "%s%s", type, (self->reverse ? "!" : ""));
+    }
+    return str;
 }
 
 /***************************************************************************
  * ScoreSortField
  ***************************************************************************/
 
+void sf_score_get_val(void *index, Hit *hit, Comparable *comparable)
+{
+    (void)index;
+    comparable->val.f = hit->score;
+}
+
 int sf_score_compare(void *index_ptr, Hit *hit2, Hit *hit1)
 {
-  float val1 = hit1->score;
-  float val2 = hit2->score;
-  if (val1 > val2) return 1;
-  else if (val1 < val2) return -1;
-  else return 0;
+    float val1 = hit1->score;
+    float val2 = hit2->score;
+    (void)index_ptr;
+
+    if (val1 > val2) return 1;
+    else if (val1 < val2) return -1;
+    else return 0;
 }
 
-SortField *sort_field_score_create(bool reverse)
+SortField *sort_field_score_new(bool reverse)
 {
-  SortField *self = sort_field_alloc(NULL, SORT_TYPE_SCORE, reverse);
-  self->compare = &sf_score_compare;
-  return self;
+    SortField *self = sort_field_alloc(NULL, SORT_TYPE_SCORE, reverse);
+    self->compare   = &sf_score_compare;
+    self->get_val   = &sf_score_get_val;
+    return self;
 }
 
-SortField SORT_FIELD_SCORE = {
-  MUTEX_INITIALIZER,
-  /* field */NULL,
-  /* type */SORT_TYPE_SCORE,
-  /* reverse */false,
-  /* index */NULL,
-  /* compare */&sf_score_compare,
-  /* create_index */NULL,
-  /* destroy_index */NULL,
-  /* handle_term */NULL
+const SortField SORT_FIELD_SCORE = {
+    MUTEX_INITIALIZER,
+    NULL,               /* field */
+    SORT_TYPE_SCORE,    /* type */
+    false,              /* reverse */
+    NULL,               /* index */
+    &sf_score_compare,  /* compare */
+    &sf_score_get_val,  /* get_val */
+    NULL,               /* create_index */
+    NULL,               /* destroy_index */
+    NULL,               /* handle_term */
 };
 
-SortField SORT_FIELD_SCORE_REV = {
-  MUTEX_INITIALIZER,
-  /* field */NULL,
-  /* type */SORT_TYPE_SCORE,
-  /* reverse */true,
-  /* index */NULL,
-  /* compare */&sf_score_compare,
-  /* create_index */NULL,
-  /* destroy_index */NULL,
-  /* handle_term */NULL
+const SortField SORT_FIELD_SCORE_REV = {
+    MUTEX_INITIALIZER,
+    NULL,               /* field */
+    SORT_TYPE_SCORE,    /* type */
+    true,               /* reverse */
+    NULL,               /* index */
+    &sf_score_compare,  /* compare */
+    &sf_score_get_val,  /* get_val */
+    NULL,               /* create_index */
+    NULL,               /* destroy_index */
+    NULL,               /* handle_term */
 };
 
 /**************************************************************************
  * DocSortField
  ***************************************************************************/
 
+void sf_doc_get_val(void *index, Hit *hit, Comparable *comparable)
+{
+    (void)index;
+    comparable->val.i = hit->doc;
+}
+
 int sf_doc_compare(void *index_ptr, Hit *hit1, Hit *hit2)
 {
-  int val1 = hit1->doc;
-  int val2 = hit2->doc;
-  if (val1 > val2) return 1;
-  else if (val1 < val2) return -1;
-  else return 0;
+    int val1 = hit1->doc;
+    int val2 = hit2->doc;
+    (void)index_ptr;
+
+    if (val1 > val2) return 1;
+    else if (val1 < val2) return -1;
+    else return 0;
 }
 
-SortField *sort_field_doc_create(bool reverse)
+SortField *sort_field_doc_new(bool reverse)
 {
-  SortField *self = sort_field_alloc(NULL, SORT_TYPE_DOC, reverse);
-  self->compare = &sf_doc_compare;
-  return self;
+    SortField *self = sort_field_alloc(NULL, SORT_TYPE_DOC, reverse);
+    self->compare   = &sf_doc_compare;
+    self->get_val   = &sf_doc_get_val;
+    return self;
 }
 
-SortField SORT_FIELD_DOC = {
-  MUTEX_INITIALIZER,
-  /* field */NULL,
-  /* type */SORT_TYPE_DOC,
-  /* reverse */false,
-  /* index */NULL,
-  /* compare */&sf_doc_compare,
-  /* create_index */NULL,
-  /* destroy_index */NULL,
-  /* handle_term */NULL
+const SortField SORT_FIELD_DOC = {
+    MUTEX_INITIALIZER,
+    NULL,               /* field */
+    SORT_TYPE_DOC,      /* type */
+    false,              /* reverse */
+    NULL,               /* index */
+    &sf_doc_compare,    /* compare */
+    &sf_doc_get_val,    /* get_val */
+    NULL,               /* create_index */
+    NULL,               /* destroy_index */
+    NULL,               /* handle_term */
 };
 
-SortField SORT_FIELD_DOC_REV = {
-  MUTEX_INITIALIZER,
-  /* field */NULL,
-  /* type */SORT_TYPE_DOC,
-  /* reverse */true,
-  /* index */NULL,
-  /* compare */&sf_doc_compare,
-  /* create_index */NULL,
-  /* destroy_index */NULL,
-  /* handle_term */NULL
+const SortField SORT_FIELD_DOC_REV = {
+    MUTEX_INITIALIZER,
+    NULL,               /* field */
+    SORT_TYPE_DOC,      /* type */
+    true,               /* reverse */
+    NULL,               /* index */
+    &sf_doc_compare,    /* compare */
+    &sf_doc_get_val,    /* get_val */
+    NULL,               /* create_index */
+    NULL,               /* destroy_index */
+    NULL,               /* handle_term */
 };
 
+/***************************************************************************
+ * ByteSortField
+ ***************************************************************************/
+
+static void sf_byte_get_val(void *index, Hit *hit, Comparable *comparable)
+{
+    comparable->val.i = ((int *)index)[hit->doc];
+}
+
+static int sf_byte_compare(void *index, Hit *hit1, Hit *hit2)
+{
+    int val1 = ((int *)index)[hit1->doc];
+    int val2 = ((int *)index)[hit2->doc];
+    if (val1 > val2) return 1;
+    else if (val1 < val2) return -1;
+    else return 0;
+}
+
+static void *sf_byte_create_index(int size)
+{
+    int *index = ALLOC_AND_ZERO_N(int, size + 1);
+    return &index[1];
+}
+
+static void sf_byte_destroy_index(void *p)
+{
+    int *index = (int *)p;
+    free(&index[-1]);
+}
+
+static void sf_byte_handle_term(void *index_ptr, TermDocEnum *tde, char *text)
+{
+    int *index = (int *)index_ptr;
+    int val = index[-1]++;
+    (void)text;
+    while (tde->next(tde)) {
+        index[tde->doc_num(tde)] = val;
+    }
+}
+
+static void sort_field_byte_methods(SortField *self)
+{
+    self->type          = SORT_TYPE_BYTE;
+    self->compare       = &sf_byte_compare;
+    self->get_val       = &sf_byte_get_val;
+    self->create_index  = &sf_byte_create_index;
+    self->destroy_index = &sf_byte_destroy_index;
+    self->handle_term   = &sf_byte_handle_term;
+}
+
+SortField *sort_field_byte_new(char *field, bool reverse)
+{
+    SortField *self = sort_field_alloc(field, SORT_TYPE_BYTE, reverse);
+    sort_field_byte_methods(self);
+    return self;
+}
+
 /***************************************************************************
  * IntegerSortField
  ***************************************************************************/
 
-int sf_int_compare(void *index_ptr, Hit *hit1, Hit *hit2)
+void sf_int_get_val(void *index, Hit *hit, Comparable *comparable)
 {
-  int *index = (int *)index_ptr;
-  int val1 = index[hit1->doc];
-  int val2 = index[hit2->doc];
-  if (val1 > val2) return 1;
-  else if (val1 < val2) return -1;
-  else return 0;
+    comparable->val.i = ((int *)index)[hit->doc];
+}
+
+int sf_int_compare(void *index, Hit *hit1, Hit *hit2)
+{
+    int val1 = ((int *)index)[hit1->doc];
+    int val2 = ((int *)index)[hit2->doc];
+    if (val1 > val2) return 1;
+    else if (val1 < val2) return -1;
+    else return 0;
 }
 
 void *sf_int_create_index(int size)
 {
-  return ALLOC_N(int, size);
+    return ALLOC_AND_ZERO_N(int, size);
 }
 
 void sf_int_handle_term(void *index_ptr, TermDocEnum *tde, char *text)
 {
-  int *index = (int *)index_ptr;
-  int val;
-  sscanf(text, "%d", &val);
-  while (tde->next(tde)) {
-    index[tde->doc_num(tde)] = val;
-  }
+    int *index = (int *)index_ptr;
+    int val;
+    sscanf(text, "%d", &val);
+    while (tde->next(tde)) {
+        index[tde->doc_num(tde)] = val;
+    }
 }
 
 void sort_field_int_methods(SortField *self)
 {
-  self->compare = &sf_int_compare;
-  self->create_index = &sf_int_create_index;
-  self->handle_term = &sf_int_handle_term;
+    self->type          = SORT_TYPE_INTEGER;
+    self->compare       = &sf_int_compare;
+    self->get_val       = &sf_int_get_val;
+    self->create_index  = &sf_int_create_index;
+    self->handle_term   = &sf_int_handle_term;
 }
 
-SortField *sort_field_int_create(char *field, bool reverse)
+SortField *sort_field_int_new(char *field, bool reverse)
 {
-  SortField *self = sort_field_alloc(field, SORT_TYPE_INTEGER, reverse);
-  sort_field_int_methods(self);
-  return self;
+    SortField *self = sort_field_alloc(field, SORT_TYPE_INTEGER, reverse);
+    sort_field_int_methods(self);
+    return self;
 }
 
 /***************************************************************************
  * FloatSortField
  ***************************************************************************/
 
-int sf_float_compare(void *index_ptr, Hit *hit1, Hit *hit2)
+void sf_float_get_val(void *index, Hit *hit, Comparable *comparable)
 {
-  float *index = (float *)index_ptr;
-  float val1 = index[hit1->doc];
-  float val2 = index[hit2->doc];
-  if (val1 > val2) return 1;
-  else if (val1 < val2) return -1;
-  else return 0;
+    comparable->val.f = ((float *)index)[hit->doc];
+}
+
+int sf_float_compare(void *index, Hit *hit1, Hit *hit2)
+{
+    float val1 = ((float *)index)[hit1->doc];
+    float val2 = ((float *)index)[hit2->doc];
+    if (val1 > val2) return 1;
+    else if (val1 < val2) return -1;
+    else return 0;
 }
 
 void *sf_float_create_index(int size)
 {
-  return ALLOC_N(float, size);
+    return ALLOC_AND_ZERO_N(float, size);
 }
 
 void sf_float_handle_term(void *index_ptr, TermDocEnum *tde, char *text)
 {
-  float *index = (float *)index_ptr;
-  float val;
-  sscanf(text, "%g", &val);
-  while (tde->next(tde)) {
-    index[tde->doc_num(tde)] = val;
-  }
+    float *index = (float *)index_ptr;
+    float val;
+    sscanf(text, "%g", &val);
+    while (tde->next(tde)) {
+        index[tde->doc_num(tde)] = val;
+    }
 }
 
 void sort_field_float_methods(SortField *self)
 {
-  self->compare = &sf_float_compare;
-  self->create_index = &sf_float_create_index;
-  self->handle_term = &sf_float_handle_term;
+    self->type          = SORT_TYPE_FLOAT;
+    self->compare       = &sf_float_compare;
+    self->get_val       = &sf_float_get_val;
+    self->create_index  = &sf_float_create_index;
+    self->handle_term   = &sf_float_handle_term;
 }
 
-SortField *sort_field_float_create(char *field, bool reverse)
+SortField *sort_field_float_new(char *field, bool reverse)
 {
-  SortField *self = sort_field_alloc(field, SORT_TYPE_FLOAT, reverse);
-  sort_field_float_methods(self);
-  return self;
+    SortField *self = sort_field_alloc(field, SORT_TYPE_FLOAT, reverse);
+    sort_field_float_methods(self);
+    return self;
 }
 
 /***************************************************************************
@@ -309,78 +405,99 @@ SortField *sort_field_float_create(char *field, bool reverse)
 
 #define VALUES_ARRAY_START_SIZE 8
 typedef struct StringIndex {
-  int size;
-  int *index;
-  char **values;
-  int v_cnt;
-  int v_size;
+    int size;
+    int *index;
+    char **values;
+    int v_size;
+    int v_capa;
 } StringIndex;
 
-int sf_string_compare(void *index_ptr, Hit *hit1, Hit *hit2)
+void sf_string_get_val(void *index, Hit *hit, Comparable *comparable)
 {
-  StringIndex *index = (StringIndex *)index_ptr;
-  return strcoll(index->values[index->index[hit1->doc]],
-    index->values[index->index[hit2->doc]]);
-  /*
-  int val1 = index->index[hit1->doc];
-  int val2 = index->index[hit2->doc];
-  if (val1 > val2) return 1;
-  else if (val1 < val2) return -1;
-  else return 0;
-  */
+    comparable->val.s
+        = ((StringIndex *)index)->values[
+        ((StringIndex *)index)->index[hit->doc]];
+}
+
+int sf_string_compare(void *index, Hit *hit1, Hit *hit2)
+{
+    char *s1 = ((StringIndex *)index)->values[
+        ((StringIndex *)index)->index[hit1->doc]];
+    char *s2 = ((StringIndex *)index)->values[
+        ((StringIndex *)index)->index[hit2->doc]];
+
+    if (s1 == NULL) return s1 ? -1 : 0;
+    if (s2 == NULL) return 1;
+
+#ifdef POSH_OS_WIN32
+    return strcmp(s1, s2);
+#else
+    return strcoll(s1, s2);
+#endif
+
+    /*
+     * TODO: investigate whether it would be a good idea to presort strings.
+     *
+    int val1 = index->index[hit1->doc];
+    int val2 = index->index[hit2->doc];
+    if (val1 > val2) return 1;
+    else if (val1 < val2) return -1;
+    else return 0;
+    */
 }
 
 void *sf_string_create_index(int size)
 {
-  StringIndex *self = ALLOC(StringIndex);
-  ZEROSET(self, StringIndex, 1);
-  self->size = size;
-  self->index = ALLOC_N(int, size);
-  ZEROSET(self->index, int, size);
-  self->v_size = VALUES_ARRAY_START_SIZE;
-  self->values = ALLOC_N(char *, VALUES_ARRAY_START_SIZE);
-  return self;
+    StringIndex *self = ALLOC_AND_ZERO(StringIndex);
+    self->size = size;
+    self->index = ALLOC_AND_ZERO_N(int, size);
+    self->v_capa = VALUES_ARRAY_START_SIZE;
+    self->v_size = 1; /* leave the first value as NULL */
+    self->values = ALLOC_AND_ZERO_N(char *, VALUES_ARRAY_START_SIZE);
+    return self;
 }
 
 void sf_string_destroy_index(void *p)
 {
-  StringIndex *self = (StringIndex *)p;
-  int i;
-  free(self->index);
-  for (i = 0; i < self->v_cnt; i++) {
-    free(self->values[i]);
-  }
-  free(self->values);
-  free(self);
+    StringIndex *self = (StringIndex *)p;
+    int i;
+    free(self->index);
+    for (i = 0; i < self->v_size; i++) {
+        free(self->values[i]);
+    }
+    free(self->values);
+    free(self);
 }
 
 void sf_string_handle_term(void *index_ptr, TermDocEnum *tde, char *text)
 {
-  StringIndex *index = (StringIndex *)index_ptr;
-  if (index->v_cnt >= index->v_size) {
-    index->v_size *= 2;
-    index->values = REALLOC_N(index->values, char *, index->v_size);
-  }
-  index->values[index->v_cnt] = estrdup(text);
-  while (tde->next(tde)) {
-    index->index[tde->doc_num(tde)] = index->v_cnt;
-  }
-  index->v_cnt++;
+    StringIndex *index = (StringIndex *)index_ptr;
+    if (index->v_size >= index->v_capa) {
+        index->v_capa *= 2;
+        index->values = REALLOC_N(index->values, char *, index->v_capa);
+    }
+    index->values[index->v_size] = estrdup(text);
+    while (tde->next(tde)) {
+        index->index[tde->doc_num(tde)] = index->v_size;
+    }
+    index->v_size++;
 }
 
 void sort_field_string_methods(SortField *self)
 {
-  self->compare = &sf_string_compare;
-  self->create_index = &sf_string_create_index;
-  self->destroy_index = &sf_string_destroy_index;
-  self->handle_term = &sf_string_handle_term;
+    self->type          = SORT_TYPE_STRING;
+    self->compare       = &sf_string_compare;
+    self->get_val       = &sf_string_get_val;
+    self->create_index  = &sf_string_create_index;
+    self->destroy_index = &sf_string_destroy_index;
+    self->handle_term   = &sf_string_handle_term;
 }
 
-SortField *sort_field_string_create(char *field, bool reverse)
+SortField *sort_field_string_new(char *field, bool reverse)
 {
-  SortField *self = sort_field_alloc(field, SORT_TYPE_STRING, reverse);
-  sort_field_string_methods(self);
-  return self;
+    SortField *self = sort_field_alloc(field, SORT_TYPE_STRING, reverse);
+    sort_field_string_methods(self);
+    return self;
 }
 
 /***************************************************************************
@@ -389,27 +506,27 @@ SortField *sort_field_string_create(char *field, bool reverse)
 
 void sort_field_auto_evaluate(SortField *sf, char *text)
 {
-  int int_val;
-  float float_val;
-  size_t text_len = 0, scan_len = 0;
+    int int_val;
+    float float_val;
+    int text_len = 0, scan_len = 0;
 
-  text_len = strlen(text);
-  sscanf(text, "%d%n", &int_val, &scan_len);
-  if (scan_len == text_len) {
-    sort_field_int_methods(sf);
-  } else {
-    sscanf(text, "%f%n", &float_val, &scan_len);
+    text_len = (int)strlen(text);
+    sscanf(text, "%d%n", &int_val, &scan_len);
     if (scan_len == text_len) {
-      sort_field_float_methods(sf);
+        sort_field_int_methods(sf);
     } else {
-      sort_field_string_methods(sf);
+        sscanf(text, "%f%n", &float_val, &scan_len);
+        if (scan_len == text_len) {
+            sort_field_float_methods(sf);
+        } else {
+            sort_field_string_methods(sf);
+        }
     }
-  }
 }
 
-SortField *sort_field_auto_create(char *field, bool reverse)
+SortField *sort_field_auto_new(char *field, bool reverse)
 {
-  return sort_field_alloc(field, SORT_TYPE_AUTO, reverse);
+    return sort_field_alloc(field, SORT_TYPE_AUTO, reverse);
 }
 
 /***************************************************************************
@@ -420,58 +537,60 @@ SortField *sort_field_auto_create(char *field, bool reverse)
 
 void *field_cache_get_index(IndexReader *ir, SortField *sf)
 {
-  void *index = NULL;
-  int length = 0;
-  Term term;
-  TermBuffer *tb;
-  TermEnum *volatile te = NULL;
-  TermDocEnum *volatile tde = NULL;
-  char *field = sf->field;
-  SortField *sf_clone;
-
-  mutex_lock(&sf->mutex);
-  if (!ir->sort_cache) {
-    ir->sort_cache = h_new(&sort_field_hash, &sort_field_eq,
-        &sort_field_destroy, NULL);
-  }
-  index = h_get(ir->sort_cache, sf);
-
-  if (index == NULL) {
-    length = ir->max_doc(ir);
-    if (length > 0) {
-      TRY
-        tde = ir->term_docs(ir);
-        term.field = field;
-        term.text = "";
-        te = ir->terms_from(ir, &term);
-        if (te->tb_curr == NULL) {
-          RAISE(ARG_ERROR, NO_TERM_ERROR_MSG);
-        }
+    void *index = NULL;
+    int length = 0;
+    TermEnum *volatile te = NULL;
+    TermDocEnum *volatile tde = NULL;
+    SortField *sf_clone;
+    const int field_num = fis_get_field_num(ir->fis, sf->field);
+
+    if (field_num < 0) {
+        RAISE(ARG_ERROR,
+              "Cannot sort by field \"%s\". It doesn't exist in the index.",
+              sf->field);
+    }
 
-        if (sf->type == SORT_TYPE_AUTO) {
-          sort_field_auto_evaluate(sf, te->tb_curr->text);
-        }
+    mutex_lock(&sf->mutex);
+    if (!ir->sort_cache) {
+        ir->sort_cache = h_new(&sort_field_hash, &sort_field_cache_eq,
+                               &sort_field_destroy, NULL);
+    }
 
-        index = sf->create_index(length);
-
-        do { 
-          tb = te->tb_curr;
-          if (strcmp(tb->field, field) != 0) break;
-          term.text = tb->text;
-          tde->seek(tde, &term);
-          sf->handle_term(index, tde, tb->text);
-        } while (te->next(te));
-      XFINALLY
-        tde->close(tde);
+    if (sf->type == SORT_TYPE_AUTO) {
+        te = ir->terms(ir, field_num);
+        if (!te->next(te)) {
+            RAISE(ARG_ERROR,
+                  "Cannot sort by field \"%s\" as there are no terms "
+                  "in that field in the index.", sf->field);
+        }
+        sort_field_auto_evaluate(sf, te->curr_term);
         te->close(te);
-      XENDTRY
     }
-    sf_clone = sort_field_clone(sf);
-    sf_clone->index = index;
-    h_set(ir->sort_cache, sf_clone, index);
-  }
-  mutex_unlock(&sf->mutex);
-  return index;
+
+    index = h_get(ir->sort_cache, sf);
+
+    if (index == NULL) {
+        length = ir->max_doc(ir);
+        if (length > 0) {
+            TRY
+                tde = ir->term_docs(ir);
+                te = ir->terms(ir, field_num);
+                index = sf->create_index(length);
+                while (te->next(te)) {
+                    tde->seek_te(tde, te);
+                    sf->handle_term(index, tde, te->curr_term);
+                }
+            XFINALLY
+                tde->close(tde);
+            te->close(te);
+            XENDTRY
+        }
+        sf_clone = sort_field_clone(sf);
+        sf_clone->index = index;
+        h_set(ir->sort_cache, sf_clone, index);
+    }
+    mutex_unlock(&sf->mutex);
+    return index;
 }
 
 /***************************************************************************
@@ -485,19 +604,19 @@ void *field_cache_get_index(IndexReader *ir, SortField *sf)
  ***************************************************************************/
 
 typedef struct Comparator {
-  void *index;
-  bool  reverse : 1;
-  int   (*compare)(void *index_ptr, Hit *hit1, Hit *hit2);
+    void *index;
+    bool  reverse : 1;
+    int   (*compare)(void *index_ptr, Hit *hit1, Hit *hit2);
 } Comparator;
 
-Comparator *comparator_create(void *index, bool reverse,
-    int (*compare)(void *index_ptr, Hit *hit1, Hit *hit2))
+Comparator *comparator_new(void *index, bool reverse,
+                  int (*compare)(void *index_ptr, Hit *hit1, Hit *hit2))
 {
-  Comparator *self = ALLOC(Comparator);
-  self->index = index;
-  self->reverse = reverse;
-  self->compare = compare;
-  return self;
+    Comparator *self = ALLOC(Comparator);
+    self->index = index;
+    self->reverse = reverse;
+    self->compare = compare;
+    return self;
 }
 
 /***************************************************************************
@@ -505,164 +624,279 @@ Comparator *comparator_create(void *index, bool reverse,
  ***************************************************************************/
 
 typedef struct Sorter {
-  Comparator **comparators;
-  int c_cnt;
+    Comparator **comparators;
+    int c_cnt;
+    Sort *sort;
 } Sorter;
 
 Comparator *sorter_get_comparator(SortField *sf, IndexReader *ir)
 {
-  void *index = NULL;
+    void *index = NULL;
 
-  if (sf->type > SORT_TYPE_DOC) {
-    index = field_cache_get_index(ir, sf);
-  }
-  return comparator_create(index, sf->reverse, sf->compare);
+    if (sf->type > SORT_TYPE_DOC) {
+        index = field_cache_get_index(ir, sf);
+    }
+    return comparator_new(index, sf->reverse, sf->compare);
 }
 
-void sorter_destroy(void *p)
+void sorter_destroy(Sorter *self)
 {
-  int i;
-  Sorter *self = (Sorter *)p;
+    int i;
 
-  for (i = 0; i < self->c_cnt; i++) {
-    free(self->comparators[i]);
-  }
-  free(self->comparators);
-  free(self);
+    for (i = 0; i < self->c_cnt; i++) {
+        free(self->comparators[i]);
+    }
+    free(self->comparators);
+    free(self);
 }
 
-Sorter *sorter_create(int size)
+Sorter *sorter_new(Sort *sort)
 {
-  Sorter *self = ALLOC(Sorter);
-  self->c_cnt = size;
-  self->comparators = ALLOC_N(Comparator *, size);
-  ZEROSET(self->comparators, Comparator *, size);
-  return self;
+    Sorter *self = ALLOC(Sorter);
+    self->c_cnt = sort->size;
+    self->comparators = ALLOC_AND_ZERO_N(Comparator *, self->c_cnt);
+    self->sort = sort;
+    return self;
 }
 
 /***************************************************************************
  * FieldSortedHitQueue
  ***************************************************************************/
 
-bool fshq_less_than(void *hit1, void *hit2)
+bool fshq_less_than(const void *hit1, const void *hit2)
 {
-  int cmp = 0;
-  printf("Whoops, shouldn't call this.\n");
-  if (cmp != 0) {
-    return cmp;
-  } else {
-    return ((Hit *)hit1)->score < ((Hit *)hit2)->score;
-  }
+    int cmp = 0;
+    printf("Whoops, shouldn't call this.\n");
+    if (cmp != 0) {
+        return cmp;
+    } else {
+        return ((Hit *)hit1)->score < ((Hit *)hit2)->score;
+    }
 }
 
-inline bool fshq_lt(Hit *sorter_ptr, Hit *hit1, Hit *hit2)
+inline bool fshq_lt(Sorter *sorter, Hit *hit1, Hit *hit2)
 {
-  Sorter *sorter = (Sorter *)sorter_ptr;
-  Comparator *comp;
-  int diff = 0, i;
-  for (i = 0; i < sorter->c_cnt && diff == 0; i++) {
-    comp = sorter->comparators[i];
-    if (comp->reverse) {
-      diff = comp->compare(comp->index, hit2, hit1);
-    } else {
-      diff = comp->compare(comp->index, hit1, hit2);
+    Comparator *comp;
+    int diff = 0, i;
+    for (i = 0; i < sorter->c_cnt && diff == 0; i++) {
+        comp = sorter->comparators[i];
+        if (comp->reverse) {
+            diff = comp->compare(comp->index, hit2, hit1);
+        } else {
+            diff = comp->compare(comp->index, hit1, hit2);
+        }
     }
-  }
 
-  if (diff != 0) {
-    return diff > 0;
-  } else {
-    return hit1->doc > hit2->doc;
-  }
+    if (diff != 0) {
+        return diff > 0;
+    } else {
+        return hit1->doc > hit2->doc;
+    }
 }
 
 void fshq_pq_down(PriorityQueue *pq)
 {
-  register int i = 1;
-  register int j = 2; //i << 1;
-  register int k = 3; //j + 1;
-  Hit **heap = (Hit **)pq->heap;
-  Hit *node = heap[i]; // save top node
+    register int i = 1;
+    register int j = 2;     /* i << 1; */
+    register int k = 3;     /* j + 1; */
+    Hit **heap = (Hit **)pq->heap;
+    Hit *node = heap[i];    /* save top node */
+    Sorter *sorter = (Sorter *)heap[0];
 
-  if ((k <= pq->count) && fshq_lt(heap[0], heap[k], heap[j]))
-    j = k;
+    if ((k <= pq->size) && fshq_lt(sorter, heap[k], heap[j])) {
+        j = k;
+    }
 
-  while ((j <= pq->count) && fshq_lt(heap[0], heap[j], node)) {
-    heap[i] = heap[j];      // shift up child
-    i = j;
-    j = i << 1;
-    k = j + 1;
-    if ((k <= pq->count) && fshq_lt(heap[0], heap[k], heap[j]))
-      j = k;
-  }
-  heap[i] = node;
+    while ((j <= pq->size) && fshq_lt(sorter, heap[j], node)) {
+        heap[i] = heap[j];  /* shift up child */
+        i = j;
+        j = i << 1;
+        k = j + 1;
+        if ((k <= pq->size) && fshq_lt(sorter, heap[k], heap[j])) {
+            j = k;
+        }
+    }
+    heap[i] = node;
 }
 
 Hit *fshq_pq_pop(PriorityQueue *pq)
 {
-  if (pq->count > 0) {
-    Hit *result = (Hit *)pq->heap[1];  // save first value
-    pq->heap[1] = pq->heap[pq->count]; // move last to first
-    pq->heap[pq->count] = NULL;
-    pq->count--;
-    fshq_pq_down(pq);                   // adjust heap
-    return result;
-  } else {
-    return NULL;
-  }
+    if (pq->size > 0) {
+        Hit *hit = (Hit *)pq->heap[1];   /* save first value */
+        pq->heap[1] = pq->heap[pq->size];   /* move last to first */
+        pq->heap[pq->size] = NULL;
+        pq->size--;
+        fshq_pq_down(pq);                   /* adjust heap */
+        return hit;
+    } else {
+        return NULL;
+    }
 }
 
 inline void fshq_pq_up(PriorityQueue *pq)
 {
-  Hit **heap = (Hit **)pq->heap;
-  Hit *node;
-	int i = pq->count;
-	int j = i >> 1;
-  node = heap[i];
-
-	while ((j > 0) && fshq_lt(heap[0], node, heap[j])) {
-		heap[i] = heap[j];
-		i = j;
-		j = j >> 1;
-	}
-  heap[i] = node;
+    Hit **heap = (Hit **)pq->heap;
+    Hit *node;
+    int i = pq->size;
+    int j = i >> 1;
+    Sorter *sorter = (Sorter *)heap[0];
+    node = heap[i];
+
+    while ((j > 0) && fshq_lt(sorter, node, heap[j])) {
+        heap[i] = heap[j];
+        i = j;
+        j = j >> 1;
+    }
+    heap[i] = node;
 }
 
 void fshq_pq_insert(PriorityQueue *pq, Hit *hit)
 {
-  if (pq->count < pq->size) {
-    Hit *new_hit = ALLOC(Hit);
-    memcpy(new_hit, hit, sizeof(Hit));
-    pq->count++;
-    pq->heap[pq->count] = new_hit;
-    fshq_pq_up(pq);
-  } else if (pq->count > 0 &&
-      fshq_lt((Hit *)pq->heap[0], (Hit *)pq->heap[1], hit)) {
-    memcpy(pq->heap[1], hit, sizeof(Hit));
-    fshq_pq_down(pq);
-  }
+    if (pq->size < pq->capa) {
+        Hit *new_hit = ALLOC(Hit);
+        memcpy(new_hit, hit, sizeof(Hit));
+        pq->size++;
+        if (pq->size >= pq->mem_capa) {
+            pq->mem_capa <<= 1;
+            REALLOC_N(pq->heap, void *, pq->mem_capa);
+        }
+        pq->heap[pq->size] = new_hit;
+        fshq_pq_up(pq);
+    } else if (pq->size > 0
+               && fshq_lt((Sorter *)pq->heap[0], (Hit *)pq->heap[1], hit)) {
+        memcpy(pq->heap[1], hit, sizeof(Hit));
+        fshq_pq_down(pq);
+    }
 }
 
 void fshq_pq_destroy(PriorityQueue *self)
 {
-  sorter_destroy(self->heap[0]);
-  pq_destroy(self);
+    sorter_destroy(self->heap[0]);
+    pq_destroy(self);
+}
+
+PriorityQueue *fshq_pq_new(int size, Sort *sort, IndexReader *ir)
+{
+    PriorityQueue *self = pq_new(size, &fshq_less_than, &free);
+    int i;
+    Sorter *sorter = sorter_new(sort);
+    SortField *sf;
+
+    for (i = 0; i < sort->size; i++) {
+        sf = sort->sort_fields[i];
+        sorter->comparators[i] = sorter_get_comparator(sf, ir);
+    }
+    self->heap[0] = sorter;
+
+    return self;
+}
+
+Hit *fshq_pq_pop_fd(PriorityQueue *pq)
+{
+    if (pq->size <= 0) {
+        return NULL;
+    }
+    else {
+        int j;
+        Sorter *sorter = (Sorter *)pq->heap[0];
+        const int cmp_cnt = sorter->c_cnt;
+        SortField **sort_fields = sorter->sort->sort_fields;
+        Hit *hit = (Hit *)pq->heap[1];   /* save first value */
+        FieldDoc *field_doc;
+        Comparable *comparables;
+        Comparator **comparators = sorter->comparators;
+        pq->heap[1] = pq->heap[pq->size];   /* move last to first */
+        pq->heap[pq->size] = NULL;
+        pq->size--;
+        fshq_pq_down(pq);                   /* adjust heap */
+
+        field_doc = (FieldDoc *)emalloc(sizeof(FieldDoc)
+                                        + sizeof(Comparable)*cmp_cnt);
+        comparables = field_doc->comparables;
+        memcpy(field_doc, hit, sizeof(Hit));
+        field_doc->size = cmp_cnt;
+
+        for (j = 0; j < cmp_cnt; j++) {
+            SortField *sf = sort_fields[j];
+            Comparator *comparator = comparators[j];
+            sf->get_val(comparator->index, hit, &(comparables[j]));
+            comparables[j].type = sf->type;
+            comparables[j].reverse = comparator->reverse;
+        }
+        free(hit);
+        return (Hit *)field_doc;
+    }
 }
 
-PriorityQueue *fshq_pq_create(int size, Sort *sort, IndexReader *ir)
+/***************************************************************************
+ * FieldDoc
+ ***************************************************************************/
+
+void fd_destroy(FieldDoc *fd)
 {
-  PriorityQueue *self = pq_create(size, &fshq_less_than);
-  int i;
-  Sorter *sorter = sorter_create(sort->sf_cnt);
-  SortField *sf;
+    free(fd);
+}
 
-  for (i = 0; i < sort->sf_cnt; i++) {
-    sf = sort->sort_fields[i];
-    sorter->comparators[i] = sorter_get_comparator(sf, ir);
-  }
-  self->heap[0] = sorter;
-  
-  return self;
+/***************************************************************************
+ * FieldDocSortedHitQueue
+ ***************************************************************************/
+
+bool fdshq_lt(FieldDoc *fd1, FieldDoc *fd2)
+{
+    int c = 0, i;
+    Comparable *cmps1 = fd1->comparables;
+    Comparable *cmps2 = fd2->comparables;
+
+    for (i = 0; i < fd1->size && c == 0; i++) {
+        int type = cmps1[i].type;
+        switch (type) {
+            case SORT_TYPE_SCORE:
+                if (cmps1[i].val.f < cmps2[i].val.f) c =  1;
+                if (cmps1[i].val.f > cmps2[i].val.f) c = -1;
+                break;
+            case SORT_TYPE_FLOAT:
+                if (cmps1[i].val.f > cmps2[i].val.f) c =  1;
+                if (cmps1[i].val.f < cmps2[i].val.f) c = -1;
+                break;
+            case SORT_TYPE_DOC:
+                if (fd1->hit.doc > fd2->hit.doc) c =  1;
+                if (fd1->hit.doc < fd2->hit.doc) c = -1;
+                break;
+            case SORT_TYPE_INTEGER:
+                if (cmps1[i].val.i > cmps2[i].val.i) c =  1;
+                if (cmps1[i].val.i < cmps2[i].val.i) c = -1;
+                break;
+            case SORT_TYPE_BYTE:
+                if (cmps1[i].val.i > cmps2[i].val.i) c =  1;
+                if (cmps1[i].val.i < cmps2[i].val.i) c = -1;
+                break;
+            case SORT_TYPE_STRING:
+                do {
+                    char *s1 = cmps1[i].val.s;
+                    char *s2 = cmps2[i].val.s;
+                    if (s1 == NULL) c = s2 ? -1 : 0;
+                    else if (s2 == NULL) c = 1;
+#ifdef POSH_OS_WIN32
+                    else c = strcmp(s1, s2);
+#else
+                    else c = strcoll(s1, s2);
+#endif
+                } while (0);
+                break;
+            default:
+                RAISE(ERROR, "Unknown sort type: %d.", type);
+                break;
+        }
+        if (cmps1[i].reverse) {
+            c = -c;
+        }
+    }
+    if (c == 0) {
+        return fd1->hit.doc > fd2->hit.doc;
+    }
+    else {
+        return c > 0;
+    }
 }
 
 /***************************************************************************
@@ -671,75 +905,78 @@ PriorityQueue *fshq_pq_create(int size, Sort *sort, IndexReader *ir)
  *
  ***************************************************************************/
 
-Sort *sort_create()
+#define SORT_INIT_SIZE 4
+
+Sort *sort_new()
 {
-  Sort *self = ALLOC(Sort);
-  self->sf_cnt = 0;
-  self->sf_capa = ARRAY_INIT_SIZE;
-  self->sort_fields = ALLOC_N(SortField *, ARRAY_INIT_SIZE);
-  self->destroy_all = true;
+    Sort *self = ALLOC(Sort);
+    self->size = 0;
+    self->capa = SORT_INIT_SIZE;
+    self->sort_fields = ALLOC_N(SortField *, SORT_INIT_SIZE);
+    self->destroy_all = true;
+    self->start = 0;
 
-  return self;
+    return self;
 }
 
 void sort_clear(Sort *self)
 {
-  int i;
-  if (self->destroy_all) {
-    for (i = 0; i < self->sf_cnt; i++) {
-      sort_field_destroy(self->sort_fields[i]);
+    int i;
+    if (self->destroy_all) {
+        for (i = 0; i < self->size; i++) {
+            sort_field_destroy(self->sort_fields[i]);
+        }
     }
-  }
-  self->sf_cnt = 0;
+    self->size = 0;
 }
 
 void sort_destroy(void *p)
 {
-  Sort *self = (Sort *)p;
-  sort_clear(self);
-  free(self->sort_fields);
-  free(self);
+    Sort *self = (Sort *)p;
+    sort_clear(self);
+    free(self->sort_fields);
+    free(self);
 }
 
 void sort_add_sort_field(Sort *self, SortField *sf)
 {
-  if (self->sf_cnt == self->sf_capa) {
-    self->sf_capa *= 2;
-    REALLOC_N(self->sort_fields, SortField *, self->sf_capa);
-  }
+    if (self->size == self->capa) {
+        self->capa <<= 1;
+        REALLOC_N(self->sort_fields, SortField *, self->capa);
+    }
 
-  self->sort_fields[self->sf_cnt] = sf;
-  self->sf_cnt++;
+    self->sort_fields[self->size] = sf;
+    self->size++;
 }
 
 char *sort_to_s(Sort *self)
 {
-  int i, len = 20;
-  char *s;
-  char *str;
-  char **sf_strs = ALLOC_N(char *, self->sf_cnt);
-
-  for (i = 0; i < self->sf_cnt; i++) {
-    sf_strs[i] = s = sort_field_to_s(self->sort_fields[i]);
-    len += (int)strlen(s) + 2;
-  }
-
-  str = ALLOC_N(char, len);
-  s = "Sort[";
-  len = (int)strlen(s);
-  memcpy(str, s, len);
-
-  s = str + len;
-  for (i = 0; i < self->sf_cnt; i++) {
-    sprintf(s, "%s, ", sf_strs[i]);
-    s += (int)strlen(s);
-    free(sf_strs[i]);
-  }
-  free(sf_strs);
-
-  if (self->sf_cnt > 0) {
-    s -= 2;
-  }
-  sprintf(s, "]");
-  return str;
+    int i, len = 20;
+    char *s;
+    char *str;
+    char **sf_strs = ALLOC_N(char *, self->size);
+
+    for (i = 0; i < self->size; i++) {
+        sf_strs[i] = s = sort_field_to_s(self->sort_fields[i]);
+        len += (int)strlen(s) + 2;
+    }
+
+    str = ALLOC_N(char, len);
+    s = "Sort[";
+    len = (int)strlen(s);
+    memcpy(str, s, len);
+
+    s = str + len;
+    for (i = 0; i < self->size; i++) {
+        sprintf(s, "%s, ", sf_strs[i]);
+        s += (int)strlen(s);
+        free(sf_strs[i]);
+    }
+    free(sf_strs);
+
+    if (self->size > 0) {
+        s -= 2;
+    }
+    sprintf(s, "]");
+    return str;
 }