ferret 0.9.6 → 0.10.0

data/ext/search.h

@@ -9,9 +9,6 @@ typedef struct Scorer Scorer;
 #include "bitvector.h"
 #include "similarity.h"
 
-#define term_set_create() \
-    hs_create((hash_ft)&term_hash, (eq_ft)&term_eq, (free_ft)&term_destroy)
-
 /***************************************************************************
  *
  * Explanation
@@ -24,15 +21,45 @@ typedef struct Explanation
     float value;
     char *description;
     struct Explanation **details;
-    int dcnt;
-    int dcapa;
 } Explanation;
+ 
+extern Explanation *expl_new(float value, const char *description, ...);
+extern void expl_destroy(Explanation *expl);
+extern Explanation *expl_add_detail(Explanation *expl, Explanation *detail);
+extern char *expl_to_s_depth(Explanation *expl, int depth);
+extern char *expl_to_html(Explanation *expl);
 
-extern Explanation *expl_create(float value, char *description);
-extern void expl_destoy(void *p);
-extern Explanation *expl_add_detail(Explanation *self, Explanation *detail);
-extern char *expl_to_s(Explanation *self, int depth);
-extern char *expl_to_html(Explanation *self);
+#define expl_to_s(expl) expl_to_s_depth(expl, 0)
+
+/***************************************************************************
+ *
+ * Highlighter
+ *
+ ***************************************************************************/
+
+typedef struct MatchRange
+{
+    int start;
+    int end;
+    int start_offset;
+    int end_offset;
+    double score;
+} MatchRange;
+
+#define MATCH_VECTOR_INIT_CAPA 8
+typedef struct MatchVector
+{
+    int size;
+    int capa;
+    MatchRange *matches;
+} MatchVector;
+
+extern MatchVector *matchv_new();
+extern MatchVector *matchv_add(MatchVector *mp, int start, int end);
+extern MatchVector *matchv_sort(MatchVector *self);
+extern void matchv_destroy(MatchVector *self);
+extern MatchVector *matchv_compact(MatchVector *self);
+extern MatchVector *matchv_compact_with_breaks(MatchVector *self);
 
 /***************************************************************************
  *
@@ -46,8 +73,6 @@ typedef struct Hit
     float score;
 } Hit;
 
-extern bool hit_less_than(void *p1, void *p2);
-
 /***************************************************************************
  *
  * TopDocs
@@ -59,9 +84,10 @@ typedef struct TopDocs
     int total_hits;
     int size;
     Hit **hits;
+    float max_score;
 } TopDocs;
 
-extern TopDocs *td_create(int total_hits, int size, Hit **hits);
+extern TopDocs *td_new(int total_hits, int size, Hit **hits, float max_score);
 extern void td_destroy(TopDocs *td);
 extern char *td_to_s(TopDocs *td);
 
@@ -73,22 +99,23 @@ extern char *td_to_s(TopDocs *td);
 
 typedef struct Filter
 {
-    void *data;
-    char *name;
-    HshTable *cache;
-    BitVector *(*get_bv)(struct Filter *self, IndexReader *ir);
-    char      *(*to_s)(struct Filter *self);
-    uint       (*hash)(struct Filter *self);
-    int        (*eq)(struct Filter *self, struct Filter *o);
-    void       (*destroy)(struct Filter *self);
+    char       *name;
+    HashTable  *cache;
+    BitVector  *(*get_bv_i)(struct Filter *self, IndexReader *ir);
+    char       *(*to_s)(struct Filter *self);
+    ulong       (*hash)(struct Filter *self);
+    int         (*eq)(struct Filter *self, struct Filter *o);
+    void        (*destroy_i)(struct Filter *self);
+    int         ref_cnt;
 } Filter;
 
-extern Filter *filt_create(char *name);
-extern char *filt_to_s_i(Filter *self);
-extern BitVector *filt_get_bv(Filter *self, IndexReader *ir);
-extern void filt_destroy(Filter *self);
-extern uint filt_hash(Filter *self);
-extern int filt_eq(Filter *self, Filter *o);
+#define filt_new(type) filt_create(sizeof(type), #type)
+extern Filter *filt_create(size_t size, const char *name);
+extern BitVector *filt_get_bv(Filter *filt, IndexReader *ir);
+extern void filt_destroy_i(Filter *filt);
+extern void filt_deref(Filter *filt);
+extern ulong filt_hash(Filter *filt);
+extern int filt_eq(Filter *filt, Filter *o);
 
 /***************************************************************************
  *
@@ -96,8 +123,9 @@ extern int filt_eq(Filter *self, Filter *o);
  *
  ***************************************************************************/
 
-extern Filter *rfilt_create(const char *field, char *lower_term,
-                            char *upper_term, bool include_lower, bool include_upper);
+extern Filter *rfilt_new(const char *field,
+                         const char *lower_term, const char *upper_term,
+                         bool include_lower, bool include_upper);
 
 /***************************************************************************
  *
@@ -105,12 +133,8 @@ extern Filter *rfilt_create(const char *field, char *lower_term,
  *
  ***************************************************************************/
 
-typedef struct QueryFilter
-{
-    Query *query;
-} QueryFilter;
-
-extern Filter *qfilt_create(Query *query);
+extern Filter *qfilt_new(Query *query);
+extern Filter *qfilt_new_nr(Query *query);
 
 /***************************************************************************
  *
@@ -120,12 +144,11 @@ extern Filter *qfilt_create(Query *query);
 
 struct Weight
 {
-    void *data;
-    float value;
-    float qweight;
-    float qnorm;
-    float idf;
-    Query *query;
+    float        value;
+    float        qweight;
+    float        qnorm;
+    float        idf;
+    Query       *query;
     Similarity  *similarity;
     Query       *(*get_query)(Weight *self);
     float        (*get_value)(Weight *self);
@@ -137,78 +160,26 @@ struct Weight
     void         (*destroy)(Weight *self);
 };
 
-extern Weight *w_create(Query *query);
+#define w_new(type, query) w_create(sizeof(type), query)
+extern Weight *w_create(size_t size, Query *query);
 extern void w_destroy(Weight *self);
-
 extern Query *w_get_query(Weight *self);
 extern float w_get_value(Weight *self);
 extern float w_sum_of_squared_weights(Weight *self);
 extern void w_normalize(Weight *self, float normalization_factor);
 
-/***************************************************************************
- *
- * TermWeight
- *
- ***************************************************************************/
-
-extern Weight *tw_create(Query *query, Searcher *searcher);
-
-/***************************************************************************
- *
- * BooleanWeight
- *
- ***************************************************************************/
-
-typedef struct BooleanWeight {
-    Weight **weights;
-    int w_cnt;
-} BooleanWeight;
-
-extern Weight *bw_create(Query *query, Searcher *searcher);
-
-/***************************************************************************
- *
- * PhraseWeight
- *
- ***************************************************************************/
-
-extern Weight *phw_create(Query *query, Searcher *searcher);
-
-/***************************************************************************
- *
- * ConstantScoreWeight
- *
- ***************************************************************************/
-
-extern Weight *csw_create(Query *query, Searcher *searcher);
-
-/***************************************************************************
- *
- * MatchAllWeight
- *
- ***************************************************************************/
-
-extern Weight *maw_create(Query *query, Searcher *searcher);
-
-/***************************************************************************
- *
- * SpanWeight
- *
- ***************************************************************************/
-
-extern Weight *spanw_create(Query *query, Searcher *searcher);
-
 /***************************************************************************
  *
  * Query
  *
  ***************************************************************************/
 
-enum QUERY_TYPE {
+enum QUERY_TYPE
+{
     TERM_QUERY,
+    MULTI_TERM_QUERY,
     BOOLEAN_QUERY,
     PHRASE_QUERY,
-    MULTI_PHRASE_QUERY,
     CONSTANT_QUERY,
     FILTERED_QUERY,
     MATCH_ALL_QUERY,
@@ -225,57 +196,53 @@ enum QUERY_TYPE {
 
 struct Query
 {
-    uchar type;
-    int ref_cnt;
-    void *data;
-    float boost;
-    Weight *weight;
-    Query      *(*rewrite)(Query *self, IndexReader *ir);
-    void        (*extract_terms)(Query *self, HashSet *terms);
-    Similarity *(*get_similarity)(Query *self, Searcher *searcher);
-    char       *(*to_s)(Query *self, char *field);
-    uint        (*hash)(Query *self);
-    int         (*eq)(Query *self, Query *o);
-    void        (*destroy_i)(Query *self);
-    Weight     *(*create_weight_i)(Query *self, Searcher *searcher);
-    bool destroy_all : 1;
+    int          ref_cnt;
+    float        boost;
+    Weight       *weight;
+    Query       *(*rewrite)(Query *self, IndexReader *ir);
+    void         (*extract_terms)(Query *self, HashSet *terms);
+    Similarity  *(*get_similarity)(Query *self, Searcher *searcher);
+    char        *(*to_s)(Query *self, const char *field);
+    ulong        (*hash)(Query *self);
+    int          (*eq)(Query *self, Query *o);
+    void         (*destroy_i)(Query *self);
+    Weight      *(*create_weight_i)(Query *self, Searcher *searcher);
+    MatchVector *(*get_matchv_i)(Query *self, MatchVector *mv, TermVector *tv);
+    enum QUERY_TYPE type;
 };
 
 /* Internal Query Functions */
-extern Query *q_create();
 extern Similarity *q_get_similarity_i(Query *self, Searcher *searcher);
 extern void q_destroy_i(Query *self);
 extern Weight *q_create_weight_unsup(Query *self, Searcher *searcher);
 
-
 extern void q_deref(Query *self);
+extern const char *q_get_query_name(enum QUERY_TYPE type);
 extern Weight *q_weight(Query *self, Searcher *searcher);
 extern Query *q_combine(Query **queries, int q_cnt);
-extern uint q_hash(Query *self);
+extern ulong q_hash(Query *self);
 extern int q_eq(Query *self, Query *o);
+extern Query *q_create(size_t size);
+#define q_new(type) q_create(sizeof(type))
 
 /***************************************************************************
- *
  * TermQuery
- *
  ***************************************************************************/
 
 typedef struct TermQuery
 {
-    Term *term;
+    Query super;
+    char *field;
+    char *term;
 } TermQuery;
 
-extern Query *tq_create(Term *term);
+Query *tq_new(const char *field, const char *term);
 
 /***************************************************************************
- *
  * BooleanQuery
- *
  ***************************************************************************/
 
-/***************************************************************************
- * BooleanClause
- ***************************************************************************/
+/* *** BooleanClause *** */
 
 enum BC_TYPE
 {
@@ -284,22 +251,20 @@ enum BC_TYPE
     BC_MUST_NOT
 };
 
-typedef struct BooleanClause {
+typedef struct BooleanClause
+{
     int ref_cnt;
     Query *query;
-    Query *rewritten;
     unsigned int occur : 4;
     bool is_prohibited : 1;
     bool is_required : 1;
 } BooleanClause;
 
-extern BooleanClause *bc_create(Query *query, unsigned int occur);
+extern BooleanClause *bc_new(Query *query, enum BC_TYPE occur);
 extern void bc_deref(BooleanClause *self);
-extern void bc_set_occur(BooleanClause *self, unsigned int occur);
+extern void bc_set_occur(BooleanClause *self, enum BC_TYPE occur);
 
-/***************************************************************************
- * BooleanQuery
- ***************************************************************************/
+/* *** BooleanQuery *** */
 
 #define DEFAULT_MAX_CLAUSE_COUNT 1024
 #define BOOLEAN_CLAUSES_START_CAPA 4
@@ -307,172 +272,184 @@ extern void bc_set_occur(BooleanClause *self, unsigned int occur);
 
 typedef struct BooleanQuery
 {
-    bool coord_disabled;
-    int max_clause_cnt;
-    int clause_cnt;
-    int clause_capa;
-    float original_boost;
+    Query           super;
+    bool            coord_disabled;
+    int             max_clause_cnt;
+    int             clause_cnt;
+    int             clause_capa;
+    float           original_boost;
     BooleanClause **clauses;
-    Similarity *similarity;
+    Similarity     *similarity;
 } BooleanQuery;
 
-extern Query *bq_create(bool coord_disabled);
+extern Query *bq_new(bool coord_disabled);
 extern BooleanClause *bq_add_query(Query *self, Query *sub_query,
-                                   unsigned int occur);
+                                   enum BC_TYPE occur);
+extern BooleanClause *bq_add_query_nr(Query *self, Query *sub_query,
+                                      enum BC_TYPE occur);
 extern BooleanClause *bq_add_clause(Query *self, BooleanClause *bc);
+extern BooleanClause *bq_add_clause_nr(Query *self, BooleanClause *bc);
 
 /***************************************************************************
- *
  * PhraseQuery
- *
  ***************************************************************************/
 
 #define PHQ_INIT_CAPA 4
 typedef struct PhraseQuery
 {
-    int slop;
-    Term **terms;
-    int *positions;
-    int t_cnt;
-    int t_capa;
-    char *field;
+    Query           super;
+    int             slop;
+    char           *field;
+    PhrasePosition *positions;
+    int             pos_cnt;
+    int             pos_capa;
 } PhraseQuery;
 
-extern Query *phq_create();
-extern void phq_add_term(Query *self, Term *term, int pos_inc);
+extern Query *phq_new(const char *field);
+extern void phq_add_term(Query *self, const char *term, int pos_inc);
+extern void phq_add_term_abs(Query *self, const char *term, int position);
+extern void phq_append_multi_term(Query *self, const char *term);
 
 /***************************************************************************
- *
- * MultiPhraseQuery
- *
+ * MultiTermQuery
  ***************************************************************************/
 
-typedef struct MultiPhraseQuery
+#define MULTI_TERM_QUERY_MAX_TERMS 256
+typedef struct MultiTermQuery
 {
-    int slop;
-    Term ***terms;
-    int *positions;
-    int *pt_cnt;
-    int t_cnt;
-    int t_capa;
-    char *field;
-} MultiPhraseQuery;
+    Query           super;
+    char           *field;
+    PriorityQueue  *boosted_terms;
+    float           min_boost;
+} MultiTermQuery;
 
-extern Query *mphq_create();
-extern void mphq_add_terms(Query *self, Term **ts, int t_cnt, int pos_inc);
+extern void multi_tq_add_term(Query *self, const char *term);
+extern void multi_tq_add_term_boost(Query *self, const char *term, float boost);
+extern Query *multi_tq_new(const char *field);
+extern Query *multi_tq_new_conf(const char *field, int max_terms,
+                                          float min_boost);
+
+#define MTQMaxTerms(query) (((MTQSubQuery *)(query))->max_terms)
+typedef struct MTQSubQuery
+{
+    Query super;
+    int   max_terms;
+} MTQSubQuery;
 
 /***************************************************************************
- *
  * PrefixQuery
- *
  ***************************************************************************/
 
-extern Query *prefixq_create(Term *prefix);
+#define PREFIX_QUERY_MAX_TERMS 256
+
+
+typedef struct PrefixQuery
+{
+    MTQSubQuery super;
+    char *field;
+    char *prefix;
+} PrefixQuery;
+
+extern Query *prefixq_new(const char *field, const char *prefix);
 
 /***************************************************************************
- *
  * WildCardQuery
- *
  ***************************************************************************/
 
 #define WILD_CHAR '?'
 #define WILD_STRING '*'
+#define WILD_CARD_QUERY_MAX_TERMS 256
+
+typedef struct WildCardQuery
+{
+    MTQSubQuery super;
+    char *field;
+    char *pattern;
+} WildCardQuery;
+
 
-extern Query *wcq_create(Term *term);
-extern bool wc_match(char *pattern, char *text);
+extern Query *wcq_new(const char *field, const char *pattern);
+extern bool wc_match(const char *pattern, const char *text);
 
 /***************************************************************************
- *
  * FuzzyQuery
- *
  ***************************************************************************/
 
-#define DEF_MIN_SIM 0.5
+#define DEF_MIN_SIM 0.5f
 #define DEF_PRE_LEN 0
+#define DEF_MAX_TERMS 256
 #define TYPICAL_LONGEST_WORD 20
 
 typedef struct FuzzyQuery
 {
-    Term *term;
-    char *text; /* term text after prefix */
-    int text_len;
-    int pre_len;
-    float min_sim;
-    float scale_factor;
-    int max_distances[TYPICAL_LONGEST_WORD];
-    int *da;
-    int da_capa;
+    MTQSubQuery super;
+    char       *field;
+    char       *term;
+    const char *text; /* term text after prefix */
+    int         text_len;
+    int         pre_len;
+    float       min_sim;
+    float       scale_factor;
+    int         max_distances[TYPICAL_LONGEST_WORD];
+    int        *da;
 } FuzzyQuery;
 
-extern Query *fuzq_create(Term *term);
-extern Query *fuzq_create_mp(Term *term, float min_sim, int pre_len);
+extern Query *fuzq_new(const char *term, const char *field);
+extern Query *fuzq_new_conf(const char *field, const char *term,
+                            float min_sim, int pre_len, int max_terms);
 
 /***************************************************************************
- *
  * ConstantScoreQuery
- *
  ***************************************************************************/
 
-extern Query *csq_create(Filter *filter);
+typedef struct ConstantScoreQuery
+{
+    Query   super;
+    Filter *filter;
+} ConstantScoreQuery;
+
+extern Query *csq_new(Filter *filter);
+extern Query *csq_new_nr(Filter *filter);
 
 /***************************************************************************
- *
- * FilteredQueryQuery
- *
+ * FilteredQuery
  ***************************************************************************/
 
 typedef struct FilteredQuery
 {
-    Query *query;
+    Query   super;
+    Query  *query;
     Filter *filter;
 } FilteredQuery;
 
-extern Query *fq_create(Query *query, Filter *filter);
+extern Query *fq_new(Query *query, Filter *filter);
 
 /***************************************************************************
- *
  * MatchAllQuery
- *
  ***************************************************************************/
 
-extern Query *maq_create();
+extern Query *maq_new();
 
 /***************************************************************************
- *
  * RangeQuery
- *
  ***************************************************************************/
 
-typedef struct Range
-{
-    char *field;
-    char *lower_term;
-    char *upper_term;
-    bool include_lower : 1;
-    bool include_upper : 1;
-} Range;
-
-extern Query *rq_create(const char *field, char *lower_term,
-                        char *upper_term, bool include_lower, bool include_upper);
-extern Query *rq_create_less(const char *field, char *upper_term,
-                             bool include_upper);
-extern Query *rq_create_more(const char *field, char *lower_term,
-                             bool include_lower);
+extern Query *rq_new(const char *field, const char *lower_term,
+                     const char *upper_term, bool include_lower,
+                     bool include_upper);
+extern Query *rq_new_less(const char *field, const char *upper_term,
+                          bool include_upper);
+extern Query *rq_new_more(const char *field, const char *lower_term,
+                          bool include_lower);
 
 /***************************************************************************
- *
  * SpanQuery
- *
- ***************************************************************************/
-
-/***************************************************************************
- * SpanEnum
  ***************************************************************************/
 
+/* ** SpanEnum ** */
 typedef struct SpanEnum SpanEnum;
 struct SpanEnum
 {
-    void *data;
     Query *query;
     bool (*next)(SpanEnum *self);
     bool (*skip_to)(SpanEnum *self, int target_doc);
@@ -483,107 +460,26 @@ struct SpanEnum
     void (*destroy)(SpanEnum *self);
 };
 
-/***************************************************************************
- * SpanTermEnum
- ***************************************************************************/
-
-typedef struct SpanTermEnum SpanTermEnum;
-struct SpanTermEnum
-{
-    TermDocEnum *positions;
-    int position;
-    int doc;
-    int count;
-    int freq;
-};
-
-extern SpanEnum *spante_create(Query *query, IndexReader *ir);
-
-/***************************************************************************
- * SpanFirstEnum
- ***************************************************************************/
-
-extern SpanEnum *spanfe_create(Query *query, IndexReader *ir);
-
-/***************************************************************************
- * SpanOrEnum
- ***************************************************************************/
-
-typedef struct SpanOrEnum
-{
-    PriorityQueue *queue;
-    SpanEnum **span_enums;
-    int s_cnt;
-    bool first_time;
-} SpanOrEnum;
-
-extern SpanEnum *spanoe_create(Query *query, IndexReader *ir);
-
-/***************************************************************************
- * SpanEnumCell
- ***************************************************************************/
-
-typedef struct SpanEnumCell
-{
-    SpanEnum *parent;
-    SpanEnum *se;
-    int index;
-    int length;
-} SpanEnumCell;
-
-extern SpanEnum *spanec_create(Query *parent, Query *child, int index);
-
-/***************************************************************************
- * SpanNearEnum
- ***************************************************************************/
-
-typedef struct SpanNearEnum
-{
-    SpanEnum **span_enums;
-    int s_cnt;
-    int slop;
-    int current;
-    bool first_time : 1;
-    bool in_order : 1;
-    int doc;
-    int start;
-    int end;
-} SpanNearEnum;
-
-extern SpanEnum *spanne_create(Query *query, IndexReader *ir);
-
-/***************************************************************************
- * SpanNotEnum
- ***************************************************************************/
-
-typedef struct SpanNotEnum
+/* ** SpanQuery ** */
+typedef struct SpanQuery
 {
-    SpanEnum *inc;
-    SpanEnum *exc;
-    bool more_inc : 1;
-    bool more_exc : 1;
-} SpanNotEnum;
-
-extern SpanEnum *spanxe_create(Query *query, IndexReader *ir);
+    Query        super;
+    char        *field;
+    SpanEnum    *(*get_spans)(Query *self, IndexReader *ir);
+    HashSet     *(*get_terms)(Query *self);
+} SpanQuery;
 
 /***************************************************************************
- * SpanQuery
+ * SpanTermQuery
  ***************************************************************************/
 
-typedef struct SpanQuery SpanQuery;
-struct SpanQuery
+typedef struct SpanTermQuery
 {
-    void *data;
-    char *field;
-    SpanEnum *(*get_spans)(Query *self, IndexReader *ir);
-    HashSet  *(*get_terms)(Query *self);
-};
+    SpanQuery super;
+    char     *term;
+} SpanTermQuery;
+extern Query *spantq_new(const char *field, const char *term);
 
-/***************************************************************************
- * SpanTermQuery
- ***************************************************************************/
-
-extern Query *spantq_create(Term *term);
 
 /***************************************************************************
  * SpanFirstQuery
@@ -591,11 +487,13 @@ extern Query *spantq_create(Term *term);
 
 typedef struct SpanFirstQuery
 {
-    int end;
-    Query *match;
+    SpanQuery   super;
+    int         end;
+    Query      *match;
 } SpanFirstQuery;
 
-extern Query *spanfq_create(Query *match, int end);
+extern Query *spanfq_new(Query *match, int end);
+extern Query *spanfq_new_nr(Query *match, int end);
 
 /***************************************************************************
  * SpanOrQuery
@@ -603,11 +501,15 @@ extern Query *spanfq_create(Query *match, int end);
 
 typedef struct SpanOrQuery
 {
-    Query **clauses;
-    int c_cnt;
+    SpanQuery   super;
+    Query     **clauses;
+    int         c_cnt;
+    int         c_capa;
 } SpanOrQuery;
 
-extern Query *spanoq_create(Query **clauses, int c_cnt);
+extern Query *spanoq_new();
+extern Query *spanoq_add_clause(Query *self, Query *clause);
+extern Query *spanoq_add_clause_nr(Query *self, Query *clause);
 
 /***************************************************************************
  * SpanNearQuery
@@ -615,15 +517,17 @@ extern Query *spanoq_create(Query **clauses, int c_cnt);
 
 typedef struct SpanNearQuery
 {
-    Query **clauses;
-    int c_cnt;
-    int slop;
-    bool in_order;
+    SpanQuery   super;
+    Query     **clauses;
+    int         c_cnt;
+    int         c_capa;
+    int         slop;
+    bool        in_order : 1;
 } SpanNearQuery;
 
-extern Query *spannq_create(Query **clauses, int c_cnt, int slop,
-                            bool in_order);
-
+extern Query *spannq_new(int slop, bool in_order);
+extern Query *spannq_add_clause(Query *self, Query *clause);
+extern Query *spannq_add_clause_nr(Query *self, Query *clause);
 
 /***************************************************************************
  * SpanNotQuery
@@ -631,11 +535,15 @@ extern Query *spannq_create(Query **clauses, int c_cnt, int slop,
 
 typedef struct SpanNotQuery
 {
-    Query *inc;
-    Query *exc;
+    SpanQuery   super;
+    Query      *inc;
+    Query      *exc;
 } SpanNotQuery;
 
-extern Query *spanxq_create(Query *inc, Query *exc);
+extern Query *spanxq_new(Query *inc, Query *exc);
+extern Query *spanxq_new_nr(Query *inc, Query *exc);
+
+
 
 /***************************************************************************
  *
@@ -643,13 +551,15 @@ extern Query *spanxq_create(Query *inc, Query *exc);
  *
  ***************************************************************************/
 
-#define SCORER_NULLIFY(mscorer) mscorer->destroy(mscorer); mscorer = NULL
+#define SCORER_NULLIFY(mscorer) do {\
+    (mscorer)->destroy(mscorer);\
+    (mscorer) = NULL;\
+} while (0)
 
 struct Scorer
 {
-    void *data;
     Similarity  *similarity;
-    int         doc;
+    int          doc;
     float        (*score)(Scorer *self);
     bool         (*next)(Scorer *self);
     bool         (*skip_to)(Scorer *self, int doc_num);
@@ -657,300 +567,80 @@ struct Scorer
     void         (*destroy)(Scorer *self);
 };
 
+#define scorer_new(type, similarity) scorer_create(sizeof(type), similarity)
 /* Internal Scorer Function */
 extern void scorer_destroy_i(Scorer *self);
-
-extern Scorer *scorer_create(Similarity *similarity);
+extern Scorer *scorer_create(size_t size, Similarity *similarity);
 extern bool scorer_less_than(void *p1, void *p2);
-extern bool scorer_doc_less_than(void *p1, void *p2);
+extern bool scorer_doc_less_than(const Scorer *s1, const Scorer *s2);
 extern int scorer_doc_cmp(const void *p1, const void *p2);
 
-/***************************************************************************
- *
- * TermScorer
- *
- ***************************************************************************/
-
-#define SCORE_CACHE_SIZE 32
-#define TDE_READ_SIZE 32
-
-typedef struct TermScorer
-{
-    int docs[TDE_READ_SIZE];
-    int freqs[TDE_READ_SIZE];
-    int pointer;
-    int pointer_max;
-    float score_cache[SCORE_CACHE_SIZE];
-    Weight *weight;
-    TermDocEnum *tde;
-    uchar *norms;
-    float weight_value;
-} TermScorer;
-
-extern Scorer *tsc_create(Weight *weight, TermDocEnum *tde, uchar *norms);
-
-/***************************************************************************
- *
- * BooleanScorer
- *
- ***************************************************************************/
-
-/***************************************************************************
- * Coordinator
- ***************************************************************************/
-
-typedef struct Coordinator
-{
-    int max_coord;
-    float *coord_factors;
-    Similarity *similarity;
-    int num_matches;
-} Coordinator;
-
-/***************************************************************************
- * DisjunctionSumScorer
- ***************************************************************************/
-
-typedef struct DisjunctionSumScorer
-{
-    float cum_score;
-    int num_matches;
-    int min_num_matches;
-    Scorer **sub_scorers;
-    int ss_cnt;
-    PriorityQueue *scorer_queue;
-    Coordinator *coordinator;
-} DisjunctionSumScorer;
-
-/***************************************************************************
- * ConjunctionScorer
- ***************************************************************************/
-
-typedef struct ConjunctionScorer
-{
-    bool first_time : 1;
-    bool more : 1;
-    float coord;
-    int ss_cnt;
-    int ss_capa;
-    Scorer **sub_scorers;
-    int first;
-    int last;
-    Coordinator *coordinator;
-    int last_scored_doc;
-} ConjunctionScorer;
-
-/***************************************************************************
- * SingleMatchScorer
- ***************************************************************************/
-
-typedef struct SingleMatchScorer
-{
-    Coordinator *coordinator;
-    Scorer *scorer;
-} SingleMatchScorer;
-
-/***************************************************************************
- * ReqOptSumScorer
- ***************************************************************************/
-
-typedef struct ReqOptSumScorer
-{
-    Scorer *req_scorer;
-    Scorer *opt_scorer;
-    bool first_time_opt;
-} ReqOptSumScorer;
-
-/***************************************************************************
- * ReqExclScorer
- ***************************************************************************/
-
-typedef struct ReqExclScorer
-{
-    Scorer *req_scorer;
-    Scorer *excl_scorer;
-    bool first_time;
-} ReqExclScorer;
-
-/***************************************************************************
- * BooleanScorer
- ***************************************************************************/
-
-typedef struct BooleanScorer
-{
-    Scorer **required_scorers;
-    int rs_cnt;
-    int rs_capa;
-    Scorer **optional_scorers;
-    int os_cnt;
-    int os_capa;
-    Scorer **prohibited_scorers;
-    int ps_cnt;
-    int ps_capa;
-    Scorer *counting_sum_scorer;
-    Coordinator *coordinator;
-} BooleanScorer;
-
-extern Scorer *bsc_create(Similarity *similarity);
-extern void bsc_add_scorer(Scorer *self, Scorer *scorer, unsigned int occur);
-
-/***************************************************************************
- *
- * PhraseScorer
- *
- ***************************************************************************/
-
-/***************************************************************************
- * PhrasePosition
- ***************************************************************************/
-typedef struct PhrasePosition
-{
-    TermDocEnum *tpe;
-    int offset;
-    int count;
-    int doc;
-    int position;
-} PhrasePosition;
-
-extern PhrasePosition *pp_create(TermDocEnum *tpe, int offset);
-
-/***************************************************************************
- * PhraseScorer
- ***************************************************************************/
-
-typedef struct PhraseScorer
-{
-    float freq;
-    uchar *norms;
-    float value;
-    Weight *weight;
-    bool first_time : 1;
-    bool more : 1;
-    int pp_first;
-    int pp_last;
-    int pp_cnt;
-    PhrasePosition **phrase_pos;
-    float (*phrase_freq)(Scorer *self);
-    int slop;
-} PhraseScorer;
-
-extern Scorer *phsc_create(Weight *weight, TermDocEnum **term_pos_enum,
-                           int *positions, int t_cnt, Similarity *similarity, uchar *norms);
-
-/***************************************************************************
- * ExactPhraseScorer
- ***************************************************************************/
-
-extern Scorer *exact_phrase_scorer_create(Weight *weight,
-                                          TermDocEnum **term_pos_enum, int *positions, int t_cnt,
-                                          Similarity *similarity, uchar *norms);
-
-/***************************************************************************
- * SloppyPhraseScorer
- ***************************************************************************/
-
-extern Scorer *sloppy_phrase_scorer_create(Weight *weight,
-                                           TermDocEnum **term_pos_enum, int *positions, int t_cnt,
-                                           Similarity *similarity, int slop, uchar *norms);
-
-/***************************************************************************
- *
- * ConstantScoreScorer
- *
- ***************************************************************************/
-
-typedef struct ConstantScoreScorer
-{
-    BitVector *bv;
-    float score;
-} ConstantScoreScorer;
-
-extern Scorer *cssc_create(Weight *weight, IndexReader *ir);
-
-
-/***************************************************************************
- *
- * MatchAllScorer
- *
- ***************************************************************************/
-
-typedef struct MatchAllScorer
-{
-    IndexReader *ir;
-    int max_doc;
-    float score;
-} MatchAllScorer;
-
-extern Scorer *masc_create(Weight *weight, IndexReader *ir);
-
-
-/***************************************************************************
- *
- * SpanScorer
- *
- ***************************************************************************/
-
-typedef struct SpanScorer
-{
-    bool first_time : 1;
-    bool more : 1;
-    IndexReader *ir;
-    SpanEnum *spans;
-    Similarity *sim;
-    uchar *norms;
-    Weight *weight;
-    float value;
-    float freq;
-} SpanScorer;
-
-extern Scorer *spansc_create(Weight *weight, IndexReader *ir);
-
 /***************************************************************************
  *
  * Sort
  *
  ***************************************************************************/
 
-enum SORT_TYPE {
+enum SORT_TYPE
+{
     SORT_TYPE_SCORE,
     SORT_TYPE_DOC,
+    SORT_TYPE_BYTE,
     SORT_TYPE_INTEGER,
     SORT_TYPE_FLOAT,
     SORT_TYPE_STRING,
     SORT_TYPE_AUTO
 };
 
+/***************************************************************************
+ * Comparable
+ ***************************************************************************/
+
+typedef struct Comparable
+{
+    int type;
+    union {
+        int i;
+        float f;
+        char *s;
+        void *p;
+    } val;
+    bool reverse : 1;
+} Comparable;
+
 /***************************************************************************
  * SortField
  ***************************************************************************/
 
 typedef struct SortField
 {
-    mutex_t mutex;
-    char *field;
-    int   type;
-    bool  reverse : 1;
-    void *index;
-    int   (*compare)(void *index_ptr, Hit *hit1, Hit *hit2);
-    void *(*create_index)(int size);
-    void  (*destroy_index)(void *p);
-    void  (*handle_term)(void *index, TermDocEnum *tde, char *text);
+    mutex_t         mutex;
+    char           *field;
+    enum SORT_TYPE  type;
+    bool            reverse : 1;
+    void           *index;
+    int             (*compare)(void *index_ptr, Hit *hit1, Hit *hit2);
+    void            (*get_val)(void *index_ptr, Hit *hit1, Comparable *comparable);
+    void           *(*create_index)(int size);
+    void            (*destroy_index)(void *p);
+    void            (*handle_term)(void *index, TermDocEnum *tde, char *text);
 } SortField;
 
-extern SortField *sort_field_create(char *field, int type, bool reverse);
-extern SortField *sort_field_score_create(bool reverse);
-extern SortField *sort_field_doc_create(bool reverse);
-extern SortField *sort_field_int_create(char *field, bool reverse);
-extern SortField *sort_field_float_create(char *field, bool reverse);
-extern SortField *sort_field_string_create(char *field, bool reverse);
-extern SortField *sort_field_auto_create(char *field, bool reverse);
+extern SortField *sort_field_new(char *field, enum SORT_TYPE type, bool reverse);
+extern SortField *sort_field_score_new(bool reverse);
+extern SortField *sort_field_doc_new(bool reverse);
+extern SortField *sort_field_int_new(char *field, bool reverse);
+extern SortField *sort_field_byte_new(char *field, bool reverse);
+extern SortField *sort_field_float_new(char *field, bool reverse);
+extern SortField *sort_field_string_new(char *field, bool reverse);
+extern SortField *sort_field_auto_new(char *field, bool reverse);
 extern void sort_field_destroy(void *p);
 extern char *sort_field_to_s(SortField *self);
 
-extern SortField SORT_FIELD_SCORE; 
-extern SortField SORT_FIELD_SCORE_REV; 
-extern SortField SORT_FIELD_DOC; 
-extern SortField SORT_FIELD_DOC_REV; 
+extern const SortField SORT_FIELD_SCORE; 
+extern const SortField SORT_FIELD_SCORE_REV; 
+extern const SortField SORT_FIELD_DOC; 
+extern const SortField SORT_FIELD_DOC_REV; 
 
 /***************************************************************************
  * Sort
@@ -959,12 +649,13 @@ extern SortField SORT_FIELD_DOC_REV;
 typedef struct Sort
 {
     SortField **sort_fields;
-    int sf_cnt;
-    int sf_capa;
+    int size;
+    int capa;
+    int start;
     bool destroy_all : 1;
 } Sort;
 
-extern Sort *sort_create();
+extern Sort *sort_new();
 extern void sort_destroy(void *p);
 extern void sort_add_sort_field(Sort *self, SortField *sf);
 extern void sort_clear(Sort *self);
@@ -978,7 +669,27 @@ extern Hit *fshq_pq_pop(PriorityQueue *pq);
 extern void fshq_pq_down(PriorityQueue *pq);
 extern void fshq_pq_insert(PriorityQueue *pq, Hit *hit);
 extern void fshq_pq_destroy(PriorityQueue *pq);
-extern PriorityQueue *fshq_pq_create(int size, Sort *sort, IndexReader *ir);
+extern PriorityQueue *fshq_pq_new(int size, Sort *sort, IndexReader *ir);
+extern Hit *fshq_pq_pop_fd(PriorityQueue *pq);
+
+/***************************************************************************
+ * FieldDoc
+ ***************************************************************************/
+
+typedef struct FieldDoc
+{
+    Hit hit;
+    int size;
+    Comparable comparables[];
+} FieldDoc;
+
+extern void fd_destroy(FieldDoc *fd);
+
+/***************************************************************************
+ * FieldDocSortedHitQueue
+ ***************************************************************************/
+
+extern bool fdshq_lt(FieldDoc *fd1, FieldDoc *fd2);
 
 /***************************************************************************
  *
@@ -986,47 +697,91 @@ extern PriorityQueue *fshq_pq_create(int size, Sort *sort, IndexReader *ir);
  *
  ***************************************************************************/
 
-struct Searcher {
-    void        *data;
-    IndexReader *ir;
+typedef bool (*filter_ft)(int doc_num, float score, Searcher *self);
+
+struct Searcher
+{
     Similarity  *similarity;
-    bool         close_ir : 1;
-    int          (*doc_freq)(Searcher *self, Term *term);
-    int         *(*doc_freqs)(Searcher *self, Term **terms, int tcnt);
+    int          (*doc_freq)(Searcher *self, const char *field,
+                             const char *term);
     Document    *(*get_doc)(Searcher *self, int doc_num);
+    LazyDoc     *(*get_lazy_doc)(Searcher *self, int doc_num);
     int          (*max_doc)(Searcher *self);
     Weight      *(*create_weight)(Searcher *self, Query *query);
     TopDocs     *(*search)(Searcher *self, Query *query, int first_doc,
-                           int num_docs, Filter *filter, Sort *sort);
+                           int num_docs, Filter *filter, Sort *sort,
+                           filter_ft filter_func,
+                           bool load_fields);
+    TopDocs     *(*search_w)(Searcher *self, Weight *weight, int first_doc,
+                             int num_docs, Filter *filter, Sort *sort,
+                             filter_ft filter_func,
+                             bool load_fields);
     void         (*search_each)(Searcher *self, Query *query, Filter *filter,
-                                void (*fn)(Searcher *, int, float, void *), void *arg);
+                                filter_ft filter_func,
+                                void (*fn)(Searcher *, int, float, void *),
+                                void *arg);
     void         (*search_each_w)(Searcher *self, Weight *weight,
-                                  Filter *filter, void (*fn)(Searcher *, int, float, void *),
+                                  Filter *filter,
+                                  filter_ft filter_func,
+                                  void (*fn)(Searcher *, int, float, void *),
                                   void *arg);
     Query       *(*rewrite)(Searcher *self, Query *original);
     Explanation *(*explain)(Searcher *self, Query *query, int doc_num);
     Explanation *(*explain_w)(Searcher *self, Weight *weight, int doc_num);
+    TermVector  *(*get_term_vector)(Searcher *self, const int doc_num,
+                                    const char *field);
     Similarity  *(*get_similarity)(Searcher *self);
     void         (*close)(Searcher *self);
+    void        *arg; /* used to pass values to Searcher functions */
 };
 
-#define sea_doc_freq(s, t)  s->doc_freq(s, t)
-#define sea_doc_freqs(s, t, c)  s->doc_freqs(s, t, c)
-#define sea_get_doc(s, dn)  s->get_doc(s, dn)
-#define sea_max_doc(s)  s->max_doc(s)
-#define sea_search(s, q, fd, nd, filt, sort)\
-    s->search(s, q, fd, nd, filt, sort)
-#define sea_search_each(s, q, filt, fn, arg)\
-    s->search_each(s, q, filt, fn, arg)
-#define sea_search_each_w(s, q, filt, fn, arg)\
-    s->search_each_w(s, q, filt, fn, arg)
-#define sea_rewrite(s, q)  s->rewrite(s, q)
-#define sea_explain(s, q, dn)  s->explain(s, q, dn)
-#define sea_explain_w(s, q, dn)  s->explain_w(s, q, dn)
-#define sea_get_similarity(s)  s->get_similarity(s)
-#define sea_close(s)  s->close(s)
-
-extern Searcher *sea_create(IndexReader *ir);
+#define searcher_doc_freq(s, t)         s->doc_freq(s, t)
+#define searcher_get_doc(s, dn)         s->get_doc(s, dn)
+#define searcher_get_lazy_doc(s, dn)    s->get_lazy_doc(s, dn)
+#define searcher_max_doc(s)             s->max_doc(s)
+#define searcher_rewrite(s, q)          s->rewrite(s, q)
+#define searcher_explain(s, q, dn)      s->explain(s, q, dn)
+#define searcher_explain_w(s, q, dn)    s->explain_w(s, q, dn)
+#define searcher_get_similarity(s)      s->get_similarity(s)
+#define searcher_close(s)               s->close(s)
+#define searcher_search(s, q, fd, nd, filt, sort, ff)\
+    s->search(s, q, fd, nd, filt, sort, ff, false)
+#define searcher_search_fd(s, q, fd, nd, filt, sort, ff)\
+    s->search(s, q, fd, nd, filt, sort, ff, true)
+#define searcher_search_each(s, q, filt, ff, fn, arg)\
+    s->search_each(s, q, filt, ff, fn, arg)
+#define searcher_search_each_w(s, q, filt, ff, fn, arg)\
+    s->search_each_w(s, q, filt, ff, fn, arg)
+
+
+extern MatchVector *searcher_get_match_vector(Searcher *self,
+                                              Query *query,
+                                              const int doc_num,
+                                              const char *field);
+extern char **searcher_highlight(Searcher *self,
+                                 Query *query,
+                                 const int doc_num,
+                                 const char *field,
+                                 const int excerpt_len,
+                                 const int num_excerpts,
+                                 const char *pre_tag,
+                                 const char *post_tag,
+                                 const char *ellipsis);
+
+/***************************************************************************
+ *
+ * IndexSearcher
+ *
+ ***************************************************************************/
+
+typedef struct IndexSearcher {
+    Searcher        super;
+    IndexReader    *ir;
+    bool            close_ir : 1;
+} IndexSearcher;
+
+extern Searcher *isea_new(IndexReader *ir);
+extern int isea_doc_freq(Searcher *self, const char *field, const char *term);
 
 /***************************************************************************
  *
@@ -1036,15 +791,15 @@ extern Searcher *sea_create(IndexReader *ir);
 
 typedef struct MultiSearcher
 {
-    int s_cnt;
-    Searcher **searchers;
-    int *starts;
-    int max_doc;
-    bool close_subs : 1;
+    Searcher    super;
+    int         s_cnt;
+    Searcher  **searchers;
+    int        *starts;
+    int         max_doc;
+    bool        close_subs : 1;
 } MultiSearcher;
 
-extern Searcher *msea_create(Searcher **searchers, int s_cnt,
-                             bool close_subs);
+extern Searcher *msea_new(Searcher **searchers, int s_cnt, bool close_subs);
 
 /***************************************************************************
  *
@@ -1052,93 +807,39 @@ extern Searcher *msea_create(Searcher **searchers, int s_cnt,
  *
  ***************************************************************************/
 
-#define CONC_WORDS 2
+#define QP_CONC_WORDS 2
+#define QP_MAX_CLAUSES 512
 
 typedef struct QParser
 {
     mutex_t mutex;
-    bool or_default : 1;
-    bool wild_lower : 1;
-    bool clean_str : 1;
-    bool handle_parse_errors : 1;
-    bool allow_any_fields : 1;
-    bool close_def_fields : 1;
     int def_slop;
+    int max_clauses;
+    int phq_pos_inc;
     char *qstr;
     char *qstrp;
-    char buf[CONC_WORDS][MAX_WORD_SIZE];
+    char buf[QP_CONC_WORDS][MAX_WORD_SIZE];
     int  buf_index;
+    HashTable *field_cache;
     HashSet *fields;
     HashSet *fields_buf;
     HashSet *def_fields;
     HashSet *all_fields;
     Analyzer *analyzer;
+    HashTable *ts_cache;
     Query *result;
+    bool or_default : 1;
+    bool wild_lower : 1;
+    bool clean_str : 1;
+    bool handle_parse_errors : 1;
+    bool allow_any_fields : 1;
+    bool close_def_fields : 1;
 } QParser;
 
-extern QParser *qp_create(HashSet *all_fields, HashSet *def_fields,
-                          Analyzer *analyzer);
+extern QParser *qp_new(HashSet *all_fields, HashSet *def_fields,
+                       Analyzer *analyzer);
 extern void qp_destroy(QParser *self);
 extern Query *qp_parse(QParser *self, char *qstr);
 extern char *qp_clean_str(char *str);
 
-/***************************************************************************
- *
- * Index
- *
- ***************************************************************************/
-
-typedef struct Index
-{
-    mutex_t mutex;
-    Store *store;
-    Analyzer *analyzer;
-    IndexReader *ir;
-    IndexWriter *iw;
-    Searcher *sea;
-    QParser *qp;
-    HashSet *key;
-    char *id_field;
-    char *def_field;
-    /* for IndexWriter */
-    bool use_compound_file : 1;
-    bool auto_flush : 1;
-    bool has_writes : 1;
-    bool check_latest : 1;
-} Index;
-
-extern Index *index_create(Store *store, Analyzer *analyzer,
-                           HashSet *def_fields, bool create);
-extern void index_destroy(Index *self);
-extern void index_flush(Index *self);
-extern int index_size(Index *self);
-extern void index_optimize(Index *self);
-extern bool index_has_del(Index *self);
-extern bool index_is_deleted(Index *self, int doc_num);
-extern void index_add_doc(Index *self, Document *doc);
-extern void index_add_doc_a(Index *self, Document *doc, Analyzer *analyzer);
-extern void index_add_string(Index *self, char *str, Analyzer *analyzer);
-extern void index_add_array(Index *self, Array *ary, Analyzer *analyzer);
-extern TopDocs *index_search_str(Index *self, char *query, int first_doc,
-                                 int num_docs, Filter *filter, Sort *sort);
-extern Query *index_get_query(Index *self, char *qstr);
-extern Document *index_get_doc(Index *self, int doc_num);
-extern Document *index_get_doc_ts(Index *self, int doc_num);
-extern Document *index_get_doc_id(Index *self, char *id);
-extern Document *index_get_doc_term(Index *self, Term *term);
-extern void index_delete(Index *self, int doc_num);
-extern void index_delete_term(Index *self, Term *term);
-extern void index_delete_id(Index *self, char *id);
-extern void index_delete_query(Index *self, Query *q, Filter *f);
-extern void index_delete_query_str(Index *self, char *qstr, Filter *f);
-extern int index_term_id(Index *self, Term *term);
-extern Explanation *index_explain(Index *self, Query *q, int doc_num);
-extern void index_auto_flush_ir(Index *self);
-extern void index_auto_flush_iw(Index *self);
-
-extern inline void ensure_searcher_open(Index *self);
-extern inline void ensure_reader_open(Index *self);
-extern inline void ensure_writer_open(Index *self);
-
 #endif
-