ferret 0.3.2 → 0.9.0

data/ext/search.h

@@ -0,0 +1,1065 @@
+#ifndef FRT_SEARCH_H
+#define FRT_SEARCH_H
+
+typedef struct Similarity Similarity;
+typedef struct Query Query;
+typedef struct Weight Weight;
+typedef struct Scorer Scorer;
+typedef struct Searcher Searcher;
+
+#include "index.h"
+#include "bitvector.h"
+
+/***************************************************************************
+ *
+ * Similarity
+ *
+ ***************************************************************************/
+
+struct Similarity {
+  void *data;
+  float norm_table[256];
+  float (*length_norm)(Similarity *self, char *field, int num_terms);
+  float (*query_norm)(Similarity *self, float sum_of_squared_weights);
+  float (*tf)(Similarity *self, float freq);
+  float (*sloppy_freq)(Similarity *self, int distance);
+  float (*idf_term)(Similarity *self, Term *term, Searcher *searcher);
+  float (*idf_phrase)(Similarity *self, Term **terms, int tcnt, Searcher *searcher);
+  float (*idf)(Similarity *self, int doc_freq, int num_docs);
+  float (*coord)(Similarity *self, int overlap, int max_overlap);
+  float (*decode_norm)(Similarity *self, uchar b);
+  float (*encode_norm)(Similarity *self, float f);
+  void  (*destroy)(void *p);
+};
+
+#define sim_length_norm(msim, field, num_terms) msim->length_norm(msim, field, num_terms)
+#define sim_query_norm(msim, sosw) msim->query_norm(msim, sosw)
+#define sim_tf(msim, freq) msim->tf(msim, freq)
+#define sim_sloppy_freq(msim, distance) msim->sloppy_freq(msim, distance)
+#define sim_idf_term(msim, term, searcher) msim->idf_term(msim, term, searcher)
+#define sim_idf_phrase(msim, terms, tcnt, searcher) msim->idf_phrase(msim, terms, tcnt, searcher)
+#define sim_idf(msim, doc_freq, num_docs) msim->idf(msim, doc_freq, num_docs)
+#define sim_coord(msim, overlap, max_overlap) msim->coord(msim, overlap, max_overlap)
+#define sim_decode_norm(msim, b) msim->decode_norm(msim, b)
+#define sim_encode_norm(msim, f) msim->encode_norm(msim, f)
+#define sim_destroy(msim) msim->destroy(msim)
+
+float byte_to_float(uchar b);
+uchar float_to_byte(float f);
+
+Similarity *sim_create_default();
+
+/***************************************************************************
+ *
+ * Explanation
+ *
+ ***************************************************************************/
+
+#define EXPLANATION_DETAILS_START_SIZE 4
+typedef struct Explanation {
+  float value;
+  char *description;
+  struct Explanation **details;
+  int dcnt;
+  int dcapa;
+} Explanation;
+
+Explanation *expl_create(float value, char *description);
+void expl_destoy(void *p);
+Explanation *expl_add_detail(Explanation *self, Explanation *detail);
+char *expl_to_s(Explanation *self, int depth);
+char *expl_to_html(Explanation *self);
+
+/***************************************************************************
+ *
+ * Hit
+ *
+ ***************************************************************************/
+
+typedef struct Hit {
+  int doc;
+  float score;
+} Hit;
+
+bool hit_less_than(void *p1, void *p2);
+ 
+/***************************************************************************
+ *
+ * TopDocs
+ *
+ ***************************************************************************/
+
+typedef struct TopDocs {
+  int total_hits;
+  int size;
+  Hit **hits;
+} TopDocs;
+
+TopDocs *td_create(int total_hits, int size, Hit **hits);
+void td_destroy(void *p);
+char *td_to_s(TopDocs *td);
+
+/***************************************************************************
+ *
+ * Filter
+ *
+ ***************************************************************************/
+
+typedef struct Filter {
+  void *data;
+  char *name;
+  HshTable *cache;
+  BitVector *(*get_bv)(struct Filter *self, IndexReader *ir);
+  char      *(*to_s)(struct Filter *self);
+  void       (*destroy)(void *p);
+} Filter;
+
+Filter *filt_create(char *name);
+char *filt_to_s(Filter *self);
+BitVector *filt_get_bv(Filter *self, IndexReader *ir);
+void filt_destroy(void *p);
+
+/***************************************************************************
+ *
+ * RangeFilter
+ *
+ ***************************************************************************/
+
+Filter *rfilt_create(const char *field, char *lower_term, char *upper_term,
+    bool include_lower, bool include_upper);
+void rfilt_destroy(void *p);
+
+/***************************************************************************
+ *
+ * QueryFilter
+ *
+ ***************************************************************************/
+
+typedef struct QueryFilter {
+  Query *query;
+} QueryFilter;
+
+Filter *qfilt_create(Query *query);
+
+
+/***************************************************************************
+ *
+ * Weight
+ *
+ ***************************************************************************/
+
+struct Weight {
+  void *data;
+  float value;
+  float qweight;
+  float qnorm;
+  float idf;
+  Query *query;
+  Similarity  *similarity;
+  Query       *(*get_query)(Weight *self);
+  float        (*get_value)(Weight *self);
+  void         (*normalize)(Weight *self, float normalization_factor);
+  Scorer      *(*scorer)(Weight *self, IndexReader *ir);
+  Explanation *(*explain)(Weight *self, IndexReader *ir, int doc_num);
+  float        (*sum_of_squared_weights)(Weight *self);
+  char        *(*to_s)(Weight *self);
+  void         (*destroy)(void *p);
+};
+
+Query *w_get_query(Weight *self);
+float w_get_value(Weight *self);
+float w_sum_of_squared_weights(Weight *self);
+void w_normalize(Weight *self, float normalization_factor);
+
+/***************************************************************************
+ *
+ * TermWeight
+ *
+ ***************************************************************************/
+
+Weight *tw_create(Query *query, Searcher *searcher);
+
+/***************************************************************************
+ *
+ * BooleanWeight
+ *
+ ***************************************************************************/
+
+typedef struct BooleanWeight {
+  Weight **weights;
+  int w_cnt;
+} BooleanWeight;
+Weight *bw_create(Query *query, Searcher *searcher);
+
+/***************************************************************************
+ *
+ * PhraseWeight
+ *
+ ***************************************************************************/
+
+Weight *phw_create(Query *query, Searcher *searcher);
+
+/***************************************************************************
+ *
+ * ConstantScoreWeight
+ *
+ ***************************************************************************/
+
+Weight *csw_create(Query *query, Searcher *searcher);
+
+/***************************************************************************
+ *
+ * MatchAllWeight
+ *
+ ***************************************************************************/
+
+Weight *maw_create(Query *query, Searcher *searcher);
+
+/***************************************************************************
+ *
+ * SpanWeight
+ *
+ ***************************************************************************/
+
+Weight *spanw_create(Query *query, Searcher *searcher);
+
+/***************************************************************************
+ *
+ * Query
+ *
+ ***************************************************************************/
+
+enum QUERY_TYPE {
+  TERM_QUERY,
+  BOOLEAN_QUERY,
+  PHRASE_QUERY,
+  MULTI_PHRASE_QUERY,
+  CONSTANT_QUERY,
+  MATCH_ALL_QUERY,
+  RANGE_QUERY,
+  WILD_CARD_QUERY,
+  FUZZY_QUERY,
+  PREFIX_QUERY,
+  SPAN_TERM_QUERY,
+  SPAN_FIRST_QUERY,
+  SPAN_OR_QUERY,
+  SPAN_NOT_QUERY,
+  SPAN_NEAR_QUERY
+};
+
+struct Query {
+  bool destroy_all : 1;
+  uchar type;
+  void *data;
+  float boost;
+  float original_boost;
+  Weight *weight;
+  Query *rewritten;
+  Weight     *(*create_weight)(Query *self, Searcher *searcher);
+  Query      *(*rewrite)(Query *self, IndexReader *ir);
+  void        (*extract_terms)(Query *self, Array *terms);
+  Similarity *(*get_similarity)(Query *self, Searcher *searcher);
+  char       *(*to_s)(Query *self, char *field);
+  void        (*destroy)(void *p);
+};
+
+Weight *q_weight(Query *self, Searcher *searcher);
+void q_destroy(Query *self);
+Similarity *q_get_similarity(Query *self, Searcher *searcher);
+void q_extract_terms(Query *self, Array *terms);
+Query *q_create();
+
+/***************************************************************************
+ *
+ * TermQuery
+ *
+ ***************************************************************************/
+
+typedef struct TermQuery {
+  Term *term;
+} TermQuery;
+
+Query *tq_create(Term *term);
+
+/***************************************************************************
+ *
+ * BooleanQuery
+ *
+ ***************************************************************************/
+
+/***************************************************************************
+ * BooleanClause
+ ***************************************************************************/
+
+enum BC_TYPE {
+  BC_SHOULD,
+  BC_MUST,
+  BC_MUST_NOT
+};
+
+typedef struct BooleanClause {
+  Query *query;
+  Query *rewritten;
+  unsigned int occur : 4;
+  bool is_prohibited : 1;
+  bool is_required : 1;
+} BooleanClause;
+
+BooleanClause *bc_create(Query *query, unsigned int occur);
+void bc_set_occur(BooleanClause *self, unsigned int occur);
+
+/***************************************************************************
+ * BooleanQuery
+ ***************************************************************************/
+
+#define DEFAULT_MAX_CLAUSE_COUNT 1024
+#define BOOLEAN_CLAUSES_START_CAPA 4
+#define QUERY_STRING_START_SIZE 64
+
+typedef struct BooleanQuery {
+  bool coord_disabled;
+  int max_clause_cnt;
+  int clause_cnt;
+  int clause_capa;
+  BooleanClause **clauses;
+  Similarity *similarity;
+} BooleanQuery;
+
+Query *bq_create(bool coord_disabled);
+void bq_add_query(Query *self, Query *sub_query, unsigned int occur);
+
+/***************************************************************************
+ *
+ * PhraseQuery
+ *
+ ***************************************************************************/
+
+#define PHQ_INIT_CAPA 4
+typedef struct PhraseQuery {
+  int slop;
+  Term **terms;
+  int *positions;
+  int t_cnt;
+  int t_capa;
+  char *field;
+} PhraseQuery;
+
+Query *phq_create();
+void phq_add_term(Query *self, Term *term, int pos_inc);
+
+/***************************************************************************
+ *
+ * MultiPhraseQuery
+ *
+ ***************************************************************************/
+
+typedef struct MultiPhraseQuery {
+  int slop;
+  Term ***terms;
+  int *positions;
+  int *pt_cnt;
+  int t_cnt;
+  int t_capa;
+  char *field;
+} MultiPhraseQuery;
+
+Query *mphq_create();
+void mphq_add_terms(Query *self, Term **terms, int t_cnt, int pos_inc);
+
+/***************************************************************************
+ *
+ * PrefixQuery
+ *
+ ***************************************************************************/
+
+Query *prefixq_create(Term *prefix);
+
+/***************************************************************************
+ *
+ * WildCardQuery
+ *
+ ***************************************************************************/
+
+#define WILD_CHAR '?'
+#define WILD_STRING '*'
+Query *wcq_create(Term *term);
+bool wc_match(char *pattern, char *text);
+
+/***************************************************************************
+ *
+ * FuzzyQuery
+ *
+ ***************************************************************************/
+
+#define DEF_MIN_SIM 0.5
+#define DEF_PRE_LEN 0
+#define TYPICAL_LONGEST_WORD 20
+
+typedef struct FuzzyQuery {
+  Term *term;
+  char *text; /* term text after prefix */
+  int text_len;
+  int pre_len;
+  float min_sim;
+  float scale_factor;
+  int max_distances[TYPICAL_LONGEST_WORD];
+  int *da;
+  int da_capa;
+} FuzzyQuery;
+
+Query *fuzq_create(Term *term);
+Query *fuzq_create_mp(Term *term, float min_sim, int pre_len);
+
+/***************************************************************************
+ *
+ * ConstantScoreQuery
+ *
+ ***************************************************************************/
+
+Query *csq_create(Filter *filter);
+
+/***************************************************************************
+ *
+ * MatchAllQuery
+ *
+ ***************************************************************************/
+
+Query *maq_create();
+
+/***************************************************************************
+ *
+ * ConstantScoreQuery
+ *
+ ***************************************************************************/
+
+Query *maq_create();
+
+/***************************************************************************
+ *
+ * RangeQuery
+ *
+ ***************************************************************************/
+
+typedef struct Range {
+  char *field;
+  char *lower_term;
+  char *upper_term;
+  bool include_lower;
+  bool include_upper;
+} Range;
+
+Query *rq_create(const char *field, char *lower_term, char *upper_term,
+    bool include_lower, bool include_upper);
+Query *rq_create_less(const char *field, char *upper_term, bool include_upper);
+Query *rq_create_more(const char *field, char *lower_term, bool include_lower);
+
+/***************************************************************************
+ *
+ * SpanQuery
+ *
+ ***************************************************************************/
+
+/***************************************************************************
+ * SpanEnum
+ ***************************************************************************/
+
+typedef struct SpanEnum SpanEnum;
+struct SpanEnum {
+  void *data;
+  Query *query;
+  bool (*next)(SpanEnum *self);
+  bool (*skip_to)(SpanEnum *self, int target_doc);
+  int  (*doc)(SpanEnum *self);
+  int  (*start)(SpanEnum *self);
+  int  (*end)(SpanEnum *self);
+  char *(*to_s)(SpanEnum *self);
+  void (*destroy)(void *p);
+};
+
+/***************************************************************************
+ * SpanTermEnum
+ ***************************************************************************/
+
+typedef struct SpanTermEnum SpanTermEnum;
+struct SpanTermEnum {
+  TermDocEnum *positions;
+  int position;
+  int doc;
+  int count;
+  int freq;
+};
+
+SpanEnum *spante_create(Query *query, IndexReader *ir);
+
+/***************************************************************************
+ * SpanFirstEnum
+ ***************************************************************************/
+
+SpanEnum *spanfe_create(Query *query, IndexReader *ir);
+
+/***************************************************************************
+ * SpanOrEnum
+ ***************************************************************************/
+
+typedef struct SpanOrEnum {
+  PriorityQueue *queue;
+  SpanEnum **span_enums;
+  int s_cnt;
+  bool first_time;
+} SpanOrEnum;
+SpanEnum *spanoe_create(Query *query, IndexReader *ir);
+
+/***************************************************************************
+ * SpanEnumCell
+ ***************************************************************************/
+
+typedef struct SpanEnumCell {
+  SpanEnum *parent;
+  SpanEnum *se;
+  int index;
+  int length;
+} SpanEnumCell;
+SpanEnum *spanec_create(Query *parent, Query *child, int index);
+
+/***************************************************************************
+ * SpanNearEnum
+ ***************************************************************************/
+
+typedef struct SpanNearEnum {
+  SpanEnum **span_enums;
+  int s_cnt;
+  int slop;
+  int current;
+  bool first_time : 1;
+  bool in_order : 1;
+  int doc;
+  int start;
+  int end;
+} SpanNearEnum;
+
+SpanEnum *spanne_create(Query *query, IndexReader *ir);
+
+/***************************************************************************
+ * SpanNotEnum
+ ***************************************************************************/
+
+typedef struct SpanNotEnum {
+  SpanEnum *inc;
+  SpanEnum *exc;
+  bool more_inc : 1;
+  bool more_exc : 1;
+} SpanNotEnum;
+
+SpanEnum *spanxe_create(Query *query, IndexReader *ir);
+
+/***************************************************************************
+ * SpanQuery
+ ***************************************************************************/
+
+typedef struct SpanQuery SpanQuery;
+struct SpanQuery {
+  void *data;
+  char *field;
+  SpanEnum *(*get_spans)(Query *self, IndexReader *ir);
+  Array    *(*get_terms)(Query *self);
+};
+
+/***************************************************************************
+ * SpanTermQuery
+ ***************************************************************************/
+
+Query *spantq_create(Term *term);
+
+/***************************************************************************
+ * SpanFirstQuery
+ ***************************************************************************/
+
+typedef struct SpanFirstQuery {
+  int end;
+  Query *match;
+} SpanFirstQuery;
+
+Query *spanfq_create(Query *match, int end);
+
+/***************************************************************************
+ * SpanOrQuery
+ ***************************************************************************/
+
+typedef struct SpanOrQuery {
+  Query **clauses;
+  int c_cnt;
+} SpanOrQuery;
+
+Query *spanoq_create(Query **clauses, int c_cnt);
+
+/***************************************************************************
+ * SpanNearQuery
+ ***************************************************************************/
+
+typedef struct SpanNearQuery {
+  Query **clauses;
+  int c_cnt;
+  int slop;
+  bool in_order;
+} SpanNearQuery;
+
+Query *spannq_create(Query **clauses, int c_cnt, int slop, bool in_order);
+
+
+/***************************************************************************
+ * SpanNotQuery
+ ***************************************************************************/
+
+typedef struct SpanNotQuery {
+  Query *inc;
+  Query *exc;
+} SpanNotQuery;
+
+Query *spanxq_create(Query *inc, Query *exc);
+
+/***************************************************************************
+ *
+ * Scorer
+ *
+ ***************************************************************************/
+
+#define SCORER_NULLIFY(mscorer) mscorer->destroy(mscorer); mscorer = NULL
+
+struct Scorer {
+  void *data;
+  Similarity  *similarity;
+  int         doc;
+  float        (*score)(Scorer *self);
+  bool         (*next)(Scorer *self);
+  bool         (*skip_to)(Scorer *self, int doc_num);
+  Explanation *(*explain)(Scorer *self, int doc_num);
+  void         (*destroy)(void *p);
+};
+
+void scorer_destroy(void *p);
+Scorer *scorer_create(Similarity *similarity);
+bool scorer_less_than(void *p1, void *p2);
+bool scorer_doc_less_than(void *p1, void *p2);
+int scorer_doc_cmp(const void *p1, const void *p2);
+
+/***************************************************************************
+ *
+ * TermScorer
+ *
+ ***************************************************************************/
+
+#define SCORE_CACHE_SIZE 32
+#define TDE_READ_SIZE 32
+
+typedef struct TermScorer {
+  int docs[TDE_READ_SIZE];
+  int freqs[TDE_READ_SIZE];
+  int pointer;
+  int pointer_max;
+  float score_cache[SCORE_CACHE_SIZE];
+  Weight *weight;
+  TermDocEnum *tde;
+  uchar *norms;
+  float weight_value;
+} TermScorer;
+
+Scorer *tsc_create(Weight *weight, TermDocEnum *tde, uchar *norms);
+
+/***************************************************************************
+ *
+ * BooleanScorer
+ *
+ ***************************************************************************/
+
+/***************************************************************************
+ * Coordinator
+ ***************************************************************************/
+
+typedef struct Coordinator {
+  int max_coord;
+  float *coord_factors;
+  Similarity *similarity;
+  int num_matches;
+} Coordinator;
+
+Coordinator *coo_create(Similarity *similarity);
+Coordinator *coo_init(Coordinator *self);
+
+/***************************************************************************
+ * DisjunctionSumScorer
+ ***************************************************************************/
+
+typedef struct DisjunctionSumScorer{
+  float cum_score;
+  int num_matches;
+  int min_num_matches;
+  Scorer **sub_scorers;
+  int ss_cnt;
+  PriorityQueue *scorer_queue;
+  Coordinator *coordinator;
+} DisjunctionSumScorer;
+
+/***************************************************************************
+ * ConjunctionScorer
+ ***************************************************************************/
+
+typedef struct ConjunctionScorer{
+  bool first_time : 1;
+  bool more : 1;
+  int coord;
+  int ss_cnt;
+  int ss_capa;
+  Scorer **sub_scorers;
+  int first;
+  int last;
+  Coordinator *coordinator;
+  int last_scored_doc;
+} ConjunctionScorer;
+
+/***************************************************************************
+ * SingleMatchScorer
+ ***************************************************************************/
+
+typedef struct SingleMatchScorer {
+  Coordinator *coordinator;
+  Scorer *scorer;
+} SingleMatchScorer;
+
+/***************************************************************************
+ * ReqOptSumScorer
+ ***************************************************************************/
+
+typedef struct ReqOptSumScorer {
+  Scorer *req_scorer;
+  Scorer *opt_scorer;
+  bool first_time_opt;
+} ReqOptSumScorer;
+
+/***************************************************************************
+ * ReqExclScorer
+ ***************************************************************************/
+
+typedef struct ReqExclScorer {
+  Scorer *req_scorer;
+  Scorer *excl_scorer;
+  bool first_time;
+} ReqExclScorer;
+
+/***************************************************************************
+ * BooleanScorer
+ ***************************************************************************/
+
+typedef struct BooleanScorer {
+  Scorer **required_scorers;
+  int rs_cnt;
+  int rs_capa;
+  Scorer **optional_scorers;
+  int os_cnt;
+  int os_capa;
+  Scorer **prohibited_scorers;
+  int ps_cnt;
+  int ps_capa;
+  Scorer *counting_sum_scorer;
+  Coordinator *coordinator;
+} BooleanScorer;
+
+Scorer *bsc_create(Similarity *similarity);
+void bsc_add_scorer(Scorer *self, Scorer *scorer, unsigned int occur);
+
+/***************************************************************************
+ *
+ * PhraseScorer
+ *
+ ***************************************************************************/
+
+/***************************************************************************
+ * PhrasePosition
+ ***************************************************************************/
+typedef struct PhrasePosition {
+  TermDocEnum *tpe;
+  int offset;
+  int count;
+  int doc;
+  int position;
+} PhrasePosition;
+
+PhrasePosition *pp_create(TermDocEnum *tpe, int offset);
+/***************************************************************************
+ * PhraseScorer
+ ***************************************************************************/
+
+typedef struct PhraseScorer {
+  float freq;
+  uchar *norms;
+  float value;
+  Weight *weight;
+  bool first_time : 1;
+  bool more : 1;
+  int pp_first;
+  int pp_last;
+  int pp_cnt;
+  PhrasePosition **phrase_pos;
+  float (*phrase_freq)(Scorer *self);
+  int slop;
+} PhraseScorer;
+
+Scorer *phsc_create(Weight *weight, TermDocEnum **term_pos_enum,
+    int *positions, int t_cnt, Similarity *similarity, uchar *norms);
+
+/***************************************************************************
+ * ExactPhraseScorer
+ ***************************************************************************/
+
+Scorer *exact_phrase_scorer_create(Weight *weight, TermDocEnum **term_pos_enum,
+    int *positions, int t_cnt, Similarity *similarity, uchar *norms);
+
+/***************************************************************************
+ * SloppyPhraseScorer
+ ***************************************************************************/
+
+Scorer *sloppy_phrase_scorer_create(Weight *weight, TermDocEnum **term_pos_enum,
+    int *positions, int t_cnt, Similarity *similarity, int slop, uchar *norms);
+
+/***************************************************************************
+ *
+ * ConstantScoreScorer
+ *
+ ***************************************************************************/
+
+typedef struct ConstantScoreScorer {
+  BitVector *bv;
+  float score;
+} ConstantScoreScorer;
+
+Scorer *cssc_create(Weight *weight, IndexReader *ir);
+
+
+/***************************************************************************
+ *
+ * MatchAllScorer
+ *
+ ***************************************************************************/
+
+typedef struct MatchAllScorer {
+  IndexReader *ir;
+  int max_doc;
+  float score;
+} MatchAllScorer;
+
+Scorer *masc_create(Weight *weight, IndexReader *ir);
+
+
+/***************************************************************************
+ *
+ * SpanScorer
+ *
+ ***************************************************************************/
+
+typedef struct SpanScorer {
+  bool first_time : 1;
+  bool more : 1;
+  IndexReader *ir;
+  SpanEnum *spans;
+  Similarity *sim;
+  uchar *norms;
+  Weight *weight;
+  float value;
+  float freq;
+} SpanScorer;
+
+Scorer *spansc_create(Weight *weight, IndexReader *ir);
+
+/***************************************************************************
+ *
+ * Sort
+ *
+ ***************************************************************************/
+
+enum SORT_TYPE {
+  SORT_TYPE_SCORE,
+  SORT_TYPE_DOC,
+  SORT_TYPE_INTEGER,
+  SORT_TYPE_FLOAT,
+  SORT_TYPE_STRING,
+  SORT_TYPE_AUTO
+};
+
+/***************************************************************************
+ * SortField
+ ***************************************************************************/
+
+typedef struct SortField {
+  mutex_t mutex;
+  char *field;
+  int   type;
+  bool  reverse : 1;
+  void *index;
+  int   (*compare)(void *index_ptr, Hit *hit1, Hit *hit2);
+  void *(*create_index)(int size);
+  void  (*destroy_index)(void *p);
+  void  (*handle_term)(void *index, TermDocEnum *tde, char *text);
+} SortField;
+
+SortField *sort_field_create(char *field, int type, bool reverse);
+SortField *sort_field_score_create(bool reverse);
+SortField *sort_field_doc_create(bool reverse);
+SortField *sort_field_int_create(char *field, bool reverse);
+SortField *sort_field_float_create(char *field, bool reverse);
+SortField *sort_field_string_create(char *field, bool reverse);
+SortField *sort_field_auto_create(char *field, bool reverse);
+void sort_field_destroy(void *p);
+
+extern SortField SORT_FIELD_SCORE; 
+extern SortField SORT_FIELD_SCORE_REV; 
+extern SortField SORT_FIELD_DOC; 
+extern SortField SORT_FIELD_DOC_REV; 
+
+/***************************************************************************
+ * Sort
+ ***************************************************************************/
+
+typedef struct Sort {
+  SortField **sort_fields;
+  int sf_cnt;
+  int sf_capa;
+  bool destroy_all : 1;
+} Sort;
+
+Sort *sort_create();
+void sort_destroy(void *p);
+void sort_add_sort_field(Sort *self, SortField *sf);
+void sort_clear(Sort *self);
+
+/***************************************************************************
+ * FieldSortedHitQueue
+ ***************************************************************************/
+
+Hit *fshq_pq_pop(PriorityQueue *pq);
+void fshq_pq_down(PriorityQueue *pq);
+void fshq_pq_push(PriorityQueue *pq, void *elem);
+void fshq_pq_destroy(void *p);
+PriorityQueue *fshq_pq_create(int size, Sort *sort, IndexReader *ir);
+
+/***************************************************************************
+ *
+ * Searcher
+ *
+ ***************************************************************************/
+
+struct Searcher {
+  IndexReader *ir;
+  Similarity  *similarity;
+  int          (*doc_freq)(Searcher *self, Term *term);
+  int         *(*doc_freqs)(Searcher *self, Term **terms, int tcnt);
+  Document    *(*get_doc)(Searcher *self, int doc_num);
+  int          (*max_doc)(Searcher *self);
+  Weight      *(*create_weight)(Searcher *self, Query *query);
+  TopDocs     *(*search)(Searcher *self, Query *query, int first_doc,
+      int num_docs, Filter *filter, Sort *sort);
+  Query       *(*rewrite)(Searcher *self, Query *original);
+  Explanation *(*explain)(Searcher *self, Query *query, int doc_num);
+  Similarity  *(*get_similarity)(Searcher *self);
+  void         (*close)(Searcher *self);
+};
+
+Searcher *sea_create(IndexReader *ir);
+TopDocs *sea_search(Searcher *self, Query *query, int first_doc,
+    int num_docs, Filter *filter, Sort *sort);
+void sea_search_each(Searcher *self, Query *query, Filter *filter,
+    void (*fn)(Searcher *self, int doc_num, void *arg), void *arg);
+Explanation *sea_explain(Searcher *self, Query *query, int doc_num);
+Similarity *sea_get_similarity(Searcher *self);
+Query *sea_rewrite(Searcher *self, Query *original);
+void sea_close(Searcher *self);
+Document *sea_get_doc(Searcher *self, int doc_num);
+Weight *sea_create_weight(Searcher *self, Query *query);
+int sea_doc_freq(Searcher *self, Term *term);
+
+/***************************************************************************
+ *
+ * QParser
+ *
+ ***************************************************************************/
+
+#define CONC_WORDS 2
+
+typedef struct QParser {
+  mutex_t mutex;
+  bool or_default : 1;
+  bool wild_lower : 1;
+  bool clean_str : 1;
+  bool handle_parse_errors : 1;
+  bool allow_any_fields : 1;
+  bool close_def_fields : 1;
+  int def_slop;
+  char *qstr;
+  char *qstrp;
+  char buf[CONC_WORDS][MAX_WORD_SIZE];
+  int  buf_index;
+  HashSet *fields;
+  HashSet *fields_buf;
+  HashSet *def_fields;
+  HashSet *all_fields;
+  Analyzer *analyzer;
+  Query *result;
+} QParser;
+
+QParser *qp_create(HashSet *all_fields, HashSet *def_fields, Analyzer *analyzer);
+void qp_destroy(void *p);
+Query *qp_parse(QParser *self, char *qstr);
+char *qp_clean_str(char *str);
+
+/***************************************************************************
+ *
+ * Index
+ *
+ ***************************************************************************/
+
+typedef struct Index {
+  mutex_t mutex;
+  Store *store;
+  Analyzer *analyzer;
+  IndexReader *ir;
+  IndexWriter *iw;
+  Searcher *sea;
+  QParser *qp;
+  HashSet *key;
+  char *id_field;
+  char *def_field;
+  bool close_analyzer : 1;
+  bool close_store : 1;
+  /* for IndexWriter */
+  bool use_compound_file : 1;
+  bool auto_flush : 1;
+  bool has_writes : 1;
+} Index;
+
+Index *index_create(Store *store, Analyzer *analyzer, HashSet *def_fields,
+    bool create);
+void index_destroy(Index *self);
+void index_flush(Index *self);
+int index_size(Index *self);
+void index_optimize(Index *self);
+bool index_has_del(Index *self);
+bool index_is_deleted(Index *self, int doc_num);
+void index_add_doc(Index *self, Document *doc);
+void index_add_doc_a(Index *self, Document *doc, Analyzer *analyzer);
+void index_add_string(Index *self, char *str, Analyzer *analyzer);
+void index_add_array(Index *self, Array *ary, Analyzer *analyzer);
+TopDocs *index_search_str(Index *self, char *query, int first_doc,
+    int num_docs, Filter *filter, Sort *sort);
+Query *index_get_query(Index *self, char *qstr);
+Document *index_get_doc(Index *self, int doc_num);
+Document *index_get_doc_ts(Index *self, int doc_num);
+Document *index_get_doc_id(Index *self, char *id);
+Document *index_get_doc_term(Index *self, Term *term);
+void index_delete(Index *self, int doc_num);
+void index_delete_term(Index *self, Term *term);
+void index_delete_id(Index *self, char *id);
+void index_delete_query(Index *self, Query *q, Filter *f);
+void index_delete_query_str(Index *self, char *qstr, Filter *f);
+int index_term_id(Index *self, Term *term);
+Explanation *index_explain(Index *self, Query *q, int doc_num);
+void index_auto_flush_ir(Index *self);
+void index_auto_flush_iw(Index *self);
+#endif