ferret 0.3.2 → 0.9.0

data/ext/index.h

@@ -0,0 +1,884 @@
+#ifndef FRT_INDEX_H
+#define FRT_INDEX_H
+
+#include <limits.h>
+#include "global.h"
+#include "array.h"
+#include "bitvector.h"
+#include "hashset.h"
+#include "priorityqueue.h"
+#include "hash.h"
+#include "store.h"
+#include "document.h"
+#include "analysis.h"
+
+#define SEGMENT_NAME_MAX_LENGTH 100
+
+typedef struct Config {
+  int merge_factor;
+  int min_merge_docs;
+  int max_merge_docs;
+  int max_field_length;
+  int term_index_interval;
+} FerretConfig;
+
+extern FerretConfig config;
+
+typedef struct IndexReader IndexReader;
+typedef struct IndexWriter IndexWriter;
+typedef struct SegmentReader SegmentReader;
+
+/***************************************************************************
+ *
+ * CacheObject
+ *
+ ***************************************************************************/
+
+typedef struct CacheObject {
+  HshTable *ref_tab1;
+  HshTable *ref_tab2;
+  void *ref1;
+  void *ref2;
+  void *obj;
+  void (*destroy)(void *p);
+} CacheObject;
+
+void cache_destroy(CacheObject *co);
+CacheObject *co_create(HshTable *ref_tab1, HshTable *ref_tab2,
+    void *ref1, void *ref2, void (*destroy)(void *p), void *obj);
+unsigned int co_hash(const void *key);
+int co_eq(const void *key1, const void *key2);
+HshTable *co_hsh_create();
+
+/****************************************************************************
+ *
+ * FieldInfo
+ *
+ ****************************************************************************/
+
+typedef struct FieldInfo {
+  char *name;
+  int number;
+  bool is_indexed   : 1;
+  bool store_tv     : 1;
+  bool store_offset : 1;
+  bool store_pos    : 1;
+  bool omit_norms   : 1;
+} FieldInfo;
+
+FieldInfo *fi_create(char *name,
+                     int  number,
+                     bool is_indexed,
+                     bool store_tv,
+                     bool store_pos,
+                     bool store_offset,
+                     bool omit_norms);
+void fi_destroy(void *p);
+
+/****************************************************************************
+ *
+ * FieldInfos
+ *
+ ****************************************************************************/
+
+typedef struct FieldInfos {
+  HashEntry **by_name;
+  FieldInfo **by_number;
+  int fcnt;
+} FieldInfos;
+
+FieldInfos *fis_create();
+FieldInfos *fis_open(Store *store, char *filename);
+void fis_destroy(void *p);
+FieldInfo *fis_add(FieldInfos *fis,
+    char *name,
+    bool is_indexed,
+    bool store_tv,
+    bool store_offset,
+    bool store_pos,
+    bool omit_norms);
+
+void fis_add_fields(FieldInfos *fis,
+    HashSet *field_names,
+    bool is_indexed,
+    bool store_tv,
+    bool store_offset,
+    bool store_pos,
+    bool omit_norms);
+bool fis_has_vectors(FieldInfos *fis);
+void fis_write(FieldInfos *fis, Store *store, char *segment, char *ext);
+FieldInfos *fis_read(FieldInfos *fis, InStream *is);
+FieldInfos *fis_add_doc(FieldInfos *fis, Document *doc);
+unsigned long long fis_get_number(FieldInfos *fis, char *name);
+FieldInfo *fis_get_fi(FieldInfos *fis, char *name);
+
+
+/****************************************************************************
+ *
+ * Term
+ *
+ ****************************************************************************/
+
+typedef struct Term {
+  char *field;
+  char *text;
+} Term;
+
+Term *term_clone(Term *term);
+Term *term_create(const char *field, char *text);
+void term_destroy(void *p);
+int term_cmp(void *t1, void *t2);
+int term_eq(const void *t1, const void *t2);
+unsigned int term_hash(const void *t);
+char *term_to_s(Term *term);
+
+/****************************************************************************
+ *
+ * TermBuffer
+ *
+ ****************************************************************************/
+
+typedef struct TermBuffer {
+  char *field;
+  char text[MAX_WORD_SIZE];
+} TermBuffer;
+
+TermBuffer *tb_create();
+void tb_destroy(void *p);
+TermBuffer *tb_set_term(TermBuffer *tb, Term *t);
+Term *tb_get_term(TermBuffer *tb);
+int tb_cmp(TermBuffer *tb1, TermBuffer *tb2);
+int tb_term_cmp(TermBuffer *tb, Term *t);
+TermBuffer *tb_cpy(TermBuffer *tb1, TermBuffer *tb2);
+TermBuffer *tb_read(TermBuffer *tb, InStream *is, FieldInfos *fis);
+
+/****************************************************************************
+ *
+ * TermInfo
+ *
+ ****************************************************************************/
+
+typedef struct TermInfo {
+  int doc_freq;
+  int freq_pointer;
+  int prox_pointer;
+  int skip_offset;
+} TermInfo;
+
+TermInfo *ti_create(int doc_freq, int freq_pointer, int prox_pointer, int skip_offset);
+TermInfo *ti_set(TermInfo *ti, int df, int fp, int pp, int so);
+void ti_destroy(void *p);
+TermInfo *ti_cpy(TermInfo *ti1, TermInfo *ti2);
+TermInfo *ti_clone(TermInfo *other);
+int ti_eq(TermInfo *ti1, TermInfo *ti2);
+
+/****************************************************************************
+ *
+ * TermEnum
+ *
+ ****************************************************************************/
+
+typedef struct TermEnumFilter TermEnumFilter;
+typedef struct TermEnum TermEnum;
+struct TermEnum {
+  void *data;
+  TermBuffer *(*next)(TermEnum *te);
+  void (*close)(TermEnum *te);
+  TermEnum *(*clone)(TermEnum *te);
+  TermBuffer *tb_curr;
+  TermBuffer *tb_prev;
+  TermInfo *ti_curr;
+};
+
+TermBuffer *te_skip_to(struct TermEnum *te, Term *t);
+
+Term *te_get_term(struct TermEnum *te);
+TermInfo *te_get_ti(struct TermEnum *te);
+
+/* * SegmentTermEnum * */
+
+typedef struct SegmentTermEnum {
+  FieldInfos *fis;
+  int is_index;
+  InStream *is;
+  int size;
+  int pos;
+  int index_pointer;
+  int index_interval;
+  int skip_interval;
+  int format_m1skip_interval;
+  int format;
+} SegmentTermEnum;
+
+
+TermEnum *ste_create(InStream *is, FieldInfos *fis, int is_index);
+TermBuffer *ste_next(struct TermEnum *te);
+void ste_close(struct TermEnum *te);
+
+/* * MultiTermEnum * */
+
+typedef struct MultiTermEnum {
+  int doc_freq;
+  PriorityQueue *smi_queue;
+} MultiTermEnum;
+
+TermEnum *mte_create(IndexReader **readers, int *starts, int rcnt, Term *term);
+
+/****************************************************************************
+ *
+ * TermInfosWriter
+ *
+ ****************************************************************************/
+
+#define TERM_INFO_FORMAT -2
+
+typedef struct TermInfosWriter {
+  int index_interval;
+  int skip_interval;
+  int size;
+  int last_index_pointer;
+  bool is_index;
+  OutStream *os;
+  struct TermInfosWriter *other;
+  Term *last_term;
+  TermInfo *last_term_info;
+  FieldInfos *fis;
+  char *curr_field;
+  int curr_field_num;
+} TermInfosWriter;
+
+TermInfosWriter *tiw_open(Store *store,
+                          char *segment,
+                          FieldInfos *fis,
+                          int interval);
+void tiw_add(TermInfosWriter *tiw, Term *t, TermInfo *ti);
+void tiw_close(TermInfosWriter *tiw);
+
+/****************************************************************************
+ *
+ * TermInfosReader
+ *
+ ****************************************************************************/
+
+typedef struct TermInfosReader {
+  mutex_t mutex;
+  TermEnum *orig_te;
+  thread_key_t thread_te;
+  Array *te_bucket;
+  TermEnum *index_te;
+  int size;
+  int skip_interval;
+  int index_size;
+  Term **index_terms;
+  TermInfo **index_term_infos;
+  int *index_pointers;
+} TermInfosReader;
+
+TermInfosReader *tir_open(Store *store, char *segment, FieldInfos *fis);
+void tir_close(TermInfosReader *tir);
+Term *tir_get_term(TermInfosReader *tir, int position);
+int tir_get_term_pos(TermInfosReader *tir, Term *t);
+TermInfo *tir_get_ti(TermInfosReader *tir, Term *t);
+
+/****************************************************************************
+ *
+ * TVOffsetInfo
+ *
+ ****************************************************************************/
+
+typedef struct TVOffsetInfo {
+  int start;
+  int end;
+} TVOffsetInfo;
+
+TVOffsetInfo *tvoi_create(int start, int end);
+void tvoi_destroy(void *p);
+
+/****************************************************************************
+ *
+ * TVField
+ *
+ ****************************************************************************/
+
+typedef struct TVField {
+  int tvf_pointer;
+  int number;
+  unsigned int store_positions : 1;
+  unsigned int store_offsets : 1;
+} TVField;
+
+TVField *tvf_create(int number, int store_positions, int store_offsets);
+void tvf_destroy(void *p);
+
+/****************************************************************************
+ *
+ * TVTerm
+ *
+ ****************************************************************************/
+
+typedef struct TVTerm {
+  char *text;
+  int freq;
+  int *positions;
+  TVOffsetInfo **offsets;
+} TVTerm;
+
+TVTerm *tvt_create(char *text,
+                   int freq,
+                   int *positions,
+                   TVOffsetInfo **offsets);
+void tvt_destroy(void *p);
+
+/****************************************************************************
+ *
+ * TermVector
+ *
+ ****************************************************************************/
+
+typedef struct TermVector {
+  char *field;
+  char **terms;
+  int tcnt;
+  int *freqs;
+  int **positions;
+  TVOffsetInfo ***offsets;
+} TermVector;
+
+TermVector *tv_create(const char *field,
+                      char **terms,
+                      int  tcnt,
+                      int  *freqs,
+                      int  **positions,
+                      TVOffsetInfo ***offsets);
+void tv_destroy(void *p);
+
+/****************************************************************************
+ *
+ * TermVectorsWriter
+ *
+ ****************************************************************************/
+
+#define STORE_POSITIONS_WITH_TERMVECTOR 0x1
+#define STORE_OFFSET_WITH_TERMVECTOR 0x2
+    
+#define FORMAT_VERSION 2
+#define FORMAT_SIZE 4
+    
+#define TVX_EXTENSION ".tvx"
+#define TVD_EXTENSION ".tvd"
+#define TVF_EXTENSION ".tvf"
+
+typedef struct TermVectorsWriter {
+  TVField *curr_field;
+  int curr_doc_pointer;
+  OutStream *tvx;
+  OutStream *tvd;
+  OutStream *tvf;
+  FieldInfos *fis;
+  TVField **fields;
+  int fcnt;
+  int fsize;
+  TVTerm **terms;
+  int tcnt;
+  int tsize;
+} TermVectorsWriter;
+
+TermVectorsWriter *tvw_open(Store *store, char *segment, FieldInfos *fis);
+void tvw_close(TermVectorsWriter *tvw);
+void tvw_open_doc(TermVectorsWriter *tvw);
+void tvw_close_doc(TermVectorsWriter *tvw);
+void tvw_open_field(TermVectorsWriter *tvw, char *field);
+void tvw_close_field(TermVectorsWriter *tvw);
+void tvw_add_term(TermVectorsWriter *tvw, char *text, int freq, int *positions, TVOffsetInfo **offsets);
+void tvw_add_all_doc_vectors(TermVectorsWriter *tvw, Array *vectors);
+
+
+/****************************************************************************
+ *
+ * TermVectorsReader
+ *
+ ****************************************************************************/
+
+typedef struct TermVectorsReader {
+  int size;
+  InStream *tvx;
+  InStream *tvd;
+  InStream *tvf;
+  FieldInfos *fis;
+  int tvd_format;
+  int tvf_format;
+} TermVectorsReader;
+
+TermVectorsReader *tvr_open(Store *store, char *segment, FieldInfos *fis);
+TermVectorsReader *tvr_clone(TermVectorsReader *orig);
+void tvr_close(TermVectorsReader *tvr);
+TermVector *tvr_read_term_vector(TermVectorsReader *tvr,
+    char *field, int tvf_pointer);
+Array *tvr_get_tv(TermVectorsReader *tvr, int doc_num);
+TermVector *tvr_get_field_tv(TermVectorsReader *tvr, int doc_num, char *field);
+
+/****************************************************************************
+ *
+ * FieldsWriter
+ *
+ ****************************************************************************/
+
+#define FIELD_IS_TOKENIZED 0X1
+#define FIELD_IS_BINARY 0X2
+#define FIELD_IS_COMPRESSED 0X4 
+
+typedef struct FieldsWriter {
+  FieldInfos *fis;
+  OutStream *fields_out;
+  OutStream *index_out;
+} FieldsWriter;
+
+FieldsWriter *fw_open(Store *store, char *segment, FieldInfos *fis);
+void fw_close(FieldsWriter *fw);
+void fw_add_doc(FieldsWriter *fw, Document *doc);
+
+/****************************************************************************
+ *
+ * TermDocEnum
+ *
+ ****************************************************************************/
+
+typedef struct TermDocEnum TermDocEnum;
+struct TermDocEnum {
+  void *data;
+  void (*seek)(TermDocEnum *tde, Term *term);
+  int  (*doc_num)(TermDocEnum *tde);
+  int  (*freq)(TermDocEnum *tde);
+  bool (*next)(TermDocEnum *tde);
+  int  (*read)(TermDocEnum *tde, int *docs, int *freqs, int req_num);
+  bool (*skip_to)(TermDocEnum *tde, int target);
+  int  (*next_position)(TermDocEnum *tde);
+  void (*close)(TermDocEnum *tde);
+};
+
+/* * SegmentTermDocEnum * */
+
+typedef struct SegmentTermDocEnum SegmentTermDocEnum;
+struct SegmentTermDocEnum {
+  SegmentReader *parent;
+  InStream *freq_in;
+  int count;               // the number of docs for this term that we have skipped
+  int doc_freq;            // the number of doc this term appears in
+  BitVector *deleted_docs;
+  int doc_num;
+  int freq;
+  int skip_interval;
+  int num_skips;
+  int skip_count;
+  InStream *skip_in;
+  int skip_doc;
+  int freq_pointer;
+  int prox_pointer;
+  int skip_pointer;
+  unsigned int have_skipped : 1;
+  void (*skip_prox)(SegmentTermDocEnum *stde);
+  InStream *prox_in;
+  int prox_cnt;
+  int position;
+  void (*seek_prox)(SegmentTermDocEnum *stde, int prox_pointer);
+};
+
+TermDocEnum *stde_create(IndexReader *ir);
+void stde_seek_ti(TermDocEnum *tde, TermInfo *ti);
+
+/* * SegmentTermPosEnum * */
+TermDocEnum *stpe_create(IndexReader *ir);
+
+/* * MultiTermDocEnum * */
+typedef struct MultiTermDocEnum MultiTermDocEnum;
+struct MultiTermDocEnum {
+  IndexReader **irs;
+  int *starts;
+  int ir_cnt;
+  Term *term;
+  int base;
+  int pointer;
+  TermDocEnum **irs_tde;
+  TermDocEnum *curr_tde;
+  TermDocEnum *(*term_docs_from_reader)(IndexReader *ir);
+};
+
+TermDocEnum *mtde_create(IndexReader **readers, int *starts, int ir_cnt);
+
+/* * MultiTermPosEnum * */
+TermDocEnum *mtpe_create(IndexReader **readers, int *starts, int ir_cnt);
+
+/****************************************************************************
+ * MultipleTermDocPosEnum
+ ****************************************************************************/
+
+#define  MTDPE_POS_QUEUE_INIT_CAPA 8
+typedef struct {
+  int doc_num;
+  int freq;
+  PriorityQueue *pq;
+  int *pos_queue;
+  int pos_queue_index;
+  int pos_queue_capa;
+} MultipleTermDocPosEnum;
+
+TermDocEnum *mtdpe_create(IndexReader *ir, Term **terms, int t_cnt);
+
+/****************************************************************************
+ *
+ * FieldsReader
+ *
+ ****************************************************************************/
+
+typedef struct FieldsReader {
+  int len;
+  FieldInfos *fis;
+  InStream *fields_in;
+  InStream *index_in;
+} FieldsReader;
+
+FieldsReader *fr_open(Store *store, char *segment, FieldInfos *fis);
+void fr_close(FieldsReader *fr);
+Document *fr_get_doc(FieldsReader *fr, int doc_num);
+
+/****************************************************************************
+ *
+ * Posting
+ *
+ ****************************************************************************/
+
+typedef struct Posting {
+  Term *term;
+  int freq;
+  int size;
+  int *positions;
+  TVOffsetInfo **offsets;
+} Posting;
+
+Posting *p_create(Term *term, int position, TVOffsetInfo *offset);
+void p_destroy(void *p);
+void p_add_occurance(Posting *p, int position, TVOffsetInfo *offset);
+
+
+/****************************************************************************
+ *
+ * DocumentWriter
+ *
+ ****************************************************************************/
+
+#include "search.h"
+
+typedef struct DocumentWriter {
+  Store *store;
+  Analyzer *analyzer;
+  Similarity *similarity;
+  HshTable *postingtable;
+  int pcnt;
+  FieldInfos *fis;
+  float *field_boosts;
+  int *field_lengths;
+  int *field_positions;
+  int *field_offsets;
+  int max_field_length;
+  int term_index_interval;
+} DocumentWriter; 
+
+DocumentWriter *dw_open(Store *store, Analyzer *analyzer,
+    Similarity *similarity, int max_field_length, int term_index_interval);
+void dw_close(DocumentWriter *dw);
+void dw_add_doc(DocumentWriter *dw, char *segment, Document *doc);
+
+/****************************************************************************
+ *
+ * SegmentInfo
+ *
+ ****************************************************************************/
+
+typedef struct SegmentInfo {
+  char *name;
+  int doc_cnt;
+  Store *store;
+} SegmentInfo;
+
+SegmentInfo *si_create(char *name, int doc_cnt, Store *store);
+void si_destroy(void *p);
+bool si_has_deletions(SegmentInfo *si);
+bool si_uses_compound_file(SegmentInfo *si);
+bool si_has_separate_norms(SegmentInfo *si);
+
+/****************************************************************************
+ *
+ * SegmentInfos
+ *
+ ****************************************************************************/
+
+typedef struct SegmentInfos {
+  Store *store;
+  SegmentInfo **segs;
+  int scnt;
+  int size;
+  int counter;
+  unsigned int version;
+  int format;
+} SegmentInfos;
+
+SegmentInfos *sis_create();
+void sis_destroy(void *p);
+void sis_add_si(SegmentInfos *sis, SegmentInfo *si);
+void sis_del_at(SegmentInfos *sis, int at);
+void sis_del_from_to(SegmentInfos *sis, int from, int to);
+void sis_clear(SegmentInfos *sis);
+void sis_read(SegmentInfos *sis, Store *store);
+void sis_write(SegmentInfos *sis, Store *store);
+int sis_read_current_version(Store *store);
+
+/****************************************************************************
+ *
+ * IndexReader
+ *
+ ****************************************************************************/
+
+enum FIELD_TYPE {
+  // all fields
+  IR_ALL,
+  // all indexed fields
+  IR_INDEXED,
+  // all fields which are not indexed
+  IR_UNINDEXED,
+  // all fields which are indexed with termvectors enables
+  IR_INDEXED_WITH_TERM_VECTOR,
+  // all fields which are indexed but don't have termvectors enabled
+  IR_INDEXED_NO_TERM_VECTOR,
+  // all fields where termvectors are enabled. Please note that only standard
+  // termvector fields are returned
+  IR_TERM_VECTOR,
+  // all field with termvectors wiht positions enabled
+  IR_TERM_VECTOR_WITH_POSITION,
+  // all fields where termvectors with offset position are set
+  IR_TERM_VECTOR_WITH_OFFSET,
+  // all fields where termvectors with offset and position values set
+  IR_TERM_VECTOR_WITH_POSITION_OFFSET
+};
+
+struct IndexReader {
+  mutex_t mutex;
+  HshTable *cache;
+  HshTable *sort_cache;
+  void *data;
+  Store *store;
+  Lock *write_lock;
+  SegmentInfos *sis;
+  bool has_changes : 1;
+  bool is_stale    : 1;
+  bool is_owner    : 1;
+  bool close_store : 1;
+  TermVector  *(*get_term_vector)(IndexReader *ir, int doc_num, char *field);
+  Array       *(*get_term_vectors)(IndexReader *ir, int doc_num);
+  int          (*num_docs)(IndexReader *ir);
+  int          (*max_doc)(IndexReader *ir);
+  Document    *(*get_doc)(IndexReader *ir, int doc_num);
+  uchar       *(*get_norms)(IndexReader *ir, char *field);
+  uchar       *(*get_norms_always)(IndexReader *ir, char *field);
+  void         (*do_set_norm)(IndexReader *ir, int doc_num, char *field, uchar val);
+  void         (*get_norms_into)(IndexReader *ir, char *field, uchar *buf, int offset);
+  TermEnum    *(*terms)(IndexReader *ir);
+  TermEnum    *(*terms_from)(IndexReader *ir, Term *term);
+  int          (*doc_freq)(IndexReader *ir, Term *t);
+  TermDocEnum *(*term_docs)(IndexReader *ir);
+  TermDocEnum *(*term_positions)(IndexReader *ir);
+  void         (*do_delete_doc)(IndexReader *ir, int doc_num);
+  void         (*do_undelete_all)(IndexReader *ir);
+  bool         (*is_deleted)(IndexReader *ir, int doc_num);
+  bool         (*has_deletions)(IndexReader *ir);
+  bool         (*has_norms)(IndexReader *ir, char *field);
+  HashSet     *(*get_field_names)(IndexReader *ir, int field_type);
+  void         (*do_commit)(IndexReader *ir);
+  void         (*do_close)(IndexReader *ir);
+  void         (*acquire_write_lock)(IndexReader *ir);
+};
+
+IndexReader *ir_create(Store *store, SegmentInfos *sis, int is_owner, int close_store);
+IndexReader *ir_open(Store *store, int close_store);
+bool ir_index_exists(Store *store);
+void ir_close(IndexReader *ir);
+void ir_commit(IndexReader *ir);
+void ir_delete_doc(IndexReader *ir, int doc_num);
+void ir_undelete_all(IndexReader *ir);
+void ir_set_norm(IndexReader *ir, int doc_num, char *field, uchar val);
+void ir_destroy(void *p);
+Document *ir_get_doc_with_term(IndexReader *ir, Term *term);
+TermDocEnum *ir_term_docs_for(IndexReader *ir, Term *term);
+TermDocEnum *ir_term_positions_for(IndexReader *ir, Term *term);
+void ir_add_cache(IndexReader *ir);
+bool ir_is_latest(IndexReader *ir);
+
+/****************************************************************************
+ *
+ * Norm
+ *
+ ****************************************************************************/
+
+typedef struct Norm {
+  bool is_dirty : 1;
+  int field_num;
+  InStream *is;
+  uchar *bytes;
+} Norm;
+
+/****************************************************************************
+ *
+ * SegmentReader
+ *
+ ****************************************************************************/
+
+struct SegmentReader {
+  FieldInfos *fis;
+  FieldsReader *fr;
+  char *segment;
+  BitVector *deleted_docs;
+  bool deleted_docs_dirty : 1;
+  bool undelete_all : 1;
+  bool norms_dirty : 1;
+  InStream *freq_in;
+  InStream *prox_in;
+  TermInfosReader *tir;
+  TermVectorsReader *orig_tvr;
+  thread_key_t thread_tvr;
+  Array *tvr_bucket;
+  HshTable *norms;
+  Store *cfs_store;
+  uchar *fake_norms;
+};
+
+IndexReader *sr_open(SegmentInfos *sis, int si_num, int is_owner, int close_store);
+IndexReader *sr_open_si(SegmentInfo *si);
+//int sr_has_deletions(IndexReader *ir);
+
+/****************************************************************************
+ *
+ * MultiReader
+ *
+ ****************************************************************************/
+
+typedef struct MultiReader {
+  bool has_deletions : 1;
+  int max_doc;
+  int num_docs_cache;
+  int  rcnt;
+  int *starts;
+  IndexReader **sub_readers;
+  HshTable *norms_cache;
+} MultiReader;
+
+IndexReader *mr_open(Store *store, SegmentInfos *sis, IndexReader **readers,
+    int rcnt, int close_store);
+
+/****************************************************************************
+ *
+ * SegmentMergeInfo
+ *
+ ****************************************************************************/
+
+typedef struct SegmentMergeInfo {
+  int base;
+  IndexReader *ir;
+  TermEnum *te;
+  TermBuffer *tb;
+  TermDocEnum *postings;
+  int *doc_map;
+} SegmentMergeInfo;
+
+SegmentMergeInfo *smi_create(int base, TermEnum *te, IndexReader *ir);
+void smi_destroy(void *p);
+TermBuffer *smi_next(SegmentMergeInfo *smi);
+bool smi_lt(void *p1, void *p2);
+
+/****************************************************************************
+ *
+ * SegmentMerger
+ *
+ ****************************************************************************/
+
+typedef struct SegmentMerger {
+  Store *store;
+  char *name;
+  Array *readers;
+  FieldInfos *fis;
+  OutStream *freq_out;
+  OutStream *prox_out;
+  TermInfosWriter *tiw;
+  Term *terms_buf;
+  int terms_buf_pointer;
+  int terms_buf_size;
+  PriorityQueue *queue;
+  TermInfo *ti;
+  int term_index_interval;
+  OutStream *skip_buffer;
+  int skip_interval;
+  int last_skip_doc;
+  int last_skip_freq_pointer;
+  int last_skip_prox_pointer;
+} SegmentMerger;
+
+SegmentMerger *sm_create(Store *store, char *name, int term_index_interval);
+void sm_destroy(void *p);
+void sm_add(SegmentMerger *sm, IndexReader *ir);
+int sm_merge(SegmentMerger *sm);
+Array *sm_create_compound_file(SegmentMerger *sm, char *fname);
+
+
+/****************************************************************************
+ *
+ * IndexWriter
+ *
+ ****************************************************************************/
+
+#define WRITE_LOCK_NAME "write"
+#define COMMIT_LOCK_NAME "commit"
+struct IndexWriter {
+  mutex_t mutex;
+  int merge_factor;
+  int min_merge_docs;
+  int max_merge_docs;
+  int max_field_length;
+  int term_index_interval;
+  Store *store;
+  Analyzer *analyzer;
+  Similarity *similarity;
+  SegmentInfos *sis;
+  Store *ram_store;
+  Lock *write_lock;
+  bool close_store : 1;
+  bool close_analyzer : 1;
+  bool use_compound_file : 1;
+};
+
+IndexWriter *iw_open(Store *store, Analyzer *analyzer,
+    bool create, bool close_store, bool close_analyzer);
+void iw_flush_ram_segments(IndexWriter *iw);
+void iw_close(IndexWriter *iw);
+int  iw_doc_count(IndexWriter *iw);
+void iw_add_doc(IndexWriter *iw, Document *doc);
+void iw_optimize(IndexWriter *iw);
+void iw_add_indexes(IndexWriter *iw, Store **stores, int cnt);
+void iw_add_readers(IndexWriter *iw, IndexReader **stores, int cnt);
+
+/****************************************************************************
+ *
+ * CompoundWriter
+ *
+ ****************************************************************************/
+
+typedef struct CompoundWriter {
+  Store *store;
+  const char *name;
+  HashSet *ids;
+  Array *file_entries;
+  bool merged;
+} CompoundWriter;
+
+CompoundWriter *open_cw(Store *store, char *name);
+void cw_add_file(CompoundWriter *cw, char *id);
+void cw_close(CompoundWriter *cw);
+
+#endif