ferret 0.9.6 → 0.10.0

data/ext/index.h

@@ -1,35 +1,40 @@
 #ifndef FRT_INDEX_H
 #define FRT_INDEX_H
 
-#include <limits.h>
 #include "global.h"
-#include "array.h"
-#include "bitvector.h"
-#include "hashset.h"
-#include "priorityqueue.h"
-#include "hash.h"
-#include "store.h"
 #include "document.h"
 #include "analysis.h"
+#include "hash.h"
+#include "hashset.h"
+#include "store.h"
+#include "mem_pool.h"
 #include "similarity.h"
+#include "bitvector.h"
+#include "priorityqueue.h"
 
+typedef struct IndexReader IndexReader;
+typedef struct MultiReader MultiReader;
 
-#define SEGMENT_NAME_MAX_LENGTH 100
-#define NOT_A_FIELD 0xFFFFFFFF
-
-typedef struct Config {
-  int merge_factor;
-  int min_merge_docs;
-  int max_merge_docs;
-  int max_field_length;
-  int term_index_interval;
-} FerretConfig;
+/****************************************************************************
+ *
+ * Config
+ *
+ ****************************************************************************/
 
-extern FerretConfig config;
+typedef struct Config
+{
+    int chunk_size;
+    int max_buffer_memory;
+    int index_interval;
+    int skip_interval;
+    int merge_factor;
+    int max_buffered_docs;
+    int max_merge_docs;
+    int max_field_length;
+    bool use_compound_file;
+} Config;
 
-typedef struct IndexReader IndexReader;
-typedef struct IndexWriter IndexWriter;
-typedef struct SegmentReader SegmentReader;
+extern const Config default_config;
 
 /***************************************************************************
  *
@@ -38,20 +43,18 @@ typedef struct SegmentReader SegmentReader;
  ***************************************************************************/
 
 typedef struct CacheObject {
-  HshTable *ref_tab1;
-  HshTable *ref_tab2;
-  void *ref1;
-  void *ref2;
-  void *obj;
-  void (*destroy)(void *p);
+    HashTable *ref_tab1;
+    HashTable *ref_tab2;
+    void *ref1;
+    void *ref2;
+    void *obj;
+    void (*destroy)(void *p);
 } CacheObject;
 
-void cache_destroy(CacheObject *co);
-CacheObject *co_create(HshTable *ref_tab1, HshTable *ref_tab2,
-    void *ref1, void *ref2, void (*destroy)(void *p), void *obj);
-unsigned int co_hash(const void *key);
-int co_eq(const void *key1, const void *key2);
-HshTable *co_hsh_create();
+extern void cache_destroy(CacheObject *co);
+extern CacheObject *co_create(HashTable *ref_tab1, HashTable *ref_tab2,
+            void *ref1, void *ref2, void (*destroy)(void *p), void *obj);
+extern HashTable *co_hash_create();
 
 /****************************************************************************
  *
@@ -59,384 +62,310 @@ HshTable *co_hsh_create();
  *
  ****************************************************************************/
 
-typedef struct FieldInfo {
-  char *name;
-  int number;
-  bool is_indexed   : 1;
-  bool store_tv     : 1;
-  bool store_offset : 1;
-  bool store_pos    : 1;
-  bool omit_norms   : 1;
-} FieldInfo;
-
-FieldInfo *fi_create(char *name,
-                     int  number,
-                     bool is_indexed,
-                     bool store_tv,
-                     bool store_pos,
-                     bool store_offset,
-                     bool omit_norms);
-void fi_destroy(FieldInfo *fi);
-
-/****************************************************************************
- *
- * FieldInfos
- *
- ****************************************************************************/
-
-typedef struct FieldInfos {
-  HashEntry **by_name;
-  FieldInfo **by_number;
-  int fcnt;
-} FieldInfos;
+enum StoreValues
+{
+    STORE_NO = 0,    
+    STORE_YES = 1,    
+    STORE_COMPRESS = 2
+};
 
-FieldInfos *fis_create();
-FieldInfos *fis_open(Store *store, char *filename);
-void fis_destroy(FieldInfos *fis);
-FieldInfo *fis_add(FieldInfos *fis,
-    char *name,
-    bool is_indexed,
-    bool store_tv,
-    bool store_offset,
-    bool store_pos,
-    bool omit_norms);
-
-void fis_add_fields(FieldInfos *fis,
-    HashSet *field_names,
-    bool is_indexed,
-    bool store_tv,
-    bool store_offset,
-    bool store_pos,
-    bool omit_norms);
-bool fis_has_vectors(FieldInfos *fis);
-void fis_write(FieldInfos *fis, Store *store, char *segment, char *ext);
-FieldInfos *fis_read(FieldInfos *fis, InStream *is);
-FieldInfos *fis_add_doc(FieldInfos *fis, Document *doc);
-ullong fis_get_number(FieldInfos *fis, char *name);
-FieldInfo *fis_get_fi(FieldInfos *fis, char *name);
-bool fis_reorder_required(FieldInfos *fis, Document *doc);
+enum IndexValues
+{
+    INDEX_NO = 0,    
+    INDEX_YES = 1,    
+    INDEX_UNTOKENIZED = 3,
+    INDEX_YES_OMIT_NORMS = 5,
+    INDEX_UNTOKENIZED_OMIT_NORMS = 7
+};
 
-/****************************************************************************
- *
- * TermBuffer
- *
- ****************************************************************************/
+enum TermVectorValues
+{
+    TERM_VECTOR_NO = 0,    
+    TERM_VECTOR_YES = 1,    
+    TERM_VECTOR_WITH_POSITIONS = 3,
+    TERM_VECTOR_WITH_OFFSETS = 5,
+    TERM_VECTOR_WITH_POSITIONS_OFFSETS = 7
+};
 
-typedef struct TermBuffer {
-  char *field;
-  char text[MAX_WORD_SIZE];
-} TermBuffer;
+#define FI_IS_STORED_BM         0x001
+#define FI_IS_COMPRESSED_BM     0x002
+#define FI_IS_INDEXED_BM        0x004
+#define FI_IS_TOKENIZED_BM      0x008
+#define FI_OMIT_NORMS_BM        0x010
+#define FI_STORE_TERM_VECTOR_BM 0x020
+#define FI_STORE_POSITIONS_BM   0x040
+#define FI_STORE_OFFSETS_BM     0x080
+
+typedef struct FieldInfo
+{
+    char *name;
+    float boost;
+    unsigned int bits;
+    int number;
+    int ref_cnt;
+} FieldInfo;
 
-TermBuffer *tb_create();
-void tb_destroy(TermBuffer *tb);
-TermBuffer *tb_set_term(TermBuffer *tb, Term *t);
-Term *tb_get_term(TermBuffer *tb);
-int tb_cmp(TermBuffer *tb1, TermBuffer *tb2);
-int tb_term_cmp(TermBuffer *tb, Term *t);
-TermBuffer *tb_cpy(TermBuffer *tb1, TermBuffer *tb2);
-TermBuffer *tb_read(TermBuffer *tb, InStream *is, FieldInfos *fis);
+extern FieldInfo *fi_new(const char *name,
+                         enum StoreValues store,
+                         enum IndexValues index,
+                         enum TermVectorValues term_vector);
+extern char *fi_to_s(FieldInfo *fi);
+extern void fi_deref(FieldInfo *fi);
+
+#define fi_is_stored(fi)         (((fi)->bits & FI_IS_STORED_BM) != 0)
+#define fi_is_compressed(fi)     (((fi)->bits & FI_IS_COMPRESSED_BM) != 0)
+#define fi_is_indexed(fi)        (((fi)->bits & FI_IS_INDEXED_BM) != 0)
+#define fi_is_tokenized(fi)      (((fi)->bits & FI_IS_TOKENIZED_BM) != 0)
+#define fi_omit_norms(fi)        (((fi)->bits & FI_OMIT_NORMS_BM) != 0)
+#define fi_store_term_vector(fi) (((fi)->bits & FI_STORE_TERM_VECTOR_BM) != 0)
+#define fi_store_positions(fi)   (((fi)->bits & FI_STORE_POSITIONS_BM) != 0)
+#define fi_store_offsets(fi)     (((fi)->bits & FI_STORE_OFFSETS_BM) != 0)
+#define fi_has_norms(fi)\
+    (((fi)->bits & (FI_OMIT_NORMS_BM|FI_IS_INDEXED_BM)) == FI_IS_INDEXED_BM)
 
 /****************************************************************************
  *
- * TermInfo
+ * FieldInfos
  *
  ****************************************************************************/
 
-typedef struct TermInfo {
-  int doc_freq;
-  int freq_pointer;
-  int prox_pointer;
-  int skip_offset;
-} TermInfo;
+#define FIELD_INFOS_INIT_CAPA 4
+typedef struct FieldInfos
+{
+    int store;
+    int index;
+    int term_vector;
+    int size;
+    int capa;
+    FieldInfo **fields;
+    HashTable *field_dict;
+    int ref_cnt;
+} FieldInfos;
 
-TermInfo *ti_create(int doc_freq, int freq_pointer, int prox_pointer, int skip_offset);
-TermInfo *ti_set(TermInfo *ti, int df, int fp, int pp, int so);
-void ti_destroy(TermInfo *ti);
-TermInfo *ti_cpy(TermInfo *ti1, TermInfo *ti2);
-TermInfo *ti_clone(TermInfo *other);
-int ti_eq(TermInfo *ti1, TermInfo *ti2);
+extern FieldInfos *fis_new(int store, int index, int term_vector);
+extern FieldInfo *fis_add_field(FieldInfos *fis, FieldInfo *fi);
+extern FieldInfo *fis_get_field(FieldInfos *fis, const char *name);
+extern int fis_get_field_num(FieldInfos *fis, const char *name);
+extern FieldInfo *fis_get_or_add_field(FieldInfos *fis, const char *name);
+extern void fis_write(FieldInfos *fis, Store *store);
+extern FieldInfos *fis_read(Store *store);
+extern char *fis_to_s(FieldInfos *fis);
+extern void fis_deref(FieldInfos *fis);
 
 /****************************************************************************
  *
- * TermEnum
+ * SegmentInfo
  *
  ****************************************************************************/
 
-typedef struct TermEnum TermEnum;
-struct TermEnum {
-  void *data;
-  TermBuffer *(*next)(TermEnum *te);
-  void (*close)(TermEnum *te);
-  TermEnum *(*clone)(TermEnum *te);
-  TermBuffer *tb_curr;
-  TermBuffer *tb_prev;
-  TermInfo *ti_curr;
-};
-
-TermBuffer *te_skip_to(struct TermEnum *te, Term *t);
-
-Term *te_get_term(struct TermEnum *te);
-TermInfo *te_get_ti(struct TermEnum *te);
-
-/* * SegmentTermEnum * */
-
-typedef struct SegmentTermEnum {
-  FieldInfos *fis;
-  int is_index;
-  InStream *is;
-  int size;
-  int pos;
-  int index_pointer;
-  int index_interval;
-  int skip_interval;
-  int format_m1skip_interval;
-  int format;
-} SegmentTermEnum;
-
-
-TermEnum *ste_create(InStream *is, FieldInfos *fis, int is_index);
-TermBuffer *ste_next(struct TermEnum *te);
-void ste_close(struct TermEnum *te);
-
-/* * MultiTermEnum * */
+#define SEGMENT_NAME_MAX_LENGTH 100
 
-typedef struct MultiTermEnum {
-  int doc_freq;
-  PriorityQueue *smi_queue;
-} MultiTermEnum;
+typedef struct SegmentInfo
+{
+    char *name;
+    int doc_cnt;
+    Store *store;
+} SegmentInfo;
 
-TermEnum *mte_create(IndexReader **readers, int *starts, int rcnt, Term *term);
+extern SegmentInfo *si_new(char *name, int doc_cnt, Store *store);
+extern void si_destroy(SegmentInfo *si);
+extern bool si_has_deletions(SegmentInfo *si);
+extern bool si_uses_compound_file(SegmentInfo *si);
+extern bool si_has_separate_norms(SegmentInfo *si);
 
 /****************************************************************************
  *
- * TermInfosWriter
+ * SegmentInfos
  *
  ****************************************************************************/
 
-#define TERM_INFO_FORMAT -2
-
-typedef struct TermInfosWriter {
-  int index_interval;
-  int skip_interval;
-  int size;
-  int last_index_pointer;
-  bool is_index;
-  OutStream *os;
-  struct TermInfosWriter *other;
-  Term *last_term;
-  TermInfo *last_term_info;
-  FieldInfos *fis;
-  char *curr_field;
-  ullong curr_field_num;
-} TermInfosWriter;
+typedef struct SegmentInfos
+{
+    f_u64 counter;
+    f_u64 version;
+    f_u32 format;
+    Store *store;
+    SegmentInfo **segs;
+    int size;
+    int capa;
+} SegmentInfos;
 
-TermInfosWriter *tiw_open(Store *store,
-                          char *segment,
-                          FieldInfos *fis,
-                          int interval);
-void tiw_add(TermInfosWriter *tiw, Term *t, TermInfo *ti);
-void tiw_close(TermInfosWriter *tiw);
+extern SegmentInfos *sis_new();
+extern SegmentInfo *sis_new_segment(SegmentInfos *sis, int dcnt, Store *store);
+extern SegmentInfo *sis_add_si(SegmentInfos *sis, SegmentInfo *si);
+extern void sis_del_at(SegmentInfos *sis, int at);
+extern void sis_del_from_to(SegmentInfos *sis, int from, int to);
+extern void sis_clear(SegmentInfos *sis);
+extern SegmentInfos *sis_read(Store *store);
+extern void sis_write(SegmentInfos *sis, Store *store);
+extern f_u64 sis_read_current_version(Store *store);
+extern void sis_destroy(SegmentInfos *sis);
 
 /****************************************************************************
  *
- * TermInfosReader
+ * TermInfo
  *
  ****************************************************************************/
 
-typedef struct TermInfosReader {
-  mutex_t mutex;
-  TermEnum *orig_te;
-  thread_key_t thread_te;
-  Array *te_bucket;
-  TermEnum *index_te;
-  int size;
-  int skip_interval;
-  int index_size;
-  Term **index_terms;
-  TermInfo **index_term_infos;
-  int *index_pointers;
-} TermInfosReader;
+typedef struct TermInfo
+{
+    int doc_freq;
+    off_t frq_ptr;
+    off_t prx_ptr;
+    off_t skip_offset;
+} TermInfo;
 
-TermInfosReader *tir_open(Store *store, char *segment, FieldInfos *fis);
-void tir_close(TermInfosReader *tir);
-Term *tir_get_term(TermInfosReader *tir, int position);
-int tir_get_term_pos(TermInfosReader *tir, Term *t);
-TermInfo *tir_get_ti(TermInfosReader *tir, Term *t);
+#define ti_set(ti, mdf, mfp, mpp, mso) do {\
+    (ti).doc_freq = mdf;\
+    (ti).frq_ptr = mfp;\
+    (ti).prx_ptr = mpp;\
+    (ti).skip_offset = mso;\
+} while (0)
 
 /****************************************************************************
  *
- * TVOffsetInfo
+ * TermEnum
  *
  ****************************************************************************/
 
-typedef struct TVOffsetInfo {
-  int start;
-  int end;
-} TVOffsetInfo;
-
-TVOffsetInfo *tvoi_create(int start, int end);
-void tvoi_destroy(void *p);
-
-/****************************************************************************
- *
- * TVField
- *
- ****************************************************************************/
+typedef struct TermEnum TermEnum;
 
-typedef struct TVField {
-  int tvf_pointer;
-  int number;
-  unsigned int store_positions : 1;
-  unsigned int store_offsets : 1;
-} TVField;
+struct TermEnum
+{
+    char        curr_term[MAX_WORD_SIZE];
+    char        prev_term[MAX_WORD_SIZE];
+    TermInfo    curr_ti;
+    int         curr_term_len;
+    int         field_num;
+    TermEnum *(*set_field)(TermEnum *te, int field_num);
+    char     *(*next)(TermEnum *te);
+    char     *(*skip_to)(TermEnum *te, const char *term);
+    void      (*close)(TermEnum *te);
+    TermEnum *(*clone)(TermEnum *te);
+};
 
-TVField *tvf_create(int number, int store_positions, int store_offsets);
-void tvf_destroy(void *p);
+char *te_get_term(struct TermEnum *te);
+TermInfo *te_get_ti(struct TermEnum *te);
 
 /****************************************************************************
  *
- * TVTerm
+ * SegmentTermEnum
  *
  ****************************************************************************/
 
-typedef struct TVTerm {
-    int field_num;
-    char *text;
-    int freq;
-    int *positions;
-    TVOffsetInfo **offsets;
-} TVTerm;
+/* * SegmentTermIndex * */
 
-TVTerm *tvt_create(char *text,
-                   int freq,
-                   int *positions,
-                   TVOffsetInfo **offsets);
-void tvt_destroy(void *p);
+typedef struct SegmentTermIndex
+{
+    off_t       index_ptr;
+    off_t       ptr;
+    int         index_size;
+    int         size;
+    char      **index_terms;
+    int        *index_term_lens;
+    TermInfo   *index_term_infos;
+    off_t      *index_ptrs;
+} SegmentTermIndex;
 
-/****************************************************************************
- *
- * TermVector
- *
- ****************************************************************************/
+/* * SegmentFieldIndex * */
 
-typedef struct TermVector {
-    char *field;
-    char **terms;
-    int tcnt;
-    int *freqs;
-    int **positions;
-    TVOffsetInfo ***offsets;
-} TermVector;
+typedef struct SegmentTermEnum SegmentTermEnum;
 
-TermVector *tv_create(const char *field,
-                      char **terms,
-                      int  tcnt,
-                      int  *freqs,
-                      int  **positions,
-                      TVOffsetInfo ***offsets);
-void tv_destroy(TermVector *tv);
+typedef struct SegmentFieldIndex
+{
+    mutex_t     mutex;
+    int         skip_interval;
+    int         index_interval;
+    off_t       index_ptr;
+    TermEnum   *index_te;
+    HashTable  *field_dict;
+} SegmentFieldIndex;
 
-/****************************************************************************
- *
- * TermVectorsWriter
- *
- ****************************************************************************/
+extern SegmentFieldIndex *sfi_open(Store *store, const char *segment);
+extern void sfi_close(SegmentFieldIndex *sfi);
 
-#define STORE_POSITIONS_WITH_TERMVECTOR 0x1
-#define STORE_OFFSET_WITH_TERMVECTOR 0x2
 
-#define FORMAT_VERSION 2
-#define FORMAT_SIZE 4
+/* * SegmentTermEnum * */
+struct SegmentTermEnum
+{
+    TermEnum    te;
+    InStream   *is;
+    int         size;
+    int         pos;
+    int         skip_interval;
+    SegmentFieldIndex *sfi;
+};
 
-#define TVX_EXTENSION ".tvx"
-#define TVD_EXTENSION ".tvd"
-#define TVF_EXTENSION ".tvf"
+extern void ste_close(TermEnum *te);
+extern TermEnum *ste_clone(TermEnum *te);
+extern TermEnum *ste_new(InStream *is, SegmentFieldIndex *sfi);
 
-typedef struct TermVectorsWriter {
-    TVField *curr_field;
-    int curr_doc_pointer;
-    OutStream *tvx;
-    OutStream *tvd;
-    OutStream *tvf;
-    FieldInfos *fis;
-    TVField **fields;
-    int fcnt;
-    int fsize;
-    TVTerm **terms;
-    int tcnt;
-    int tsize;
-} TermVectorsWriter;
-
-TermVectorsWriter *tvw_open(Store *store, char *segment, FieldInfos *fis);
-void tvw_close(TermVectorsWriter *tvw);
-void tvw_open_doc(TermVectorsWriter *tvw);
-void tvw_close_doc(TermVectorsWriter *tvw);
-void tvw_open_field(TermVectorsWriter *tvw, char *field);
-void tvw_close_field(TermVectorsWriter *tvw);
-void tvw_add_term(TermVectorsWriter *tvw, char *text, int freq, int *positions, TVOffsetInfo **offsets);
-void tvw_add_all_doc_vectors(TermVectorsWriter *tvw, Array *vectors);
+/* * MultiTermEnum * */
 
+extern TermEnum *mte_new(MultiReader *mr, int field_num, const char *term);
 
 /****************************************************************************
  *
- * TermVectorsReader
+ * TermInfosReader
  *
  ****************************************************************************/
 
-typedef struct TermVectorsReader {
-    int size;
-    InStream *tvx;
-    InStream *tvd;
-    InStream *tvf;
-    FieldInfos *fis;
-    int tvd_format;
-    int tvf_format;
-} TermVectorsReader;
-
-TermVectorsReader *tvr_open(Store *store, char *segment, FieldInfos *fis);
-TermVectorsReader *tvr_clone(TermVectorsReader *orig);
-void tvr_close(TermVectorsReader *tvr);
-TermVector *tvr_read_term_vector(TermVectorsReader *tvr,
-                                 char *field, int tvf_pointer);
-Array *tvr_get_tv(TermVectorsReader *tvr, int doc_num);
-TermVector *tvr_get_field_tv(TermVectorsReader *tvr, int doc_num, char *field);
+#define TE_BUCKET_INIT_CAPA 1
 
-/****************************************************************************
- *
- * FieldsReader
- *
- ****************************************************************************/
-
-typedef struct FieldsReader {
-    int len;
-    FieldInfos *fis;
-    InStream *fields_in;
-    InStream *index_in;
-} FieldsReader;
+typedef struct TermInfosReader
+{
+    thread_key_t thread_te;
+    void       **te_bucket;
+    TermEnum     *orig_te;
+    int          field_num;
+} TermInfosReader;
 
-FieldsReader *fr_open(Store *store, char *segment, FieldInfos *fis);
-void fr_close(FieldsReader *fr);
-Document *fr_get_doc(FieldsReader *fr, int doc_num);
+extern TermInfosReader *tir_open(Store *store,
+                                 SegmentFieldIndex *sfi,
+                                 const char *segment);
+extern TermInfosReader *tir_set_field(TermInfosReader *tir, int field_num);
+extern TermInfo *tir_get_ti(TermInfosReader *tir, const char *term);
+extern char *tir_get_term(TermInfosReader *tir, int pos);
+extern void tir_close(TermInfosReader *tir);
 
 /****************************************************************************
  *
- * FieldsWriter
+ * TermInfosWriter
  *
  ****************************************************************************/
 
-#define FIELD_IS_TOKENIZED 0X1
-#define FIELD_IS_BINARY 0X2
-#define FIELD_IS_COMPRESSED 0X4 
+#define INDEX_INTERVAL 128
+#define SKIP_INTERVAL 16
 
-typedef struct FieldsWriter {
-    FieldInfos *fis;
-    OutStream *fields_out;
-    OutStream *index_out;
-} FieldsWriter;
+typedef struct TermWriter
+{
+    int counter;
+    const char *last_term;
+    TermInfo last_term_info;
+    OutStream *os;
+} TermWriter;
+
+typedef struct TermInfosWriter
+{
+    int field_count;
+    int index_interval;
+    int skip_interval;
+    off_t last_index_ptr;
+    OutStream *tfx_out;
+    TermWriter *tix_writer;
+    TermWriter *tis_writer;
+} TermInfosWriter;
 
-FieldsWriter *fw_open(Store *store, char *segment, FieldInfos *fis);
-void fw_close(FieldsWriter *fw);
-void fw_add_doc(FieldsWriter *fw, Document *doc);
+extern TermInfosWriter *tiw_open(Store *store,
+                                 const char *segment,
+                                 int index_interval, 
+                                 int skip_interval);
+extern void tiw_start_field(TermInfosWriter *tiw, int field_num);
+extern void tiw_add(TermInfosWriter *tiw,
+                    const char *term,
+                    int t_len,
+                    TermInfo *ti);
+extern void tiw_close(TermInfosWriter *tiw);
 
 /****************************************************************************
  *
@@ -445,9 +374,10 @@ void fw_add_doc(FieldsWriter *fw, Document *doc);
  ****************************************************************************/
 
 typedef struct TermDocEnum TermDocEnum;
-struct TermDocEnum {
-    void *data;
-    void (*seek)(TermDocEnum *tde, Term *term);
+struct TermDocEnum
+{
+    void (*seek)(TermDocEnum *tde, int field_num, const char *term);
+    void (*seek_te)(TermDocEnum *tde, TermEnum *te);
     int  (*doc_num)(TermDocEnum *tde);
     int  (*freq)(TermDocEnum *tde);
     bool (*next)(TermDocEnum *tde);
@@ -460,71 +390,72 @@ struct TermDocEnum {
 /* * SegmentTermDocEnum * */
 
 typedef struct SegmentTermDocEnum SegmentTermDocEnum;
-
-struct SegmentTermDocEnum {
-    SegmentReader *parent;
-    InStream *freq_in;
+struct SegmentTermDocEnum
+{
+    TermDocEnum tde;
+    void (*seek_prox)(SegmentTermDocEnum *stde, int prx_ptr);
+    void (*skip_prox)(SegmentTermDocEnum *stde);
+    TermInfosReader *tir;
+    InStream        *frq_in;
+    InStream        *prx_in;
+    InStream        *skip_in;
+    BitVector       *deleted_docs;
     int count;               /* number of docs for this term  skipped */
     int doc_freq;            /* number of doc this term appears in */
-    BitVector *deleted_docs;
     int doc_num;
     int freq;
-    int skip_interval;
     int num_skips;
+    int skip_interval;
     int skip_count;
-    InStream *skip_in;
     int skip_doc;
-    int freq_pointer;
-    int prox_pointer;
-    int skip_pointer;
-    unsigned int have_skipped : 1;
-    void (*skip_prox)(SegmentTermDocEnum *stde);
-    InStream *prox_in;
-    int prox_cnt;
+    int frq_ptr;
+    int prx_ptr;
+    int skip_ptr;
+    int prx_cnt;
     int position;
-    void (*seek_prox)(SegmentTermDocEnum *stde, int prox_pointer);
+    bool have_skipped : 1;
 };
 
-TermDocEnum *stde_create(IndexReader *ir);
-void stde_seek_ti(TermDocEnum *tde, TermInfo *ti);
+extern TermDocEnum *stde_new(TermInfosReader *tir, InStream *frq_in,
+                             BitVector *deleted_docs, int skip_interval);
 
-/* * SegmentTermPosEnum * */
-TermDocEnum *stpe_create(IndexReader *ir);
-
-/* * MultiTermDocEnum * */
-typedef struct MultiTermDocEnum MultiTermDocEnum;
-struct MultiTermDocEnum {
-    IndexReader **irs;
-    int *starts;
-    int ir_cnt;
-    Term *term;
-    int base;
-    int pointer;
-    TermDocEnum **irs_tde;
-    TermDocEnum *curr_tde;
-    TermDocEnum *(*term_docs_from_reader)(IndexReader *ir);
-};
+/* * SegmentTermDocEnum * */
+extern TermDocEnum *stpe_new(TermInfosReader *tir, InStream *frq_in,
+                             InStream *prx_in, BitVector *deleted_docs,
+                             int skip_interval);
 
-TermDocEnum *mtde_create(IndexReader **readers, int *starts, int ir_cnt);
+/****************************************************************************
+ * MultipleTermDocPosEnum
+ ****************************************************************************/
 
-/* * MultiTermPosEnum * */
-TermDocEnum *mtpe_create(IndexReader **readers, int *starts, int ir_cnt);
+extern TermDocEnum *mtdpe_new(IndexReader *ir, int field_num, char **terms,
+                              int t_cnt);
 
 /****************************************************************************
- * MultipleTermDocPosEnum
+ *
+ * Offset
+ *
  ****************************************************************************/
 
-#define  MTDPE_POS_QUEUE_INIT_CAPA 8
-typedef struct {
-    int doc_num;
-    int freq;
-    PriorityQueue *pq;
-    int *pos_queue;
-    int pos_queue_index;
-    int pos_queue_capa;
-} MultipleTermDocPosEnum;
+typedef struct Offset
+{
+    int start;
+    int end;
+} Offset;
 
-TermDocEnum *mtdpe_create(IndexReader *ir, Term **terms, int t_cnt);
+extern Offset *offset_new(int start, int end);
+
+/****************************************************************************
+ *
+ * Occurence
+ *
+ ****************************************************************************/
+
+typedef struct Occurence
+{
+    struct Occurence *next;
+    int pos;
+} Occurence;
 
 /****************************************************************************
  *
@@ -532,283 +463,388 @@ TermDocEnum *mtdpe_create(IndexReader *ir, Term **terms, int t_cnt);
  *
  ****************************************************************************/
 
-typedef struct Posting {
-    Term *term;
+typedef struct Posting
+{
     int freq;
-    int size;
-    int *positions;
-    TVOffsetInfo **offsets;
+    int doc_num;
+    Occurence *first_occ;
+    struct Posting *next;
 } Posting;
 
-Posting *p_create(Term *term, int position, TVOffsetInfo *offset);
-void p_destroy(Posting *self);
-void p_add_occurance(Posting *self, int position, TVOffsetInfo *offset);
+extern inline Posting *p_new(MemoryPool *mp, int doc_num, int pos);
 
+/****************************************************************************
+ *
+ * PostingList
+ *
+ ****************************************************************************/
+
+typedef struct PostingList
+{
+    const char *term;
+    int term_len;
+    Posting *first;
+    Posting *last;
+    Occurence *last_occ;
+} PostingList;
+
+extern PostingList *pl_new(MemoryPool *mp, const char *term,
+                           int term_len, Posting *p);
+extern void pl_add_occ(MemoryPool *mp, PostingList *pl, int pos);
 
 /****************************************************************************
  *
- * DocumentWriter
+ * TVField
  *
  ****************************************************************************/
 
-typedef struct DocumentWriter {
-    Store *store;
-    Analyzer *analyzer;
-    Similarity *similarity;
-    HshTable *postingtable;
-    int pcnt;
-    FieldInfos *fis;
-    float *field_boosts;
-    int *field_lengths;
-    int *field_positions;
-    int *field_offsets;
-    int max_field_length;
-    int term_index_interval;
-} DocumentWriter; 
+typedef struct TVField
+{
+    int field_num;
+    int size;
+} TVField;
+
+/****************************************************************************
+ *
+ * TVTerm
+ *
+ ****************************************************************************/
 
-DocumentWriter *dw_open(Store *store, Analyzer *analyzer,
-                        Similarity *similarity, int max_field_length, int term_index_interval);
-void dw_close(DocumentWriter *dw);
-void dw_add_doc(DocumentWriter *dw, char *segment, Document *doc);
+typedef struct TVTerm
+{
+    char   *text;
+    int     freq;
+    int    *positions;
+} TVTerm;
 
 /****************************************************************************
  *
- * SegmentInfo
+ * TermVector
  *
  ****************************************************************************/
 
-typedef struct SegmentInfo {
-    char *name;
-    int doc_cnt;
-    Store *store;
-} SegmentInfo;
+typedef struct TermVector
+{
+    int     field_num;
+    char   *field;
+    int     term_cnt;
+    TVTerm *terms;
+    int     offset_cnt;
+    Offset *offsets;
+} TermVector;
 
-SegmentInfo *si_create(char *name, int doc_cnt, Store *store);
-void si_destroy(SegmentInfo *si);
-bool si_has_deletions(SegmentInfo *si);
-bool si_uses_compound_file(SegmentInfo *si);
-bool si_has_separate_norms(SegmentInfo *si);
+extern void tv_destroy(TermVector *tv);
+extern int tv_get_tv_term_index(TermVector *tv, const char *term);
+extern TVTerm *tv_get_tv_term(TermVector *tv, const char *term);
 
 /****************************************************************************
  *
- * SegmentInfos
+ * TermVectorsWriter
  *
  ****************************************************************************/
 
-typedef struct SegmentInfos {
-    Store *store;
-    SegmentInfo **segs;
-    int scnt;
-    int size;
-    int counter;
-    int version;
-    int format;
-} SegmentInfos;
+#define TV_FIELD_INIT_CAPA 8
+
+typedef struct TermVectorsWriter
+{
+    OutStream *tvx_out;
+    OutStream *tvd_out;
+    FieldInfos *fis;
+    TVField *fields;
+    off_t tvd_ptr;
+} TermVectorsWriter;
 
-SegmentInfos *sis_create();
-void sis_destroy(SegmentInfos *sis);
-void sis_add_si(SegmentInfos *sis, SegmentInfo *si);
-void sis_del_at(SegmentInfos *sis, int at);
-void sis_del_from_to(SegmentInfos *sis, int from, int to);
-void sis_clear(SegmentInfos *sis);
-void sis_read(SegmentInfos *sis, Store *store);
-void sis_write(SegmentInfos *sis, Store *store);
-int sis_read_current_version(Store *store);
+extern TermVectorsWriter *tvw_open(Store *store,
+                                   const char *segment,
+                                   FieldInfos *fis);
+extern void tvw_open_doc(TermVectorsWriter *tvw);
+extern void tvw_close_doc(TermVectorsWriter *tvw);
+extern void tvw_add_postings(TermVectorsWriter *tvw,
+                             int field_num,
+                             PostingList **plists,
+                             int posting_count,
+                             Offset *offsets,
+                             int offset_count);
+extern void tvw_close(TermVectorsWriter *tvw);
 
 /****************************************************************************
  *
- * IndexReader
+ * TermVectorsReader
  *
  ****************************************************************************/
 
-enum FIELD_TYPE {
-    /* all fields */
-    IR_ALL,
-    /* all indexed fields */
-    IR_INDEXED,
-    /* all fields which are not indexed */
-    IR_UNINDEXED,
-    /* all fields which are indexed with termvectors enables */
-    IR_INDEXED_WITH_TERM_VECTOR,
-    /* all fields which are indexed but don't have termvectors enabled */
-    IR_INDEXED_NO_TERM_VECTOR,
-    /* all fields where termvectors are enabled. Please note that only standard */
-    /* termvector fields are returned */
-    IR_TERM_VECTOR,
-    /* all field with termvectors wiht positions enabled */
-    IR_TERM_VECTOR_WITH_POSITION,
-    /* all fields where termvectors with offset position are set */
-    IR_TERM_VECTOR_WITH_OFFSET,
-    /* all fields where termvectors with offset and position values set */
-    IR_TERM_VECTOR_WITH_POSITION_OFFSET
-};
+typedef struct TermVectorsReader
+{
+  int size;
+  InStream *tvx_in;
+  InStream *tvd_in;
+  FieldInfos *fis;
+} TermVectorsReader;
 
-struct IndexReader {
-    mutex_t mutex;
-    HshTable *cache;
-    HshTable *sort_cache;
-    void *data;
-    Store *store;
-    Lock *write_lock;
-    SegmentInfos *sis;
-    bool has_changes : 1;
-    bool is_stale    : 1;
-    bool is_owner    : 1;
-    TermVector  *(*get_term_vector)(IndexReader *ir, int doc_num, char *field);
-    Array       *(*get_term_vectors)(IndexReader *ir, int doc_num);
-    int          (*num_docs)(IndexReader *ir);
-    int          (*max_doc)(IndexReader *ir);
-    Document    *(*get_doc)(IndexReader *ir, int doc_num);
-    uchar       *(*get_norms)(IndexReader *ir, char *field);
-    uchar       *(*get_norms_always)(IndexReader *ir, char *field);
-    void         (*do_set_norm)(IndexReader *ir, int doc_num, char *field,
-                                uchar val);
-    void         (*get_norms_into)(IndexReader *ir, char *field, uchar *buf,
-                                   int offset);
-    TermEnum    *(*terms)(IndexReader *ir);
-    TermEnum    *(*terms_from)(IndexReader *ir, Term *term);
-    int          (*doc_freq)(IndexReader *ir, Term *t);
-    TermDocEnum *(*term_docs)(IndexReader *ir);
-    TermDocEnum *(*term_positions)(IndexReader *ir);
-    void         (*do_delete_doc)(IndexReader *ir, int doc_num);
-    void         (*do_undelete_all)(IndexReader *ir);
-    bool         (*is_deleted)(IndexReader *ir, int doc_num);
-    bool         (*has_deletions)(IndexReader *ir);
-    bool         (*has_norms)(IndexReader *ir, char *field);
-    HashSet     *(*get_field_names)(IndexReader *ir, int field_type);
-    void         (*do_commit)(IndexReader *ir);
-    void         (*do_close)(IndexReader *ir);
-    void         (*acquire_write_lock)(IndexReader *ir);
+extern TermVectorsReader *tvr_open(Store *store,
+                                   const char *segment,
+                                   FieldInfos *fis);
+extern TermVectorsReader *tvr_clone(TermVectorsReader *orig);
+extern void tvr_close(TermVectorsReader *tvr);
+extern HashTable *tvr_get_tv(TermVectorsReader *tvr, int doc_num);
+extern TermVector *tvr_get_field_tv(TermVectorsReader *tvr,
+                                    int doc_num,
+                                    int field_num);
+
+/****************************************************************************
+ *
+ * LazyDoc
+ *
+ ****************************************************************************/
+
+/* * * LazyDocField * * */
+typedef struct LazyDocFieldData
+{
+    int   start;
+    int   length;
+    char *text;
+} LazyDocFieldData;
+
+typedef struct LazyDoc LazyDoc;
+typedef struct LazyDocField
+{
+    char             *name;
+    int               size; /* number of data elements */
+    LazyDocFieldData *data;
+    int               len;  /* length of data elements concatenated */
+    LazyDoc          *doc;
+} LazyDocField;
+
+extern char *lazy_df_get_data(LazyDocField *self, int i);
+extern void lazy_df_get_bytes(LazyDocField *self, char *buf,
+                              int start, int len);
+
+/* * * LazyDoc * * */
+struct LazyDoc
+{
+    HashTable *field_dict;
+    int size;
+    LazyDocField **fields;
+    InStream *fields_in;
 };
 
-IndexReader *ir_create(Store *store, SegmentInfos *sis, int is_owner);
-IndexReader *ir_open(Store *store);
-bool ir_index_exists(Store *store);
-void ir_close(IndexReader *ir);
-void ir_commit(IndexReader *ir);
-void ir_delete_doc(IndexReader *ir, int doc_num);
-void ir_undelete_all(IndexReader *ir);
-void ir_set_norm(IndexReader *ir, int doc_num, char *field, uchar val);
-void ir_destroy(IndexReader *self);
-Document *ir_get_doc_with_term(IndexReader *ir, Term *term);
-TermDocEnum *ir_term_docs_for(IndexReader *ir, Term *term);
-TermDocEnum *ir_term_positions_for(IndexReader *ir, Term *term);
-void ir_add_cache(IndexReader *ir);
-bool ir_is_latest(IndexReader *ir);
+extern void lazy_doc_close(LazyDoc *self);
 
 /****************************************************************************
  *
- * Norm
+ * FieldsReader
  *
  ****************************************************************************/
 
-typedef struct Norm {
-    bool is_dirty : 1;
-    int field_num;
-    InStream *is;
-    uchar *bytes;
-} Norm;
+typedef struct FieldsReader
+{
+    int         size;
+    FieldInfos *fis;
+    Store      *store;
+    InStream   *fdx_in;
+    InStream   *fdt_in;
+} FieldsReader;
+
+extern FieldsReader *fr_open(Store *store,
+                             const char *segment, FieldInfos *fis);
+extern FieldsReader *fr_clone(FieldsReader *orig);
+extern void fr_close(FieldsReader *fr);
+extern Document *fr_get_doc(FieldsReader *fr, int doc_num);
+extern LazyDoc *fr_get_lazy_doc(FieldsReader *fr, int doc_num);
+extern HashTable *fr_get_tv(FieldsReader *fr, int doc_num);
+extern TermVector *fr_get_field_tv(FieldsReader *fr, int doc_num,
+                            int field_num);
 
 /****************************************************************************
  *
- * SegmentReader
+ * FieldsWriter
  *
  ****************************************************************************/
 
-struct SegmentReader {
+typedef struct FieldsWriter
+{
     FieldInfos *fis;
-    FieldsReader *fr;
-    char *segment;
-    BitVector *deleted_docs;
-    bool deleted_docs_dirty : 1;
-    bool undelete_all : 1;
-    bool norms_dirty : 1;
-    InStream *freq_in;
-    InStream *prox_in;
-    TermInfosReader *tir;
-    TermVectorsReader *orig_tvr;
-    thread_key_t thread_tvr;
-    Array *tvr_bucket;
-    HshTable *norms;
-    Store *cfs_store;
-    uchar *fake_norms;
-};
+    OutStream  *fdt_out;
+    OutStream  *fdx_out;
+    TVField    *tv_fields;
+    off_t       start_ptr;
+} FieldsWriter;
 
-IndexReader *sr_open(SegmentInfos *sis, int si_num, bool is_owner);
-IndexReader *sr_open_si(SegmentInfo *si);
+extern FieldsWriter *fw_open(Store *store,
+                             const char *segment, FieldInfos *fis);
+extern void fw_close(FieldsWriter *fw);
+extern void fw_add_doc(FieldsWriter *fw, Document *doc);
+extern void fw_add_postings(FieldsWriter *fw,
+                            int field_num,
+                            PostingList **plists,
+                            int posting_count,
+                            Offset *offsets,
+                            int offset_count);
+extern void fw_write_tv_index(FieldsWriter *fw);
 
 /****************************************************************************
  *
- * MultiReader
+ * IndexReader
  *
  ****************************************************************************/
 
-typedef struct MultiReader {
-    bool has_deletions : 1;
+#define WRITE_LOCK_NAME "write"
+#define COMMIT_LOCK_NAME "commit"
+
+struct IndexReader
+{
+    int           (*num_docs)(IndexReader *ir);
+    int           (*max_doc)(IndexReader *ir);
+    Document     *(*get_doc)(IndexReader *ir, int doc_num);
+    LazyDoc      *(*get_lazy_doc)(IndexReader *ir, int doc_num);
+    uchar        *(*get_norms)(IndexReader *ir, int field_num);
+    uchar        *(*get_norms_into)(IndexReader *ir, int field_num,
+                                    uchar *buf);
+    TermEnum     *(*terms)(IndexReader *ir, int field_num);
+    TermEnum     *(*terms_from)(IndexReader *ir, int field_num,
+                                const char *term);
+    int           (*doc_freq)(IndexReader *ir, int field_num,
+                              const char *term);
+    TermDocEnum  *(*term_docs)(IndexReader *ir);
+    TermDocEnum  *(*term_positions)(IndexReader *ir);
+    TermVector   *(*term_vector)(IndexReader *ir, int doc_num,
+                                 const char *field);
+    HashTable    *(*term_vectors)(IndexReader *ir, int doc_num);
+    bool          (*is_deleted)(IndexReader *ir, int doc_num);
+    bool          (*has_deletions)(IndexReader *ir);
+    void          (*acquire_write_lock)(IndexReader *ir);
+    void          (*set_norm_i)(IndexReader *ir, int doc_num, int field_num,
+                        uchar val);
+    void          (*delete_doc_i)(IndexReader *ir, int doc_num);
+    void          (*undelete_all_i)(IndexReader *ir);
+    void          (*commit_i)(IndexReader *ir);
+    void          (*close_i)(IndexReader *ir);
+    int           ref_cnt;
+    Store        *store;
+    Lock         *write_lock;
+    SegmentInfos *sis;
+    FieldInfos   *fis;
+    HashTable    *cache;
+    HashTable    *sort_cache;
+    uchar        *fake_norms;
+    mutex_t       mutex;
+    bool          has_changes : 1;
+    bool          is_stale    : 1;
+    bool          is_owner    : 1;
+};
+
+extern IndexReader *ir_create(Store *store, SegmentInfos *sis, int is_owner);
+extern IndexReader *ir_open(Store *store);
+extern int ir_get_field_num(IndexReader *ir, const char *field);
+extern bool ir_index_exists(Store *store);
+extern void ir_close(IndexReader *ir);
+extern void ir_commit(IndexReader *ir);
+extern void ir_delete_doc(IndexReader *ir, int doc_num);
+extern void ir_undelete_all(IndexReader *ir);
+extern int ir_doc_freq(IndexReader *ir, const char *field, const char *term);
+extern void ir_set_norm(IndexReader *ir, int doc_num, const char *field,
+                        uchar val);
+extern uchar *ir_get_norms(IndexReader *ir, const char *field);
+extern uchar *ir_get_norms_into(IndexReader *ir, const char *field, uchar *buf);
+extern void ir_destroy(IndexReader *self);
+extern Document *ir_get_doc_with_term(IndexReader *ir, const char *field,
+                                      const char *term);
+extern TermEnum *ir_terms(IndexReader *ir, const char *field);
+extern TermEnum *ir_terms_from(IndexReader *ir, const char *field,
+                               const char *t);
+extern TermDocEnum *ir_term_docs_for(IndexReader *ir, const char *field,
+                                     const char *term);
+extern TermDocEnum *ir_term_positions_for(IndexReader *ir, const char *fld,
+                                          const char *t);
+extern void ir_add_cache(IndexReader *ir);
+extern bool ir_is_latest(IndexReader *ir);
+
+/****************************************************************************
+ * MultiReader
+ ****************************************************************************/
+
+struct MultiReader {
+    IndexReader ir;
     int max_doc;
     int num_docs_cache;
-    int  rcnt;
+    int r_cnt;
     int *starts;
     IndexReader **sub_readers;
-    HshTable *norms_cache;
-} MultiReader;
+    HashTable *norms_cache;
+    bool has_deletions : 1;
+    int **field_num_map;
+};
+
+extern int mr_get_field_num(MultiReader *mr, int ir_num, int f_num);
+extern IndexReader *mr_open(IndexReader **sub_readers, const int r_cnt);
 
-IndexReader *mr_open(Store *store, SegmentInfos *sis, IndexReader **readers,
-                     int rcnt);
 
 /****************************************************************************
  *
- * SegmentMergeInfo
+ * Boost
  *
  ****************************************************************************/
 
-typedef struct SegmentMergeInfo {
-    int base;
-    IndexReader *ir;
-    TermEnum *te;
-    TermBuffer *tb;
-    TermDocEnum *postings;
-    int *doc_map;
-} SegmentMergeInfo;
+typedef struct Boost
+{
+    float val;
+    int doc_num;
+    struct Boost *next;
+} Boost;
 
-SegmentMergeInfo *smi_create(int base, TermEnum *te, IndexReader *ir);
-void smi_destroy(SegmentMergeInfo *smi);
-TermBuffer *smi_next(SegmentMergeInfo *smi);
-bool smi_lt(SegmentMergeInfo *smi1, SegmentMergeInfo *smi2);
+/****************************************************************************
+ *
+ * FieldInverter
+ *
+ ****************************************************************************/
+
+typedef struct FieldInverter
+{
+    HashTable *plists;
+    uchar *norms;
+    FieldInfo *fi;
+    int length;
+    bool is_tokenized : 1;
+    bool store_term_vector : 1;
+    bool store_offsets : 1;
+    bool has_norms : 1;
+} FieldInverter;
 
 /****************************************************************************
  *
- * SegmentMerger
+ * DocWriter
  *
  ****************************************************************************/
 
-typedef struct SegmentMerger {
+#define DW_OFFSET_INIT_CAPA 512
+typedef struct IndexWriter IndexWriter;
+
+typedef struct DocWriter
+{
     Store *store;
-    char *name;
-    Array *readers;
+    const char *segment;
     FieldInfos *fis;
-    OutStream *freq_out;
-    OutStream *prox_out;
-    TermInfosWriter *tiw;
-    Term *terms_buf;
-    int terms_buf_pointer;
-    int terms_buf_size;
-    PriorityQueue *queue;
-    TermInfo *ti;
-    int term_index_interval;
-    OutStream *skip_buffer;
+    TermVectorsWriter *tvw;
+    FieldsWriter *fw;
+    MemoryPool *mp;
+    Analyzer *analyzer;
+    HashTable *curr_plists;
+    HashTable *fields;
+    Similarity *similarity;
+    Offset *offsets;
+    int offsets_size;
+    int offsets_capa;
+    int doc_num;
+    int index_interval;
     int skip_interval;
-    int last_skip_doc;
-    int last_skip_freq_pointer;
-    int last_skip_prox_pointer;
-} SegmentMerger;
-
-SegmentMerger *sm_create(Store *store, char *name, int term_index_interval);
-void sm_destroy(SegmentMerger *sm);
-void sm_add(SegmentMerger *sm, IndexReader *ir);
-int sm_merge(SegmentMerger *sm);
-Array *sm_create_compound_file(SegmentMerger *sm, char *fname);
+    int max_field_length;
+    int max_buffered_docs;
+} DocWriter;
 
+extern DocWriter *dw_open(IndexWriter *is, const char *segment);
+extern void dw_close(DocWriter *dw);
+extern void dw_add_doc(DocWriter *dw, Document *doc);
+extern void dw_new_segment(DocWriter *dw, char *segment);
 
 /****************************************************************************
  *
@@ -816,35 +852,38 @@ Array *sm_create_compound_file(SegmentMerger *sm, char *fname);
  *
  ****************************************************************************/
 
-#define WRITE_LOCK_NAME "write"
-#define COMMIT_LOCK_NAME "commit"
-struct IndexWriter {
+typedef struct DelTerm
+{
+    int field_num;
+    char *term;
+} DelTerm;
+
+struct IndexWriter
+{
+    Config config;
     mutex_t mutex;
-    HshTable *postings;
-    FieldInfos *fis;
-    int merge_factor;
-    int min_merge_docs;
-    int max_merge_docs;
-    int max_field_length;
-    int term_index_interval;
     Store *store;
     Analyzer *analyzer;
-    Similarity *similarity;
     SegmentInfos *sis;
-    Store *ram_store;
+    FieldInfos *fis;
+    DocWriter *dw;
+    Similarity *similarity;
+    DelTerm **del_terms;
     Lock *write_lock;
-    bool use_compound_file : 1;
 };
 
-IndexWriter *iw_open(Store *store, Analyzer *analyzer,
-                     bool create);
-void iw_flush_ram_segments(IndexWriter *iw);
-void iw_close(IndexWriter *iw);
-int  iw_doc_count(IndexWriter *iw);
-void iw_add_doc(IndexWriter *iw, Document *doc);
-void iw_optimize(IndexWriter *iw);
-void iw_add_indexes(IndexWriter *iw, Store **stores, int cnt);
-void iw_add_readers(IndexWriter *iw, IndexReader **stores, int cnt);
+extern void index_create(Store *store, FieldInfos *fis);
+extern IndexWriter *iw_open(Store *store, Analyzer *analyzer,
+                            const Config *config);
+extern void iw_delete_term(IndexWriter *iw, const char *field,
+                           const char *term);
+extern void iw_close(IndexWriter *iw);
+extern void iw_add_doc(IndexWriter *iw, Document *doc);
+extern int iw_doc_count(IndexWriter *iw);
+extern void iw_commit(IndexWriter *iw);
+extern void iw_optimize(IndexWriter *iw);
+extern void iw_add_readers(IndexWriter *iw, IndexReader **readers,
+                           const int r_cnt);
 
 /****************************************************************************
  *
@@ -852,16 +891,24 @@ void iw_add_readers(IndexWriter *iw, IndexReader **stores, int cnt);
  *
  ****************************************************************************/
 
+#define CW_INIT_CAPA 16
+typedef struct CWFileEntry
+{
+    char *name;
+    off_t dir_offset;
+    off_t data_offset;
+} CWFileEntry;
+
 typedef struct CompoundWriter {
     Store *store;
     const char *name;
     HashSet *ids;
-    Array *file_entries;
-    bool merged;
+    CWFileEntry *file_entries;
 } CompoundWriter;
 
-CompoundWriter *open_cw(Store *store, char *name);
-void cw_add_file(CompoundWriter *cw, char *id);
-void cw_close(CompoundWriter *cw);
+extern CompoundWriter *open_cw(Store *store, char *name);
+extern void cw_add_file(CompoundWriter *cw, char *id);
+extern void cw_close(CompoundWriter *cw);
+
 
 #endif