isomorfeus-ferret 0.12.7 → 0.13.2

data/ext/isomorfeus_ferret_ext/frt_index.h

@@ -13,6 +13,7 @@
 #include "frt_priorityqueue.h"
 
 typedef struct FrtIndexReader FrtIndexReader;
+typedef struct FrtSegmentReader FrtSegmentReader;
 typedef struct FrtMultiReader FrtMultiReader;
 typedef struct FrtDeleter FrtDeleter;
 
@@ -22,8 +23,7 @@ typedef struct FrtDeleter FrtDeleter;
  *
  ****************************************************************************/
 
-typedef struct FrtConfig
-{
+typedef struct FrtConfig {
     int chunk_size;
     int max_buffer_memory;
     int index_interval;
@@ -52,8 +52,7 @@ typedef struct FrtCacheObject {
     void (*destroy)(void *p);
 } FrtCacheObject;
 
-extern FrtCacheObject *frt_co_create(FrtHash *ref_tab1,
-                              FrtHash *ref_tab2,
+extern FrtCacheObject *frt_co_create(FrtHash *ref_tab1, FrtHash *ref_tab2,
             void *ref1, void *ref2, frt_free_ft destroy, void *obj);
 extern FrtHash *frt_co_hash_create();
 
@@ -63,15 +62,12 @@ extern FrtHash *frt_co_hash_create();
  *
  ****************************************************************************/
 
-typedef enum
-{
+typedef enum {
     FRT_STORE_NO = 0,
     FRT_STORE_YES = 1,
-    FRT_STORE_COMPRESS = 2
 } FrtStoreValue;
 
-typedef enum
-{
+typedef enum {
     FRT_INDEX_NO = 0,
     FRT_INDEX_UNTOKENIZED = 1,
     FRT_INDEX_YES = 3,
@@ -79,8 +75,7 @@ typedef enum
     FRT_INDEX_YES_OMIT_NORMS = 7
 } FrtIndexValue;
 
-typedef enum
-{
+typedef enum {
     FRT_TERM_VECTOR_NO = 0,
     FRT_TERM_VECTOR_YES = 1,
     FRT_TERM_VECTOR_WITH_POSITIONS = 3,
@@ -88,39 +83,44 @@ typedef enum
     FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS = 7
 } FrtTermVectorValue;
 
-#define FRT_FI_IS_STORED_BM         0x001
-#define FRT_FI_IS_COMPRESSED_BM     0x002
-#define FRT_FI_IS_INDEXED_BM        0x004
-#define FRT_FI_IS_TOKENIZED_BM      0x008
-#define FRT_FI_OMIT_NORMS_BM        0x010
-#define FRT_FI_STORE_TERM_VECTOR_BM 0x020
-#define FRT_FI_STORE_POSITIONS_BM   0x040
-#define FRT_FI_STORE_OFFSETS_BM     0x080
-
-typedef struct FrtFieldInfo
-{
-    FrtSymbol name;
-    float boost;
+#define FRT_FI_IS_STORED_BM          0x001
+#define FRT_FI_IS_COMPRESSED_BM      0x002
+#define FRT_FI_IS_INDEXED_BM         0x004
+#define FRT_FI_IS_TOKENIZED_BM       0x008
+#define FRT_FI_OMIT_NORMS_BM         0x010
+#define FRT_FI_STORE_TERM_VECTOR_BM  0x020
+#define FRT_FI_STORE_POSITIONS_BM    0x040
+#define FRT_FI_STORE_OFFSETS_BM      0x080
+#define FRT_FI_COMPRESSION_BROTLI_BM 0x100
+#define FRT_FI_COMPRESSION_BZ2_BM    0x200
+#define FRT_FI_COMPRESSION_LZ4_BM    0x400
+
+typedef struct FrtFieldInfo {
+    ID           name;
+    float        boost;
     unsigned int bits;
-    int number;
-    int ref_cnt;
+    int          number;
+    int          ref_cnt;
+    VALUE        rfi;
 } FrtFieldInfo;
 
-extern FrtFieldInfo *frt_fi_new(FrtSymbol name,
-                                FrtStoreValue store,
-                                FrtIndexValue index,
-                                FrtTermVectorValue term_vector);
+extern FrtFieldInfo *frt_fi_alloc();
+extern FrtFieldInfo *frt_fi_init(FrtFieldInfo *fi, ID name, FrtStoreValue store, FrtCompressionType compression, FrtIndexValue index, FrtTermVectorValue term_vector);
+extern FrtFieldInfo *frt_fi_new(ID name, FrtStoreValue store, FrtCompressionType compression, FrtIndexValue index, FrtTermVectorValue term_vector);
 extern char *frt_fi_to_s(FrtFieldInfo *fi);
 extern void frt_fi_deref(FrtFieldInfo *fi);
 
-#define fi_is_stored(fi)         (((fi)->bits & FRT_FI_IS_STORED_BM) != 0)
-#define fi_is_compressed(fi)     (((fi)->bits & FRT_FI_IS_COMPRESSED_BM) != 0)
-#define fi_is_indexed(fi)        (((fi)->bits & FRT_FI_IS_INDEXED_BM) != 0)
-#define fi_is_tokenized(fi)      (((fi)->bits & FRT_FI_IS_TOKENIZED_BM) != 0)
-#define fi_omit_norms(fi)        (((fi)->bits & FRT_FI_OMIT_NORMS_BM) != 0)
-#define fi_store_term_vector(fi) (((fi)->bits & FRT_FI_STORE_TERM_VECTOR_BM) != 0)
-#define fi_store_positions(fi)   (((fi)->bits & FRT_FI_STORE_POSITIONS_BM) != 0)
-#define fi_store_offsets(fi)     (((fi)->bits & FRT_FI_STORE_OFFSETS_BM) != 0)
+#define fi_is_stored(fi)            (((fi)->bits & FRT_FI_IS_STORED_BM) != 0)
+#define fi_is_compressed(fi)        (((fi)->bits & FRT_FI_IS_COMPRESSED_BM) != 0)
+#define fi_is_compressed_brotli(fi) (((fi)->bits & FRT_FI_COMPRESSION_BROTLI_BM) != 0)
+#define fi_is_compressed_bz2(fi)    (((fi)->bits & FRT_FI_COMPRESSION_BZ2_BM) != 0)
+#define fi_is_compressed_lz4(fi)    (((fi)->bits & FRT_FI_COMPRESSION_LZ4_BM) != 0)
+#define fi_is_indexed(fi)           (((fi)->bits & FRT_FI_IS_INDEXED_BM) != 0)
+#define fi_is_tokenized(fi)         (((fi)->bits & FRT_FI_IS_TOKENIZED_BM) != 0)
+#define fi_omit_norms(fi)           (((fi)->bits & FRT_FI_OMIT_NORMS_BM) != 0)
+#define fi_store_term_vector(fi)    (((fi)->bits & FRT_FI_STORE_TERM_VECTOR_BM) != 0)
+#define fi_store_positions(fi)      (((fi)->bits & FRT_FI_STORE_POSITIONS_BM) != 0)
+#define fi_store_offsets(fi)        (((fi)->bits & FRT_FI_STORE_OFFSETS_BM) != 0)
 #define fi_has_norms(fi)\
     (((fi)->bits & (FRT_FI_OMIT_NORMS_BM|FRT_FI_IS_INDEXED_BM)) == FRT_FI_IS_INDEXED_BM)
 
@@ -132,25 +132,26 @@ extern void frt_fi_deref(FrtFieldInfo *fi);
 
 #define FIELD_INFOS_INIT_CAPA 4
 /* carry changes over to dummy_fis in test/test_segments.c */
-typedef struct FrtFieldInfos
-{
-    FrtStoreValue store;
-    FrtIndexValue index;
+typedef struct FrtFieldInfos {
+    FrtStoreValue      store;
+    FrtCompressionType compression;
+    FrtIndexValue      index;
     FrtTermVectorValue term_vector;
-    int size;
-    int capa;
-    FrtFieldInfo **fields;
-    FrtHash *field_dict;
-    int ref_cnt;
+    int                size;
+    int                capa;
+    FrtFieldInfo       **fields;
+    FrtHash            *field_dict;
+    int                ref_cnt;
+    VALUE              rfis;
 } FrtFieldInfos;
 
-FrtFieldInfos *frt_fis_new(FrtStoreValue store, FrtIndexValue index,
-                                  FrtTermVectorValue term_vector);
+FrtFieldInfos *frt_fis_alloc();
+FrtFieldInfos *frt_fis_init(FrtFieldInfos *fis, FrtStoreValue store, FrtCompressionType compression, FrtIndexValue index, FrtTermVectorValue term_vector);
+FrtFieldInfos *frt_fis_new(FrtStoreValue store, FrtCompressionType compression, FrtIndexValue index, FrtTermVectorValue term_vector);
 extern FrtFieldInfo *frt_fis_add_field(FrtFieldInfos *fis, FrtFieldInfo *fi);
-extern FrtFieldInfo *frt_fis_get_field(FrtFieldInfos *fis, FrtSymbol name);
-extern int frt_fis_get_field_num(FrtFieldInfos *fis, FrtSymbol name);
-extern FrtFieldInfo *frt_fis_get_or_add_field(FrtFieldInfos *fis,
-                                              FrtSymbol name);
+extern FrtFieldInfo *frt_fis_get_field(FrtFieldInfos *fis, ID name);
+extern int frt_fis_get_field_num(FrtFieldInfos *fis, ID name);
+extern FrtFieldInfo *frt_fis_get_or_add_field(FrtFieldInfos *fis, ID name);
 extern void frt_fis_write(FrtFieldInfos *fis, FrtOutStream *os);
 extern FrtFieldInfos *frt_fis_read(FrtInStream *is);
 extern char *frt_fis_to_s(FrtFieldInfos *fis);
@@ -165,8 +166,7 @@ extern void frt_fis_deref(FrtFieldInfos *fis);
 #define FRT_SEGMENT_NAME_MAX_LENGTH 100
 #define FRT_SEGMENTS_FILE_NAME "segments"
 
-typedef struct FrtSegmentInfo
-{
+typedef struct FrtSegmentInfo {
     int ref_cnt;
     char *name;
     FrtStore *store;
@@ -189,8 +189,7 @@ extern void frt_si_advance_norm_gen(FrtSegmentInfo *si, int field_num);
  *
  ****************************************************************************/
 
-typedef struct FrtSegmentInfos
-{
+typedef struct FrtSegmentInfos {
     FrtFieldInfos *fis;
     frt_u64 counter;
     frt_u64 version;
@@ -202,10 +201,7 @@ typedef struct FrtSegmentInfos
     int capa;
 } FrtSegmentInfos;
 
-extern char *frt_fn_for_generation(char *buf,
-                                   const char *base,
-                                   const char *ext,
-                                   frt_i64 gen);
+extern char *frt_fn_for_generation(char *buf, const char *base, const char *ext, frt_i64 gen);
 
 extern FrtSegmentInfos *frt_sis_new(FrtFieldInfos *fis);
 extern FrtSegmentInfo *frt_sis_new_segment(FrtSegmentInfos *sis, int dcnt, FrtStore *store);
@@ -226,9 +222,8 @@ extern void frt_sis_put(FrtSegmentInfos *sis, FILE *stream);
  *
  ****************************************************************************/
 
-typedef struct FrtTermInfo
-{
-    int doc_freq;
+typedef struct FrtTermInfo {
+    int   doc_freq;
     off_t frq_ptr;
     off_t prx_ptr;
     off_t skip_offset;
@@ -242,24 +237,21 @@ typedef struct FrtTermInfo
 } while (0)
 
 /****************************************************************************
- *
  * FrtTermEnum
- *
  ****************************************************************************/
 
 typedef struct FrtTermEnum FrtTermEnum;
 
-struct FrtTermEnum
-{
+struct FrtTermEnum {
     char        curr_term[FRT_MAX_WORD_SIZE];
     char        prev_term[FRT_MAX_WORD_SIZE];
-    FrtTermInfo    curr_ti;
+    FrtTermInfo curr_ti;
     int         curr_term_len;
     int         field_num;
     FrtTermEnum *(*set_field)(FrtTermEnum *te, int field_num);
-    char     *(*next)(FrtTermEnum *te);
-    char     *(*skip_to)(FrtTermEnum *te, const char *term);
-    void      (*close)(FrtTermEnum *te);
+    char        *(*next)(FrtTermEnum *te);
+    char        *(*skip_to)(FrtTermEnum *te, const char *term);
+    void        (*close)(FrtTermEnum *te);
     FrtTermEnum *(*clone)(FrtTermEnum *te);
 };
 
@@ -267,59 +259,54 @@ char *frt_te_get_term(struct FrtTermEnum *te);
 FrtTermInfo *frt_te_get_ti(struct FrtTermEnum *te);
 
 /****************************************************************************
- *
  * FrtSegmentTermEnum
- *
  ****************************************************************************/
 
-/* * FrtSegmentTermIndex * */
+/* FrtSegmentTermIndex */
 
-typedef struct FrtSegmentTermIndex
-{
+typedef struct FrtSegmentTermIndex {
     off_t       index_ptr;
     off_t       ptr;
     int         index_cnt;
     int         size;
-    char      **index_terms;
-    int        *index_term_lens;
-    FrtTermInfo   *index_term_infos;
-    off_t      *index_ptrs;
+    char        **index_terms;
+    int         *index_term_lens;
+    FrtTermInfo *index_term_infos;
+    off_t       *index_ptrs;
 } FrtSegmentTermIndex;
 
-/* * FrtSegmentFieldIndex * */
+/* FrtSegmentFieldIndex */
 
-typedef struct FrtSegmentTermEnum FrtSegmentTermEnum;
-
-typedef struct FrtSegmentFieldIndex
-{
-    frt_mutex_t     mutex;
+typedef struct FrtSegmentFieldIndex {
+    frt_mutex_t mutex;
     int         skip_interval;
     int         index_interval;
     off_t       index_ptr;
-    FrtTermEnum   *index_te;
-    FrtHash  *field_dict;
+    FrtTermEnum *index_te;
+    FrtHash     *field_dict;
 } FrtSegmentFieldIndex;
 
-extern FrtSegmentFieldIndex *frt_sfi_open(FrtStore *store, const char *segment);
-extern void frt_sfi_close(FrtSegmentFieldIndex *sfi);
+/* FrtSegmentTermEnum */
 
+typedef struct FrtSegmentTermEnum FrtSegmentTermEnum;
 
-/* * FrtSegmentTermEnum * */
-struct FrtSegmentTermEnum
-{
-    FrtTermEnum    te;
-    FrtInStream   *is;
+struct FrtSegmentTermEnum {
+    FrtTermEnum te;
+    FrtInStream *is;
     int         size;
     int         pos;
     int         skip_interval;
     FrtSegmentFieldIndex *sfi;
 };
 
+extern FrtSegmentFieldIndex *frt_sfi_open(FrtStore *store, const char *segment);
+extern void frt_sfi_close(FrtSegmentFieldIndex *sfi);
+
 extern void frt_ste_close(FrtTermEnum *te);
 extern FrtTermEnum *frt_ste_clone(FrtTermEnum *te);
 extern FrtTermEnum *frt_ste_new(FrtInStream *is, FrtSegmentFieldIndex *sfi);
 
-/* * MultiTermEnum * */
+/* MultiTermEnum */
 
 extern FrtTermEnum *frt_mte_new(FrtMultiReader *mr, int field_num, const char *term);
 
@@ -329,17 +316,14 @@ extern FrtTermEnum *frt_mte_new(FrtMultiReader *mr, int field_num, const char *t
  *
  ****************************************************************************/
 
-typedef struct FrtTermInfosReader
-{
+typedef struct FrtTermInfosReader {
     frt_thread_key_t thread_te;
-    void       **te_bucket;
-    FrtTermEnum     *orig_te;
-    int          field_num;
+    void             **te_bucket;
+    FrtTermEnum      *orig_te;
+    int              field_num;
 } FrtTermInfosReader;
 
-extern FrtTermInfosReader *frt_tir_open(FrtStore *store,
-                                 FrtSegmentFieldIndex *sfi,
-                                 const char *segment);
+extern FrtTermInfosReader *frt_tir_open(FrtStore *store, FrtSegmentFieldIndex *sfi, const char *segment);
 extern FrtTermInfosReader *frt_tir_set_field(FrtTermInfosReader *tir, int field_num);
 extern FrtTermInfo *frt_tir_get_ti(FrtTermInfosReader *tir, const char *term);
 extern char *frt_tir_get_term(FrtTermInfosReader *tir, int pos);
@@ -354,34 +338,26 @@ extern void frt_tir_close(FrtTermInfosReader *tir);
 #define FRT_INDEX_INTERVAL 128
 #define FRT_SKIP_INTERVAL 16
 
-typedef struct FrtTermWriter
-{
-    int counter;
-    const char *last_term;
-    FrtTermInfo last_term_info;
+typedef struct FrtTermWriter {
+    int          counter;
+    const char   *last_term;
+    FrtTermInfo  last_term_info;
     FrtOutStream *os;
 } FrtTermWriter;
 
-typedef struct FrtTermInfosWriter
-{
-    int field_count;
-    int index_interval;
-    int skip_interval;
-    off_t last_index_ptr;
-    FrtOutStream *tfx_out;
+typedef struct FrtTermInfosWriter {
+    int           field_count;
+    int           index_interval;
+    int           skip_interval;
+    off_t         last_index_ptr;
+    FrtOutStream  *tfx_out;
     FrtTermWriter *tix_writer;
     FrtTermWriter *tis_writer;
 } FrtTermInfosWriter;
 
-extern FrtTermInfosWriter *frt_tiw_open(FrtStore *store,
-                                 const char *segment,
-                                 int index_interval,
-                                 int skip_interval);
+extern FrtTermInfosWriter *frt_tiw_open(FrtStore *store, const char *segment, int index_interval, int skip_interval);
 extern void frt_tiw_start_field(FrtTermInfosWriter *tiw, int field_num);
-extern void frt_tiw_add(FrtTermInfosWriter *tiw,
-                    const char *term,
-                    int t_len,
-                    FrtTermInfo *ti);
+extern void frt_tiw_add(FrtTermInfosWriter *tiw, const char *term, int t_len, FrtTermInfo *ti);
 extern void frt_tiw_close(FrtTermInfosWriter *tiw);
 
 /****************************************************************************
@@ -391,8 +367,7 @@ extern void frt_tiw_close(FrtTermInfosWriter *tiw);
  ****************************************************************************/
 
 typedef struct FrtTermDocEnum FrtTermDocEnum;
-struct FrtTermDocEnum
-{
+struct FrtTermDocEnum {
     void (*seek)(FrtTermDocEnum *tde, int field_num, const char *term);
     void (*seek_te)(FrtTermDocEnum *tde, FrtTermEnum *te);
     void (*seek_ti)(FrtTermDocEnum *tde, FrtTermInfo *ti);
@@ -408,8 +383,7 @@ struct FrtTermDocEnum
 /* * FrtSegmentTermDocEnum * */
 
 typedef struct FrtSegmentTermDocEnum FrtSegmentTermDocEnum;
-struct FrtSegmentTermDocEnum
-{
+struct FrtSegmentTermDocEnum {
     FrtTermDocEnum tde;
     void (*seek_prox)(FrtSegmentTermDocEnum *stde, off_t prx_ptr);
     void (*skip_prox)(FrtSegmentTermDocEnum *stde);
@@ -446,8 +420,7 @@ extern FrtTermDocEnum *frt_stpe_new(FrtTermInfosReader *tir, FrtInStream *frq_in
  * MultipleTermDocPosEnum
  ****************************************************************************/
 
-extern FrtTermDocEnum *frt_mtdpe_new(FrtIndexReader *ir, int field_num, char **terms,
-                              int t_cnt);
+extern FrtTermDocEnum *frt_mtdpe_new(FrtIndexReader *ir, int field_num, char **terms, int t_cnt);
 
 /****************************************************************************
  *
@@ -455,8 +428,7 @@ extern FrtTermDocEnum *frt_mtdpe_new(FrtIndexReader *ir, int field_num, char **t
  *
  ****************************************************************************/
 
-typedef struct FrtOffset
-{
+typedef struct FrtOffset {
     off_t start;
     off_t end;
 } FrtOffset;
@@ -467,8 +439,7 @@ typedef struct FrtOffset
  *
  ****************************************************************************/
 
-typedef struct FrtOccurence
-{
+typedef struct FrtOccurence {
     struct FrtOccurence *next;
     int pos;
 } FrtOccurence;
@@ -479,8 +450,7 @@ typedef struct FrtOccurence
  *
  ****************************************************************************/
 
-typedef struct FrtPosting
-{
+typedef struct FrtPosting {
     int freq;
     int doc_num;
     FrtOccurence *first_occ;
@@ -495,17 +465,15 @@ extern FrtPosting *frt_p_new(FrtMemoryPool *mp, int doc_num, int pos);
  *
  ****************************************************************************/
 
-typedef struct FrtPostingList
-{
-    const char *term;
-    int term_len;
-    FrtPosting *first;
-    FrtPosting *last;
+typedef struct FrtPostingList {
+    const char   *term;
+    int          term_len;
+    FrtPosting   *first;
+    FrtPosting   *last;
     FrtOccurence *last_occ;
 } FrtPostingList;
 
-extern FrtPostingList *frt_pl_new(FrtMemoryPool *mp, const char *term,
-                           int term_len, FrtPosting *p);
+extern FrtPostingList *frt_pl_new(FrtMemoryPool *mp, const char *term, int term_len, FrtPosting *p);
 extern void frt_pl_add_occ(FrtMemoryPool *mp, FrtPostingList *pl, int pos);
 extern int frt_pl_cmp(const FrtPostingList **pl1, const FrtPostingList **pl2);
 
@@ -515,8 +483,7 @@ extern int frt_pl_cmp(const FrtPostingList **pl1, const FrtPostingList **pl2);
  *
  ****************************************************************************/
 
-typedef struct FrtTVField
-{
+typedef struct FrtTVField {
     int field_num;
     int size;
 } FrtTVField;
@@ -527,11 +494,10 @@ typedef struct FrtTVField
  *
  ****************************************************************************/
 
-typedef struct FrtTVTerm
-{
-    char   *text;
-    int     freq;
-    int    *positions;
+typedef struct FrtTVTerm {
+    char *text;
+    int  freq;
+    int  *positions;
 } FrtTVTerm;
 
 /****************************************************************************
@@ -541,10 +507,9 @@ typedef struct FrtTVTerm
  ****************************************************************************/
 
 #define FRT_TV_FIELD_INIT_CAPA 8
-typedef struct FrtTermVector
-{
+typedef struct FrtTermVector {
     int       field_num;
-    FrtSymbol field;
+    ID        field;
     int       term_cnt;
     FrtTVTerm *terms;
     int       offset_cnt;
@@ -563,38 +528,38 @@ extern FrtTVTerm *frt_tv_get_tv_term(FrtTermVector *tv, const char *term);
  ****************************************************************************/
 
 /* * * FrtLazyDocField * * */
-typedef struct FrtLazyDocFieldData
-{
-    off_t start;
-    int   length;
-    char *text;
+typedef struct FrtLazyDocFieldData {
+    off_t              start;
+    int                length;
+    rb_encoding        *encoding;
+    FrtCompressionType compression; /* as stored */
+    char               *text;
 } FrtLazyDocFieldData;
 
 typedef struct FrtLazyDoc FrtLazyDoc;
-typedef struct FrtLazyDocField
-{
-    FrtSymbol           name;
+typedef struct FrtLazyDocField {
+    ID                  name;
     FrtLazyDocFieldData *data;
     FrtLazyDoc          *doc;
     int                 size; /* number of data elements */
     int                 len;  /* length of data elements concatenated */
-    int                 is_compressed : 2; /* set to 2 after all data is loaded */
+    FrtCompressionType  compression; /* as configured */
+    bool                decompressed;
 } FrtLazyDocField;
 
 extern char *frt_lazy_df_get_data(FrtLazyDocField *self, int i);
 extern void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len);
 
 /* * * FrtLazyDoc * * */
-struct FrtLazyDoc
-{
-    FrtHash *field_dictionary;
-    int size;
+struct FrtLazyDoc {
+    FrtHash         *field_dictionary;
+    int             size;
     FrtLazyDocField **fields;
-    FrtInStream *fields_in;
+    FrtInStream     *fields_in;
 };
 
 extern void frt_lazy_doc_close(FrtLazyDoc *self);
-extern FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, FrtSymbol field);
+extern FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, ID field);
 
 /****************************************************************************
  *
@@ -602,8 +567,7 @@ extern FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, FrtSymbol field);
  *
  ****************************************************************************/
 
-typedef struct FrtFieldsReader
-{
+typedef struct FrtFieldsReader {
     int           size;
     FrtFieldInfos *fis;
     FrtStore      *store;
@@ -611,15 +575,13 @@ typedef struct FrtFieldsReader
     FrtInStream   *fdt_in;
 } FrtFieldsReader;
 
-extern FrtFieldsReader *frt_fr_open(FrtStore *store,
-                             const char *segment, FrtFieldInfos *fis);
+extern FrtFieldsReader *frt_fr_open(FrtStore *store, const char *segment, FrtFieldInfos *fis);
 extern FrtFieldsReader *frt_fr_clone(FrtFieldsReader *orig);
 extern void frt_fr_close(FrtFieldsReader *fr);
 extern FrtDocument *frt_fr_get_doc(FrtFieldsReader *fr, int doc_num);
 extern FrtLazyDoc *frt_fr_get_lazy_doc(FrtFieldsReader *fr, int doc_num);
 extern FrtHash *frt_fr_get_tv(FrtFieldsReader *fr, int doc_num);
-extern FrtTermVector *frt_fr_get_field_tv(FrtFieldsReader *fr, int doc_num,
-                                   int field_num);
+extern FrtTermVector *frt_fr_get_field_tv(FrtFieldsReader *fr, int doc_num, int field_num);
 
 /****************************************************************************
  *
@@ -627,18 +589,16 @@ extern FrtTermVector *frt_fr_get_field_tv(FrtFieldsReader *fr, int doc_num,
  *
  ****************************************************************************/
 
-typedef struct FrtFieldsWriter
-{
+typedef struct FrtFieldsWriter {
     FrtFieldInfos *fis;
     FrtOutStream  *fdt_out;
     FrtOutStream  *fdx_out;
     FrtOutStream  *buffer;
     FrtTVField    *tv_fields;
-    off_t       start_ptr;
+    off_t         start_ptr;
 } FrtFieldsWriter;
 
-extern FrtFieldsWriter *frt_fw_open(FrtStore *store,
-                             const char *segment, FrtFieldInfos *fis);
+extern FrtFieldsWriter *frt_fw_open(FrtStore *store, const char *segment, FrtFieldInfos *fis);
 extern void frt_fw_close(FrtFieldsWriter *fw);
 extern void frt_fw_add_doc(FrtFieldsWriter *fw, FrtDocument *doc);
 extern void frt_fw_add_postings(FrtFieldsWriter *fw,
@@ -659,11 +619,10 @@ extern void frt_fw_write_tv_index(FrtFieldsWriter *fw);
  *
  ****************************************************************************/
 
-struct FrtDeleter
-{
-    FrtStore         *store;
-    FrtSegmentInfos  *sis;
-    FrtHashSet       *pending;
+struct FrtDeleter {
+    FrtStore        *store;
+    FrtSegmentInfos *sis;
+    FrtHashSet      *pending;
 };
 
 extern FrtDeleter *frt_deleter_new(FrtSegmentInfos *sis, FrtStore *store);
@@ -681,88 +640,115 @@ extern void frt_deleter_delete_files(FrtDeleter *dlr, char **files, int file_cnt
 #define FRT_WRITE_LOCK_NAME "write"
 #define FRT_COMMIT_LOCK_NAME "commit"
 
-struct FrtIndexReader
-{
-    int                 (*num_docs)(FrtIndexReader *ir);
-    int                 (*max_doc)(FrtIndexReader *ir);
-    FrtDocument           *(*get_doc)(FrtIndexReader *ir, int doc_num);
-    FrtLazyDoc            *(*get_lazy_doc)(FrtIndexReader *ir, int doc_num);
-    frt_uchar              *(*get_norms)(FrtIndexReader *ir, int field_num);
-    frt_uchar              *(*get_norms_into)(FrtIndexReader *ir, int field_num,
-                                          frt_uchar *buf);
-    FrtTermEnum           *(*terms)(FrtIndexReader *ir, int field_num);
-    FrtTermEnum           *(*terms_from)(FrtIndexReader *ir, int field_num,
-                                      const char *term);
-    int                 (*doc_freq)(FrtIndexReader *ir, int field_num,
-                                    const char *term);
-    FrtTermDocEnum        *(*term_docs)(FrtIndexReader *ir);
-    FrtTermDocEnum        *(*term_positions)(FrtIndexReader *ir);
-    FrtTermVector         *(*term_vector)(FrtIndexReader *ir, int doc_num,
-                                          FrtSymbol field);
-    FrtHash    *(*term_vectors)(FrtIndexReader *ir, int doc_num);
-    bool                (*is_deleted)(FrtIndexReader *ir, int doc_num);
-    bool                (*has_deletions)(FrtIndexReader *ir);
-    void                (*acquire_write_lock)(FrtIndexReader *ir);
-    void                (*set_norm_i)(FrtIndexReader *ir, int doc_num,
-                                      int field_num, frt_uchar val);
-    void                (*delete_doc_i)(FrtIndexReader *ir, int doc_num);
-    void                (*undelete_all_i)(FrtIndexReader *ir);
-    void                (*set_deleter_i)(FrtIndexReader *ir, FrtDeleter *dlr);
-    bool                (*is_latest_i)(FrtIndexReader *ir);
-    void                (*commit_i)(FrtIndexReader *ir);
-    void                (*close_i)(FrtIndexReader *ir);
-    int                 ref_cnt;
-    FrtDeleter            *deleter;
-    FrtStore              *store;
-    FrtLock               *write_lock;
-    FrtSegmentInfos       *sis;
-    FrtFieldInfos         *fis;
-    FrtHash    *cache;
-    FrtHash    *field_index_cache;
-    frt_mutex_t             field_index_mutex;
-    frt_uchar              *fake_norms;
-    frt_mutex_t             mutex;
-    bool                has_changes : 1;
-    bool                is_stale    : 1;
-    bool                is_owner    : 1;
+typedef enum {
+    FRT_INDEX_READER,
+    FRT_SEGMENT_READER,
+    FRT_MULTI_READER
+} frt_index_reader_t;
+
+struct FrtIndexReader {
+    int             type;
+    int             (*num_docs)(FrtIndexReader *ir);
+    int             (*max_doc)(FrtIndexReader *ir);
+    FrtDocument     *(*get_doc)(FrtIndexReader *ir, int doc_num);
+    FrtLazyDoc      *(*get_lazy_doc)(FrtIndexReader *ir, int doc_num);
+    frt_uchar       *(*get_norms)(FrtIndexReader *ir, int field_num);
+    frt_uchar       *(*get_norms_into)(FrtIndexReader *ir, int field_num, frt_uchar *buf);
+    FrtTermEnum     *(*terms)(FrtIndexReader *ir, int field_num);
+    FrtTermEnum     *(*terms_from)(FrtIndexReader *ir, int field_num, const char *term);
+    int             (*doc_freq)(FrtIndexReader *ir, int field_num, const char *term);
+    FrtTermDocEnum  *(*term_docs)(FrtIndexReader *ir);
+    FrtTermDocEnum  *(*term_positions)(FrtIndexReader *ir);
+    FrtTermVector   *(*term_vector)(FrtIndexReader *ir, int doc_num, ID field);
+    FrtHash         *(*term_vectors)(FrtIndexReader *ir, int doc_num);
+    bool            (*is_deleted)(FrtIndexReader *ir, int doc_num);
+    bool            (*has_deletions)(FrtIndexReader *ir);
+    void            (*acquire_write_lock)(FrtIndexReader *ir);
+    void            (*set_norm_i)(FrtIndexReader *ir, int doc_num, int field_num, frt_uchar val);
+    void            (*delete_doc_i)(FrtIndexReader *ir, int doc_num);
+    void            (*undelete_all_i)(FrtIndexReader *ir);
+    void            (*set_deleter_i)(FrtIndexReader *ir, FrtDeleter *dlr);
+    bool            (*is_latest_i)(FrtIndexReader *ir);
+    void            (*commit_i)(FrtIndexReader *ir);
+    void            (*close_i)(FrtIndexReader *ir);
+    int             ref_cnt;
+    FrtDeleter      *deleter;
+    FrtStore        *store;
+    FrtLock         *write_lock;
+    FrtSegmentInfos *sis;
+    FrtFieldInfos   *fis;
+    FrtHash         *cache;
+    FrtHash         *field_index_cache;
+    frt_mutex_t     field_index_mutex;
+    frt_uchar       *fake_norms;
+    frt_mutex_t     mutex;
+    bool            has_changes : 1;
+    bool            is_stale    : 1;
+    bool            is_owner    : 1;
+    VALUE           rir;
 };
 
-extern FrtIndexReader *frt_ir_open(FrtStore *store);
+extern FrtIndexReader *frt_ir_open(FrtIndexReader *ir, FrtStore *store);
 extern void frt_ir_close(FrtIndexReader *ir);
 extern void frt_ir_commit(FrtIndexReader *ir);
 extern void frt_ir_delete_doc(FrtIndexReader *ir, int doc_num);
 extern void frt_ir_undelete_all(FrtIndexReader *ir);
-extern int frt_ir_doc_freq(FrtIndexReader *ir, FrtSymbol field, const char *term);
-extern void frt_ir_set_norm(FrtIndexReader *ir, int doc_num, FrtSymbol field, frt_uchar val);
+extern int frt_ir_doc_freq(FrtIndexReader *ir, ID field, const char *term);
+extern void frt_ir_set_norm(FrtIndexReader *ir, int doc_num, ID field, frt_uchar val);
 extern frt_uchar *frt_ir_get_norms_i(FrtIndexReader *ir, int field_num);
-extern frt_uchar *frt_ir_get_norms(FrtIndexReader *ir, FrtSymbol field);
-extern frt_uchar *frt_ir_get_norms_into(FrtIndexReader *ir, FrtSymbol field, frt_uchar *buf);
-extern FrtDocument *frt_ir_get_doc_with_term(FrtIndexReader *ir, FrtSymbol field, const char *term);
-extern FrtTermEnum *frt_ir_terms(FrtIndexReader *ir, FrtSymbol field);
-extern FrtTermEnum *frt_ir_terms_from(FrtIndexReader *ir, FrtSymbol field, const char *t);
-extern FrtTermDocEnum *ir_term_docs_for(FrtIndexReader *ir, FrtSymbol field, const char *term);
-extern FrtTermDocEnum *frt_ir_term_positions_for(FrtIndexReader *ir, FrtSymbol field, const char *t);
+extern frt_uchar *frt_ir_get_norms(FrtIndexReader *ir, ID field);
+extern frt_uchar *frt_ir_get_norms_into(FrtIndexReader *ir, ID field, frt_uchar *buf);
+extern FrtDocument *frt_ir_get_doc_with_term(FrtIndexReader *ir, ID field, const char *term);
+extern FrtTermEnum *frt_ir_terms(FrtIndexReader *ir, ID field);
+extern FrtTermEnum *frt_ir_terms_from(FrtIndexReader *ir, ID field, const char *t);
+extern FrtTermDocEnum *ir_term_docs_for(FrtIndexReader *ir, ID field, const char *term);
+extern FrtTermDocEnum *frt_ir_term_positions_for(FrtIndexReader *ir, ID field, const char *t);
 extern void frt_ir_add_cache(FrtIndexReader *ir);
 extern bool frt_ir_is_latest(FrtIndexReader *ir);
 
+/****************************************************************************
+ * FrtSegmentReader
+ ****************************************************************************/
+
+struct FrtSegmentReader {
+    FrtIndexReader       ir;
+    FrtSegmentInfo       *si;
+    char                 *segment;
+    FrtFieldsReader      *fr;
+    FrtBitVector         *deleted_docs;
+    FrtInStream          *frq_in;
+    FrtInStream          *prx_in;
+    FrtSegmentFieldIndex *sfi;
+    FrtTermInfosReader   *tir;
+    frt_thread_key_t     thread_fr;
+    void                 **fr_bucket;
+    FrtHash              *norms;
+    FrtStore             *cfs_store;
+    bool                 deleted_docs_dirty : 1;
+    bool                 undelete_all : 1;
+    bool                 norms_dirty : 1;
+};
+
+extern FrtSegmentReader *frt_sr_alloc();
+
 /****************************************************************************
  * FrtMultiReader
  ****************************************************************************/
 
 struct FrtMultiReader {
     FrtIndexReader ir;
-    int max_doc;
-    int num_docs_cache;
-    int r_cnt;
-    int *starts;
+    int            max_doc;
+    int            num_docs_cache;
+    int            r_cnt;
+    int            *starts;
     FrtIndexReader **sub_readers;
-    FrtHash *norms_cache;
-    bool has_deletions : 1;
-    int **field_num_map;
+    FrtHash        *norms_cache;
+    bool           has_deletions : 1;
+    int            **field_num_map;
 };
 
 extern int frt_mr_get_field_num(FrtMultiReader *mr, int ir_num, int f_num);
-extern FrtIndexReader *frt_mr_open(FrtIndexReader **sub_readers, const int r_cnt);
+extern FrtIndexReader *frt_mr_open(FrtIndexReader *ir, FrtIndexReader **sub_readers, const int r_cnt);
 
 /****************************************************************************
  *
@@ -770,16 +756,15 @@ extern FrtIndexReader *frt_mr_open(FrtIndexReader **sub_readers, const int r_cnt
  *
  ****************************************************************************/
 
-typedef struct FrtFieldInverter
-{
-    FrtHash *plists;
-    frt_uchar *norms;
+typedef struct FrtFieldInverter {
+    FrtHash      *plists;
+    frt_uchar    *norms;
     FrtFieldInfo *fi;
-    int length;
-    bool is_tokenized : 1;
-    bool store_term_vector : 1;
-    bool store_offsets : 1;
-    bool has_norms : 1;
+    int          length;
+    bool         is_tokenized : 1;
+    bool         store_term_vector : 1;
+    bool         store_offsets : 1;
+    bool         has_norms : 1;
 } FrtFieldInverter;
 
 /****************************************************************************
@@ -791,18 +776,17 @@ typedef struct FrtFieldInverter
 #define DW_OFFSET_INIT_CAPA 512
 typedef struct FrtIndexWriter FrtIndexWriter;
 
-typedef struct FrtDocWriter
-{
-    FrtStore *store;
-    FrtSegmentInfo *si;
-    FrtFieldInfos *fis;
+typedef struct FrtDocWriter {
+    FrtStore        *store;
+    FrtSegmentInfo  *si;
+    FrtFieldInfos   *fis;
     FrtFieldsWriter *fw;
-    FrtMemoryPool *mp;
-    FrtAnalyzer *analyzer;
-    FrtHash *curr_plists;
-    FrtHash *fields;
-    FrtSimilarity *similarity;
-    FrtOffset *offsets;
+    FrtMemoryPool   *mp;
+    FrtAnalyzer     *analyzer;
+    FrtHash         *curr_plists;
+    FrtHash         *fields;
+    FrtSimilarity   *similarity;
+    FrtOffset       *offsets;
     int offsets_size;
     int offsets_capa;
     int doc_num;
@@ -817,9 +801,7 @@ extern void frt_dw_close(FrtDocWriter *dw);
 extern void frt_dw_add_doc(FrtDocWriter *dw, FrtDocument *doc);
 extern void frt_dw_new_segment(FrtDocWriter *dw, FrtSegmentInfo *si);
 /* For testing. need to remove somehow. FIXME */
-extern FrtHash *frt_dw_invert_field(FrtDocWriter *dw,
-                                  FrtFieldInverter *fld_inv,
-                                  FrtDocField *df);
+extern FrtHash *frt_dw_invert_field(FrtDocWriter *dw, FrtFieldInverter *fld_inv, FrtDocField *df);
 extern FrtFieldInverter *frt_dw_get_fld_inv(FrtDocWriter *dw, FrtFieldInfo *fi);
 extern void frt_dw_reset_postings(FrtHash *postings);
 
@@ -829,25 +811,25 @@ extern void frt_dw_reset_postings(FrtHash *postings);
  *
  ****************************************************************************/
 
-struct FrtIndexWriter
-{
-    FrtConfig config;
-    frt_mutex_t mutex;
-    FrtStore *store;
-    FrtAnalyzer *analyzer;
+struct FrtIndexWriter {
+    FrtConfig       config;
+    frt_mutex_t     mutex;
+    FrtStore        *store;
+    FrtAnalyzer     *analyzer;
     FrtSegmentInfos *sis;
-    FrtFieldInfos *fis;
-    FrtDocWriter *dw;
-    FrtSimilarity *similarity;
-    FrtLock *write_lock;
-    FrtDeleter *deleter;
+    FrtFieldInfos   *fis;
+    FrtDocWriter    *dw;
+    FrtSimilarity   *similarity;
+    FrtLock         *write_lock;
+    FrtDeleter      *deleter;
 };
 
 extern void frt_index_create(FrtStore *store, FrtFieldInfos *fis);
 extern bool frt_index_is_locked(FrtStore *store);
-extern FrtIndexWriter *frt_iw_open(FrtStore *store, FrtAnalyzer *analyzer, const FrtConfig *config);
-extern void frt_iw_delete_term(FrtIndexWriter *iw, FrtSymbol field, const char *term);
-extern void frt_iw_delete_terms(FrtIndexWriter *iw, FrtSymbol field, char **terms, const int term_cnt);
+extern FrtIndexWriter *frt_iw_alloc();
+extern FrtIndexWriter *frt_iw_open(FrtIndexWriter *, FrtStore *store, FrtAnalyzer *analyzer, const FrtConfig *config);
+extern void frt_iw_delete_term(FrtIndexWriter *iw, ID field, const char *term);
+extern void frt_iw_delete_terms(FrtIndexWriter *iw, ID field, char **terms, const int term_cnt);
 extern void frt_iw_close(FrtIndexWriter *iw);
 extern void frt_iw_add_doc(FrtIndexWriter *iw, FrtDocument *doc);
 extern int frt_iw_doc_count(FrtIndexWriter *iw);
@@ -862,17 +844,16 @@ extern void frt_iw_add_readers(FrtIndexWriter *iw, FrtIndexReader **readers, con
  ****************************************************************************/
 
 #define FRT_CW_INIT_CAPA 16
-typedef struct FrtCWFileEntry
-{
-    char *name;
+typedef struct FrtCWFileEntry {
+    char  *name;
     off_t dir_offset;
     off_t data_offset;
 } FrtCWFileEntry;
 
 typedef struct FrtCompoundWriter {
-    FrtStore *store;
-    const char *name;
-    FrtHashSet *ids;
+    FrtStore       *store;
+    const char     *name;
+    FrtHashSet     *ids;
     FrtCWFileEntry *file_entries;
 } FrtCompoundWriter;