ferret 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +6 -5
 - data/Rakefile +34 -13
 - data/TODO +1 -0
 - data/TUTORIAL +1 -1
 - data/ext/analysis.c +87 -70
 - data/ext/analysis.h +18 -6
 - data/ext/array.c +1 -2
 - data/ext/array.h +1 -1
 - data/ext/bitvector.c +10 -6
 - data/ext/bitvector.h +2 -2
 - data/ext/compound_io.c +30 -27
 - data/ext/document.c +15 -15
 - data/ext/document.h +5 -5
 - data/ext/except.c +2 -0
 - data/ext/except.h +25 -23
 - data/ext/extconf.rb +1 -0
 - data/ext/ferret.c +10 -8
 - data/ext/ferret.h +9 -8
 - data/ext/field.c +29 -25
 - data/ext/filter.c +52 -14
 - data/ext/frtio.h +13 -0
 - data/ext/fs_store.c +115 -170
 - data/ext/global.c +9 -8
 - data/ext/global.h +17 -13
 - data/ext/hash.c +13 -19
 - data/ext/hash.h +11 -11
 - data/ext/hashset.c +5 -7
 - data/ext/hashset.h +9 -8
 - data/ext/helper.c +1 -1
 - data/ext/helper.h +2 -1
 - data/ext/inc/except.h +25 -23
 - data/ext/inc/lang.h +11 -1
 - data/ext/ind.c +33 -21
 - data/ext/index.h +44 -39
 - data/ext/index_io.c +61 -57
 - data/ext/index_rw.c +418 -361
 - data/ext/lang.c +10 -0
 - data/ext/lang.h +11 -1
 - data/ext/nix_io.c +135 -0
 - data/ext/priorityqueue.c +16 -16
 - data/ext/priorityqueue.h +9 -6
 - data/ext/q_boolean.c +128 -76
 - data/ext/q_const_score.c +20 -20
 - data/ext/q_filtered_query.c +20 -20
 - data/ext/q_fuzzy.c +37 -23
 - data/ext/q_match_all.c +15 -19
 - data/ext/q_multi_phrase.c +87 -46
 - data/ext/q_parser.c +247 -119
 - data/ext/q_phrase.c +86 -52
 - data/ext/q_prefix.c +25 -14
 - data/ext/q_range.c +59 -14
 - data/ext/q_span.c +263 -172
 - data/ext/q_term.c +62 -51
 - data/ext/q_wildcard.c +24 -13
 - data/ext/r_analysis.c +328 -80
 - data/ext/r_doc.c +11 -6
 - data/ext/r_index_io.c +40 -32
 - data/ext/r_qparser.c +15 -14
 - data/ext/r_search.c +270 -152
 - data/ext/r_store.c +32 -17
 - data/ext/ram_store.c +38 -22
 - data/ext/search.c +617 -87
 - data/ext/search.h +227 -163
 - data/ext/similarity.c +54 -45
 - data/ext/similarity.h +3 -3
 - data/ext/sort.c +132 -53
 - data/ext/store.c +21 -2
 - data/ext/store.h +14 -14
 - data/ext/tags +4322 -232
 - data/ext/term.c +140 -109
 - data/ext/termdocs.c +74 -60
 - data/ext/vector.c +181 -152
 - data/ext/w32_io.c +150 -0
 - data/lib/ferret.rb +1 -1
 - data/lib/ferret/analysis/standard_tokenizer.rb +4 -3
 - data/lib/ferret/document/field.rb +1 -1
 - data/lib/ferret/index/field_infos.rb +1 -1
 - data/lib/ferret/index/term.rb +1 -1
 - data/lib/ferret/query_parser/query_parser.tab.rb +8 -24
 - data/lib/ferret/search.rb +1 -0
 - data/lib/ferret/search/boolean_query.rb +0 -4
 - data/lib/ferret/search/index_searcher.rb +21 -8
 - data/lib/ferret/search/multi_phrase_query.rb +7 -0
 - data/lib/ferret/search/multi_searcher.rb +261 -0
 - data/lib/ferret/search/phrase_query.rb +1 -1
 - data/lib/ferret/search/query.rb +34 -5
 - data/lib/ferret/search/sort.rb +7 -3
 - data/lib/ferret/search/sort_field.rb +8 -4
 - data/lib/ferret/store/fs_store.rb +13 -6
 - data/lib/ferret/store/index_io.rb +0 -14
 - data/lib/ferret/store/ram_store.rb +3 -2
 - data/lib/rferret.rb +1 -1
 - data/test/unit/analysis/ctc_analyzer.rb +131 -0
 - data/test/unit/analysis/ctc_tokenstream.rb +98 -9
 - data/test/unit/index/tc_index.rb +40 -1
 - data/test/unit/index/tc_term.rb +7 -0
 - data/test/unit/index/th_doc.rb +8 -0
 - data/test/unit/query_parser/tc_query_parser.rb +6 -4
 - data/test/unit/search/rtc_sort_field.rb +6 -6
 - data/test/unit/search/tc_index_searcher.rb +8 -0
 - data/test/unit/search/tc_multi_searcher.rb +275 -0
 - data/test/unit/search/tc_multi_searcher2.rb +126 -0
 - data/test/unit/search/tc_search_and_sort.rb +66 -0
 - metadata +31 -26
 - data/test/unit/query_parser/rtc_query_parser.rb +0 -138
 
    
        data/ext/index_rw.c
    CHANGED
    
    | 
         @@ -1,4 +1,4 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            #include  
     | 
| 
      
 1 
     | 
    
         
            +
            #include "index.h"
         
     | 
| 
       2 
2 
     | 
    
         
             
            #include <stdlib.h>
         
     | 
| 
       3 
3 
     | 
    
         
             
            #include <string.h>
         
     | 
| 
       4 
4 
     | 
    
         
             
            #include <array.h>
         
     | 
| 
         @@ -24,11 +24,11 @@ const char *VECTOR_EXTENSIONS[] = { 
     | 
|
| 
       24 
24 
     | 
    
         
             
            };
         
     | 
| 
       25 
25 
     | 
    
         | 
| 
       26 
26 
     | 
    
         
             
            FerretConfig config = {
         
     | 
| 
       27 
     | 
    
         
            -
              10,        
     | 
| 
       28 
     | 
    
         
            -
              10,        
     | 
| 
       29 
     | 
    
         
            -
              INT_MAX,   
     | 
| 
       30 
     | 
    
         
            -
              10000,     
     | 
| 
       31 
     | 
    
         
            -
              128        
     | 
| 
      
 27 
     | 
    
         
            +
              10,       /* default merge_factor */
         
     | 
| 
      
 28 
     | 
    
         
            +
              10,       /* default min_merge_docs */
         
     | 
| 
      
 29 
     | 
    
         
            +
              INT_MAX,  /* default max_merge_docs */
         
     | 
| 
      
 30 
     | 
    
         
            +
              10000,    /* default max_field_length */
         
     | 
| 
      
 31 
     | 
    
         
            +
              128       /* default term_index_interval */
         
     | 
| 
       32 
32 
     | 
    
         
             
            };
         
     | 
| 
       33 
33 
     | 
    
         | 
| 
       34 
34 
     | 
    
         
             
            /***************************************************************************
         
     | 
| 
         @@ -47,33 +47,32 @@ int co_eq(const void *key1, const void *key2) 
     | 
|
| 
       47 
47 
     | 
    
         
             
              return (key1 == key2);
         
     | 
| 
       48 
48 
     | 
    
         
             
            }
         
     | 
| 
       49 
49 
     | 
    
         | 
| 
       50 
     | 
    
         
            -
            void co_destroy( 
     | 
| 
      
 50 
     | 
    
         
            +
            void co_destroy(CacheObject *self)
         
     | 
| 
       51 
51 
     | 
    
         
             
            {
         
     | 
| 
       52 
     | 
    
         
            -
               
     | 
| 
       53 
     | 
    
         
            -
              h_rem( 
     | 
| 
       54 
     | 
    
         
            -
               
     | 
| 
       55 
     | 
    
         
            -
               
     | 
| 
       56 
     | 
    
         
            -
              free(co);
         
     | 
| 
      
 52 
     | 
    
         
            +
              h_rem(self->ref_tab1, self->ref2, false);
         
     | 
| 
      
 53 
     | 
    
         
            +
              h_rem(self->ref_tab2, self->ref1, false);
         
     | 
| 
      
 54 
     | 
    
         
            +
              self->destroy(self->obj);
         
     | 
| 
      
 55 
     | 
    
         
            +
              free(self);
         
     | 
| 
       57 
56 
     | 
    
         
             
            }
         
     | 
| 
       58 
57 
     | 
    
         | 
| 
       59 
58 
     | 
    
         
             
            CacheObject *co_create(HshTable *ref_tab1, HshTable *ref_tab2,
         
     | 
| 
       60 
     | 
    
         
            -
                void *ref1, void *ref2,  
     | 
| 
      
 59 
     | 
    
         
            +
                void *ref1, void *ref2, free_ft destroy, void *obj)
         
     | 
| 
       61 
60 
     | 
    
         
             
            {
         
     | 
| 
       62 
     | 
    
         
            -
              CacheObject * 
     | 
| 
       63 
     | 
    
         
            -
              h_set(ref_tab1, ref2,  
     | 
| 
       64 
     | 
    
         
            -
              h_set(ref_tab2, ref1,  
     | 
| 
       65 
     | 
    
         
            -
               
     | 
| 
       66 
     | 
    
         
            -
               
     | 
| 
       67 
     | 
    
         
            -
               
     | 
| 
       68 
     | 
    
         
            -
               
     | 
| 
       69 
     | 
    
         
            -
               
     | 
| 
       70 
     | 
    
         
            -
               
     | 
| 
       71 
     | 
    
         
            -
              return  
     | 
| 
      
 61 
     | 
    
         
            +
              CacheObject *self = ALLOC(CacheObject);
         
     | 
| 
      
 62 
     | 
    
         
            +
              h_set(ref_tab1, ref2, self);
         
     | 
| 
      
 63 
     | 
    
         
            +
              h_set(ref_tab2, ref1, self);
         
     | 
| 
      
 64 
     | 
    
         
            +
              self->ref_tab1 = ref_tab1;
         
     | 
| 
      
 65 
     | 
    
         
            +
              self->ref_tab2 = ref_tab2;
         
     | 
| 
      
 66 
     | 
    
         
            +
              self->ref1 = ref1;
         
     | 
| 
      
 67 
     | 
    
         
            +
              self->ref2 = ref2;
         
     | 
| 
      
 68 
     | 
    
         
            +
              self->destroy = destroy;
         
     | 
| 
      
 69 
     | 
    
         
            +
              self->obj = obj;
         
     | 
| 
      
 70 
     | 
    
         
            +
              return self;
         
     | 
| 
       72 
71 
     | 
    
         
             
            }
         
     | 
| 
       73 
72 
     | 
    
         | 
| 
       74 
73 
     | 
    
         
             
            HshTable *co_hsh_create()
         
     | 
| 
       75 
74 
     | 
    
         
             
            {
         
     | 
| 
       76 
     | 
    
         
            -
              return h_new(&co_hash, &co_eq, NULL, &co_destroy);
         
     | 
| 
      
 75 
     | 
    
         
            +
              return h_new(&co_hash, &co_eq, (free_ft)NULL, (free_ft)&co_destroy);
         
     | 
| 
       77 
76 
     | 
    
         
             
            }
         
     | 
| 
       78 
77 
     | 
    
         | 
| 
       79 
78 
     | 
    
         
             
            /***************************************************************************
         
     | 
| 
         @@ -84,39 +83,38 @@ HshTable *co_hsh_create() 
     | 
|
| 
       84 
83 
     | 
    
         | 
| 
       85 
84 
     | 
    
         
             
            Posting *p_create(Term *term, int position, TVOffsetInfo *offset)
         
     | 
| 
       86 
85 
     | 
    
         
             
            {
         
     | 
| 
       87 
     | 
    
         
            -
              Posting * 
     | 
| 
       88 
     | 
    
         
            -
               
     | 
| 
       89 
     | 
    
         
            -
               
     | 
| 
       90 
     | 
    
         
            -
               
     | 
| 
       91 
     | 
    
         
            -
               
     | 
| 
       92 
     | 
    
         
            -
               
     | 
| 
       93 
     | 
    
         
            -
               
     | 
| 
       94 
     | 
    
         
            -
               
     | 
| 
       95 
     | 
    
         
            -
              return  
     | 
| 
      
 86 
     | 
    
         
            +
              Posting *self = ALLOC(Posting);
         
     | 
| 
      
 87 
     | 
    
         
            +
              self->freq = 1;
         
     | 
| 
      
 88 
     | 
    
         
            +
              self->size = 1;
         
     | 
| 
      
 89 
     | 
    
         
            +
              self->term = term;
         
     | 
| 
      
 90 
     | 
    
         
            +
              self->positions = ALLOC(int);
         
     | 
| 
      
 91 
     | 
    
         
            +
              self->positions[0] = position;
         
     | 
| 
      
 92 
     | 
    
         
            +
              self->offsets = ALLOC(TVOffsetInfo *);
         
     | 
| 
      
 93 
     | 
    
         
            +
              self->offsets[0] = offset;
         
     | 
| 
      
 94 
     | 
    
         
            +
              return self;
         
     | 
| 
       96 
95 
     | 
    
         
             
            }
         
     | 
| 
       97 
96 
     | 
    
         | 
| 
       98 
     | 
    
         
            -
            void p_destroy( 
     | 
| 
      
 97 
     | 
    
         
            +
            void p_destroy(Posting *self)
         
     | 
| 
       99 
98 
     | 
    
         
             
            {
         
     | 
| 
       100 
     | 
    
         
            -
               
     | 
| 
      
 99 
     | 
    
         
            +
              /* the positions and offsets will be put in a TVTerm so no need to free */
         
     | 
| 
       101 
100 
     | 
    
         
             
              int i;
         
     | 
| 
       102 
     | 
    
         
            -
               
     | 
| 
       103 
     | 
    
         
            -
               
     | 
| 
       104 
     | 
    
         
            -
             
     | 
| 
       105 
     | 
    
         
            -
             
     | 
| 
       106 
     | 
    
         
            -
              free( 
     | 
| 
       107 
     | 
    
         
            -
              free(p);
         
     | 
| 
      
 101 
     | 
    
         
            +
              free(self->positions);
         
     | 
| 
      
 102 
     | 
    
         
            +
              for (i = 0; i < self->freq; i++)
         
     | 
| 
      
 103 
     | 
    
         
            +
                tvoi_destroy(self->offsets[i]);
         
     | 
| 
      
 104 
     | 
    
         
            +
              free(self->offsets);
         
     | 
| 
      
 105 
     | 
    
         
            +
              free(self);
         
     | 
| 
       108 
106 
     | 
    
         
             
            }
         
     | 
| 
       109 
107 
     | 
    
         | 
| 
       110 
     | 
    
         
            -
            void p_add_occurance(Posting * 
     | 
| 
      
 108 
     | 
    
         
            +
            void p_add_occurance(Posting *self, int position, TVOffsetInfo *offset)
         
     | 
| 
       111 
109 
     | 
    
         
             
            {
         
     | 
| 
       112 
     | 
    
         
            -
              if ( 
     | 
| 
       113 
     | 
    
         
            -
                 
     | 
| 
       114 
     | 
    
         
            -
                REALLOC_N( 
     | 
| 
       115 
     | 
    
         
            -
                REALLOC_N( 
     | 
| 
      
 110 
     | 
    
         
            +
              if (self->freq >= self->size) {
         
     | 
| 
      
 111 
     | 
    
         
            +
                self->size *= 2;
         
     | 
| 
      
 112 
     | 
    
         
            +
                REALLOC_N(self->positions, int, self->size);
         
     | 
| 
      
 113 
     | 
    
         
            +
                REALLOC_N(self->offsets, TVOffsetInfo *, self->size);
         
     | 
| 
       116 
114 
     | 
    
         
             
              }
         
     | 
| 
       117 
     | 
    
         
            -
               
     | 
| 
       118 
     | 
    
         
            -
               
     | 
| 
       119 
     | 
    
         
            -
               
     | 
| 
      
 115 
     | 
    
         
            +
              self->positions[self->freq] = position;
         
     | 
| 
      
 116 
     | 
    
         
            +
              self->offsets[self->freq] = offset;
         
     | 
| 
      
 117 
     | 
    
         
            +
              self->freq++;
         
     | 
| 
       120 
118 
     | 
    
         
             
            }
         
     | 
| 
       121 
119 
     | 
    
         | 
| 
       122 
120 
     | 
    
         
             
            inline int p_cmp(const void *const p1, const void *const p2)
         
     | 
| 
         @@ -137,47 +135,49 @@ DocumentWriter *dw_open(Store *store, 
     | 
|
| 
       137 
135 
     | 
    
         
             
                                    int max_field_length,
         
     | 
| 
       138 
136 
     | 
    
         
             
                                    int term_index_interval)
         
     | 
| 
       139 
137 
     | 
    
         
             
            {
         
     | 
| 
       140 
     | 
    
         
            -
              DocumentWriter * 
     | 
| 
       141 
     | 
    
         
            -
               
     | 
| 
       142 
     | 
    
         
            -
               
     | 
| 
       143 
     | 
    
         
            -
               
     | 
| 
       144 
     | 
    
         
            -
               
     | 
| 
       145 
     | 
    
         
            -
               
     | 
| 
       146 
     | 
    
         
            -
             
     | 
| 
       147 
     | 
    
         
            -
             
     | 
| 
       148 
     | 
    
         
            -
               
     | 
| 
      
 138 
     | 
    
         
            +
              DocumentWriter *self = ALLOC(DocumentWriter);
         
     | 
| 
      
 139 
     | 
    
         
            +
              self->store = store;
         
     | 
| 
      
 140 
     | 
    
         
            +
              self->analyzer = analyzer;
         
     | 
| 
      
 141 
     | 
    
         
            +
              self->similarity = similarity;
         
     | 
| 
      
 142 
     | 
    
         
            +
              self->fis = NULL;
         
     | 
| 
      
 143 
     | 
    
         
            +
              self->postingtable = h_new(&term_hash, &term_eq,
         
     | 
| 
      
 144 
     | 
    
         
            +
                                         (free_ft)&term_destroy,
         
     | 
| 
      
 145 
     | 
    
         
            +
                                         (free_ft)&p_destroy);
         
     | 
| 
      
 146 
     | 
    
         
            +
              self->max_field_length = max_field_length;
         
     | 
| 
      
 147 
     | 
    
         
            +
              self->term_index_interval = term_index_interval;
         
     | 
| 
      
 148 
     | 
    
         
            +
              return self;
         
     | 
| 
       149 
149 
     | 
    
         
             
            }
         
     | 
| 
       150 
150 
     | 
    
         | 
| 
       151 
     | 
    
         
            -
            void dw_close(DocumentWriter * 
     | 
| 
      
 151 
     | 
    
         
            +
            void dw_close(DocumentWriter *self)
         
     | 
| 
       152 
152 
     | 
    
         
             
            {
         
     | 
| 
       153 
     | 
    
         
            -
              if ( 
     | 
| 
       154 
     | 
    
         
            -
              h_destroy( 
     | 
| 
       155 
     | 
    
         
            -
              free( 
     | 
| 
      
 153 
     | 
    
         
            +
              if (self->fis) fis_destroy(self->fis);
         
     | 
| 
      
 154 
     | 
    
         
            +
              h_destroy(self->postingtable);
         
     | 
| 
      
 155 
     | 
    
         
            +
              free(self);
         
     | 
| 
       156 
156 
     | 
    
         
             
            }
         
     | 
| 
       157 
157 
     | 
    
         | 
| 
       158 
     | 
    
         
            -
            void dw_add_position(DocumentWriter * 
     | 
| 
      
 158 
     | 
    
         
            +
            void dw_add_position(DocumentWriter *self, char *field, char *text,
         
     | 
| 
       159 
159 
     | 
    
         
             
                int position, TVOffsetInfo *offset) 
         
     | 
| 
       160 
160 
     | 
    
         
             
            {
         
     | 
| 
       161 
161 
     | 
    
         
             
              Term termbuf = {field, text}, *term;
         
     | 
| 
       162 
     | 
    
         
            -
              Posting *p = (Posting *)h_get( 
     | 
| 
      
 162 
     | 
    
         
            +
              Posting *p = (Posting *)h_get(self->postingtable, &termbuf);
         
     | 
| 
       163 
163 
     | 
    
         | 
| 
       164 
     | 
    
         
            -
              if (p) {  
     | 
| 
       165 
     | 
    
         
            -
                // double the size of posting to make room for more posts.
         
     | 
| 
      
 164 
     | 
    
         
            +
              if (p) { /* word seen before */
         
     | 
| 
       166 
165 
     | 
    
         
             
                if (p->freq >= p->size) {
         
     | 
| 
      
 166 
     | 
    
         
            +
                  /* double size of posting to make room for more posts. */
         
     | 
| 
       167 
167 
     | 
    
         
             
                  p->size <<= 1;
         
     | 
| 
       168 
168 
     | 
    
         
             
                  REALLOC_N(p->positions, int, p->size);
         
     | 
| 
       169 
169 
     | 
    
         
             
                  p->offsets = REALLOC_N(p->offsets, TVOffsetInfo *, p->size);
         
     | 
| 
       170 
170 
     | 
    
         
             
                }
         
     | 
| 
       171 
     | 
    
         
            -
                p->positions[p->freq] = position;     
     | 
| 
       172 
     | 
    
         
            -
                p->offsets[p->freq] = offset;         
     | 
| 
       173 
     | 
    
         
            -
                p->freq++; 
     | 
| 
       174 
     | 
    
         
            -
              } else {  
     | 
| 
      
 171 
     | 
    
         
            +
                p->positions[p->freq] = position;    /* add new position */
         
     | 
| 
      
 172 
     | 
    
         
            +
                p->offsets[p->freq] = offset;        /* add new offset */
         
     | 
| 
      
 173 
     | 
    
         
            +
                p->freq++;                           /* update frequency */
         
     | 
| 
      
 174 
     | 
    
         
            +
              } else { /* word not seen before */
         
     | 
| 
       175 
175 
     | 
    
         
             
                term = term_create(field, text);
         
     | 
| 
       176 
     | 
    
         
            -
                h_set( 
     | 
| 
      
 176 
     | 
    
         
            +
                h_set(self->postingtable, term, p_create(term, position, offset));
         
     | 
| 
       177 
177 
     | 
    
         
             
              }
         
     | 
| 
       178 
178 
     | 
    
         
             
            }
         
     | 
| 
       179 
179 
     | 
    
         | 
| 
       180 
     | 
    
         
            -
            void dw_invert_doc(DocumentWriter * 
     | 
| 
      
 180 
     | 
    
         
            +
            void dw_invert_doc(DocumentWriter *self, Document *doc)
         
     | 
| 
       181 
181 
     | 
    
         
             
            {
         
     | 
| 
       182 
182 
     | 
    
         
             
              int i;
         
     | 
| 
       183 
183 
     | 
    
         
             
              int dfcnt = doc->dfcnt;
         
     | 
| 
         @@ -191,69 +191,74 @@ void dw_invert_doc(DocumentWriter *dw, Document *doc) 
     | 
|
| 
       191 
191 
     | 
    
         
             
              for (i = 0; i < dfcnt; i++) {
         
     | 
| 
       192 
192 
     | 
    
         
             
                field = fields[i];
         
     | 
| 
       193 
193 
     | 
    
         
             
                field_name = field->name;
         
     | 
| 
       194 
     | 
    
         
            -
                fi = ((FieldInfo *)ht_get( 
     | 
| 
      
 194 
     | 
    
         
            +
                fi = ((FieldInfo *)ht_get(self->fis->by_name, field_name));
         
     | 
| 
       195 
195 
     | 
    
         
             
                field_number = fi->number;
         
     | 
| 
       196 
196 
     | 
    
         | 
| 
       197 
     | 
    
         
            -
                length =  
     | 
| 
       198 
     | 
    
         
            -
                offset =  
     | 
| 
       199 
     | 
    
         
            -
                position =  
     | 
| 
      
 197 
     | 
    
         
            +
                length = self->field_lengths[field_number];
         
     | 
| 
      
 198 
     | 
    
         
            +
                offset = self->field_offsets[field_number];
         
     | 
| 
      
 199 
     | 
    
         
            +
                position = self->field_positions[field_number];
         
     | 
| 
       200 
200 
     | 
    
         | 
| 
       201 
201 
     | 
    
         
             
                if (fi->is_indexed) {
         
     | 
| 
       202 
     | 
    
         
            -
                  if (!field->is_tokenized) { 
     | 
| 
      
 202 
     | 
    
         
            +
                  if (!field->is_tokenized) { /* un-tokenized field */
         
     | 
| 
       203 
203 
     | 
    
         
             
                    text = field->data;
         
     | 
| 
       204 
     | 
    
         
            -
                    slen = strlen(text);
         
     | 
| 
      
 204 
     | 
    
         
            +
                    slen = (int)strlen(text);
         
     | 
| 
       205 
205 
     | 
    
         
             
                    if (fi->store_offset) {
         
     | 
| 
       206 
     | 
    
         
            -
                      dw_add_position( 
     | 
| 
      
 206 
     | 
    
         
            +
                      dw_add_position(self, field_name, text, position,
         
     | 
| 
       207 
207 
     | 
    
         
             
                          tvoi_create(offset, offset+slen));
         
     | 
| 
       208 
208 
     | 
    
         
             
                    } else {
         
     | 
| 
       209 
     | 
    
         
            -
                      dw_add_position( 
     | 
| 
      
 209 
     | 
    
         
            +
                      dw_add_position(self, field_name, text, position, NULL);
         
     | 
| 
       210 
210 
     | 
    
         
             
                    }
         
     | 
| 
       211 
211 
     | 
    
         
             
                    offset += slen;
         
     | 
| 
       212 
212 
     | 
    
         
             
                    length++;
         
     | 
| 
       213 
213 
     | 
    
         
             
                  } else {
         
     | 
| 
       214 
214 
     | 
    
         | 
| 
       215 
     | 
    
         
            -
                     
     | 
| 
       216 
     | 
    
         
            -
                    stream = a_get_ts( 
     | 
| 
      
 215 
     | 
    
         
            +
                    /* Tokenize field and add to posting_table */
         
     | 
| 
      
 216 
     | 
    
         
            +
                    stream = a_get_ts(self->analyzer, field_name, field->data);
         
     | 
| 
       217 
217 
     | 
    
         | 
| 
       218 
218 
     | 
    
         
             
                    while ((token = ts_next(stream)) != NULL) {
         
     | 
| 
       219 
219 
     | 
    
         
             
                      position += (token->pos_inc - 1);
         
     | 
| 
       220 
220 
     | 
    
         | 
| 
       221 
221 
     | 
    
         
             
                      if (fi->store_offset) {
         
     | 
| 
       222 
     | 
    
         
            -
                        dw_add_position( 
     | 
| 
      
 222 
     | 
    
         
            +
                        dw_add_position(self, 
         
     | 
| 
       223 
223 
     | 
    
         
             
                            field_name,
         
     | 
| 
       224 
224 
     | 
    
         
             
                            token->text,
         
     | 
| 
       225 
225 
     | 
    
         
             
                            position,
         
     | 
| 
       226 
226 
     | 
    
         
             
                            tvoi_create(offset + token->start, offset + token->end));
         
     | 
| 
       227 
227 
     | 
    
         
             
                        position++;
         
     | 
| 
       228 
228 
     | 
    
         
             
                      } else {
         
     | 
| 
       229 
     | 
    
         
            -
                        dw_add_position( 
     | 
| 
      
 229 
     | 
    
         
            +
                        dw_add_position(self, field_name, token->text, position, NULL);
         
     | 
| 
       230 
230 
     | 
    
         
             
                        position++;
         
     | 
| 
       231 
231 
     | 
    
         
             
                      }
         
     | 
| 
       232 
232 
     | 
    
         | 
| 
       233 
233 
     | 
    
         
             
                      length++;
         
     | 
| 
       234 
     | 
    
         
            -
                       
     | 
| 
       235 
     | 
    
         
            -
                      if (length >  
     | 
| 
      
 234 
     | 
    
         
            +
                      /* stop if we reach the max field length */
         
     | 
| 
      
 235 
     | 
    
         
            +
                      if (length > self->max_field_length) {
         
     | 
| 
       236 
236 
     | 
    
         
             
                        break;
         
     | 
| 
      
 237 
     | 
    
         
            +
                      }
         
     | 
| 
       237 
238 
     | 
    
         
             
                    }
         
     | 
| 
       238 
239 
     | 
    
         | 
| 
       239 
     | 
    
         
            -
                    if (token)
         
     | 
| 
      
 240 
     | 
    
         
            +
                    if (token) {
         
     | 
| 
       240 
241 
     | 
    
         
             
                      offset += token->end + 1;
         
     | 
| 
      
 242 
     | 
    
         
            +
                    }
         
     | 
| 
       241 
243 
     | 
    
         
             
                  }
         
     | 
| 
       242 
     | 
    
         
            -
                   
     | 
| 
       243 
     | 
    
         
            -
                   
     | 
| 
       244 
     | 
    
         
            -
                   
     | 
| 
       245 
     | 
    
         
            -
                   
     | 
| 
      
 244 
     | 
    
         
            +
                  self->field_lengths[field_number] = length;
         
     | 
| 
      
 245 
     | 
    
         
            +
                  self->field_offsets[field_number] = offset;
         
     | 
| 
      
 246 
     | 
    
         
            +
                  self->field_positions[field_number] = position;
         
     | 
| 
      
 247 
     | 
    
         
            +
                  self->field_boosts[field_number] *= field->boost;
         
     | 
| 
       246 
248 
     | 
    
         
             
                }
         
     | 
| 
       247 
249 
     | 
    
         
             
              }
         
     | 
| 
       248 
250 
     | 
    
         
             
            }
         
     | 
| 
       249 
251 
     | 
    
         | 
| 
       250 
     | 
    
         
            -
            Posting **dw_sort_posting_table(DocumentWriter * 
     | 
| 
      
 252 
     | 
    
         
            +
            Posting **dw_sort_posting_table(DocumentWriter *self)
         
     | 
| 
       251 
253 
     | 
    
         
             
            {
         
     | 
| 
       252 
     | 
    
         
            -
              HshTable *ht =  
     | 
| 
       253 
     | 
    
         
            -
              int i;
         
     | 
| 
       254 
     | 
    
         
            -
              dw->pcnt = i = ht->used;
         
     | 
| 
       255 
     | 
    
         
            -
              Posting **postings = ALLOC_N(Posting *, i);
         
     | 
| 
      
 254 
     | 
    
         
            +
              HshTable *ht = self->postingtable;
         
     | 
| 
       256 
255 
     | 
    
         
             
              HshEntry *he = ht->table;
         
     | 
| 
      
 256 
     | 
    
         
            +
              Posting **postings;
         
     | 
| 
      
 257 
     | 
    
         
            +
              int i;
         
     | 
| 
      
 258 
     | 
    
         
            +
             
     | 
| 
      
 259 
     | 
    
         
            +
              self->pcnt = i = ht->used;
         
     | 
| 
      
 260 
     | 
    
         
            +
              postings = ALLOC_N(Posting *, i);
         
     | 
| 
      
 261 
     | 
    
         
            +
             
     | 
| 
       257 
262 
     | 
    
         
             
              while (i > 0) {
         
     | 
| 
       258 
263 
     | 
    
         
             
                if (he->value != NULL) {
         
     | 
| 
       259 
264 
     | 
    
         
             
                  i--;
         
     | 
| 
         @@ -261,16 +266,16 @@ Posting **dw_sort_posting_table(DocumentWriter *dw) 
     | 
|
| 
       261 
266 
     | 
    
         
             
                }
         
     | 
| 
       262 
267 
     | 
    
         
             
                he++;
         
     | 
| 
       263 
268 
     | 
    
         
             
              }
         
     | 
| 
       264 
     | 
    
         
            -
              qsort(postings,  
     | 
| 
      
 269 
     | 
    
         
            +
              qsort(postings, self->pcnt, sizeof(Posting *), &p_cmp);
         
     | 
| 
       265 
270 
     | 
    
         
             
              return postings;
         
     | 
| 
       266 
271 
     | 
    
         
             
            }
         
     | 
| 
       267 
272 
     | 
    
         | 
| 
       268 
     | 
    
         
            -
            void dw_write_postings(DocumentWriter * 
     | 
| 
      
 273 
     | 
    
         
            +
            void dw_write_postings(DocumentWriter *self, Posting **postings, char *segment)
         
     | 
| 
       269 
274 
     | 
    
         
             
            {
         
     | 
| 
       270 
275 
     | 
    
         
             
              OutStream * volatile freq_out = NULL, * volatile prox_out = NULL;
         
     | 
| 
       271 
276 
     | 
    
         
             
              TermInfosWriter * volatile tiw = NULL;
         
     | 
| 
       272 
277 
     | 
    
         
             
              TermVectorsWriter * volatile tvw = NULL;
         
     | 
| 
       273 
     | 
    
         
            -
              Store *store =  
     | 
| 
      
 278 
     | 
    
         
            +
              Store *store = self->store;
         
     | 
| 
       274 
279 
     | 
    
         
             
              TermInfo * volatile ti = NULL;
         
     | 
| 
       275 
280 
     | 
    
         
             
              Posting *posting;
         
     | 
| 
       276 
281 
     | 
    
         
             
              int i, j, posting_freq, position, last_position;
         
     | 
| 
         @@ -278,31 +283,31 @@ void dw_write_postings(DocumentWriter *dw, Posting **postings, char *segment) 
     | 
|
| 
       278 
283 
     | 
    
         
             
              strcpy(fname, segment);
         
     | 
| 
       279 
284 
     | 
    
         | 
| 
       280 
285 
     | 
    
         
             
              TRY
         
     | 
| 
       281 
     | 
    
         
            -
                 
     | 
| 
      
 286 
     | 
    
         
            +
                /* open files for inverse index storage */
         
     | 
| 
       282 
287 
     | 
    
         
             
                sprintf(fname, "%s.frq", segment);
         
     | 
| 
       283 
288 
     | 
    
         
             
                freq_out = store->create_output(store, fname);
         
     | 
| 
       284 
289 
     | 
    
         
             
                sprintf(fname, "%s.prx", segment);
         
     | 
| 
       285 
290 
     | 
    
         
             
                prox_out = store->create_output(store, fname);
         
     | 
| 
       286 
     | 
    
         
            -
                tiw = tiw_open(store, segment,  
     | 
| 
      
 291 
     | 
    
         
            +
                tiw = tiw_open(store, segment, self->fis, self->term_index_interval);
         
     | 
| 
       287 
292 
     | 
    
         
             
                ti = ti_create(0, 0, 0, 0);
         
     | 
| 
       288 
293 
     | 
    
         | 
| 
       289 
     | 
    
         
            -
                for (i = 0; i <  
     | 
| 
      
 294 
     | 
    
         
            +
                for (i = 0; i < self->pcnt; i++) {
         
     | 
| 
       290 
295 
     | 
    
         
             
                  posting = postings[i];
         
     | 
| 
       291 
296 
     | 
    
         | 
| 
       292 
     | 
    
         
            -
                   
     | 
| 
      
 297 
     | 
    
         
            +
                  /* add an entry to dictionary with pointers to prox and freq_out files */
         
     | 
| 
       293 
298 
     | 
    
         
             
                  ti_set(ti, 1, os_pos(freq_out), os_pos(prox_out), -1);
         
     | 
| 
       294 
299 
     | 
    
         
             
                  tiw_add(tiw, posting->term, ti);
         
     | 
| 
       295 
300 
     | 
    
         | 
| 
       296 
     | 
    
         
            -
                   
     | 
| 
      
 301 
     | 
    
         
            +
                  /* add an entry to the freq_out file */
         
     | 
| 
       297 
302 
     | 
    
         
             
                  posting_freq = posting->freq;
         
     | 
| 
       298 
     | 
    
         
            -
                  if (posting_freq == 1) { 
     | 
| 
       299 
     | 
    
         
            -
                    os_write_vint(freq_out, 1);             
     | 
| 
      
 303 
     | 
    
         
            +
                  if (posting_freq == 1) {                 /* optimize freq=1 */
         
     | 
| 
      
 304 
     | 
    
         
            +
                    os_write_vint(freq_out, 1);            /* set low bit of doc num */
         
     | 
| 
       300 
305 
     | 
    
         
             
                  } else {
         
     | 
| 
       301 
     | 
    
         
            -
                    os_write_vint(freq_out, 0);             
     | 
| 
       302 
     | 
    
         
            -
                    os_write_vint(freq_out, posting_freq);  
     | 
| 
      
 306 
     | 
    
         
            +
                    os_write_vint(freq_out, 0);            /* the doc number */
         
     | 
| 
      
 307 
     | 
    
         
            +
                    os_write_vint(freq_out, posting_freq); /* frequency in doc */
         
     | 
| 
       303 
308 
     | 
    
         
             
                  }
         
     | 
| 
       304 
309 
     | 
    
         | 
| 
       305 
     | 
    
         
            -
                  last_position = 0;                
     | 
| 
      
 310 
     | 
    
         
            +
                  last_position = 0;               /* write positions */
         
     | 
| 
       306 
311 
     | 
    
         | 
| 
       307 
312 
     | 
    
         
             
                  for (j = 0; j < posting_freq; j++) {
         
     | 
| 
       308 
313 
     | 
    
         
             
                    position = posting->positions[j];
         
     | 
| 
         @@ -310,16 +315,16 @@ void dw_write_postings(DocumentWriter *dw, Posting **postings, char *segment) 
     | 
|
| 
       310 
315 
     | 
    
         
             
                    last_position = position;
         
     | 
| 
       311 
316 
     | 
    
         
             
                  }
         
     | 
| 
       312 
317 
     | 
    
         | 
| 
       313 
     | 
    
         
            -
                   
     | 
| 
      
 318 
     | 
    
         
            +
                  /* check to see if we switched to a new field */
         
     | 
| 
       314 
319 
     | 
    
         
             
                  term_field = posting->term->field;
         
     | 
| 
       315 
320 
     | 
    
         
             
                  if (curr_field != term_field)  {
         
     | 
| 
       316 
321 
     | 
    
         
             
                    FieldInfo *fi;
         
     | 
| 
       317 
     | 
    
         
            -
                     
     | 
| 
      
 322 
     | 
    
         
            +
                    /* changing field - see if there is something to save */
         
     | 
| 
       318 
323 
     | 
    
         
             
                    curr_field = term_field;
         
     | 
| 
       319 
     | 
    
         
            -
                    fi = (FieldInfo *)ht_get( 
     | 
| 
      
 324 
     | 
    
         
            +
                    fi = (FieldInfo *)ht_get(self->fis->by_name, curr_field);
         
     | 
| 
       320 
325 
     | 
    
         
             
                    if (fi->store_tv) {
         
     | 
| 
       321 
326 
     | 
    
         
             
                      if (tvw == NULL) {
         
     | 
| 
       322 
     | 
    
         
            -
                        tvw = tvw_open(store, segment,  
     | 
| 
      
 327 
     | 
    
         
            +
                        tvw = tvw_open(store, segment, self->fis);
         
     | 
| 
       323 
328 
     | 
    
         
             
                        tvw_open_doc(tvw);
         
     | 
| 
       324 
329 
     | 
    
         
             
                      }
         
     | 
| 
       325 
330 
     | 
    
         
             
                      tvw_open_field(tvw, curr_field);
         
     | 
| 
         @@ -328,7 +333,7 @@ void dw_write_postings(DocumentWriter *dw, Posting **postings, char *segment) 
     | 
|
| 
       328 
333 
     | 
    
         
             
                      tvw_close_field(tvw);
         
     | 
| 
       329 
334 
     | 
    
         
             
                    }
         
     | 
| 
       330 
335 
     | 
    
         
             
                  }
         
     | 
| 
       331 
     | 
    
         
            -
                   
     | 
| 
      
 336 
     | 
    
         
            +
                  /* tvw->curr_field != NULL implies field is still open */
         
     | 
| 
       332 
337 
     | 
    
         
             
                  if (tvw != NULL && tvw->curr_field != NULL) {
         
     | 
| 
       333 
338 
     | 
    
         
             
                    tvw_add_term(tvw, posting->term->text, posting_freq, posting->positions, posting->offsets);
         
     | 
| 
       334 
339 
     | 
    
         
             
                  }
         
     | 
| 
         @@ -338,8 +343,8 @@ void dw_write_postings(DocumentWriter *dw, Posting **postings, char *segment) 
     | 
|
| 
       338 
343 
     | 
    
         
             
                  tvw_close_doc(tvw);
         
     | 
| 
       339 
344 
     | 
    
         
             
                  tvw_close(tvw);
         
     | 
| 
       340 
345 
     | 
    
         
             
                }
         
     | 
| 
       341 
     | 
    
         
            -
                 
     | 
| 
       342 
     | 
    
         
            -
             
     | 
| 
      
 346 
     | 
    
         
            +
                /* make an effort to close all streams we can but remember and re-raise
         
     | 
| 
      
 347 
     | 
    
         
            +
                 * the last exception encountered in this process */
         
     | 
| 
       343 
348 
     | 
    
         
             
                if (freq_out) os_close(freq_out);
         
     | 
| 
       344 
349 
     | 
    
         
             
                if (prox_out) os_close(prox_out);
         
     | 
| 
       345 
350 
     | 
    
         
             
                if (tiw) tiw_close(tiw);
         
     | 
| 
         @@ -347,24 +352,25 @@ void dw_write_postings(DocumentWriter *dw, Posting **postings, char *segment) 
     | 
|
| 
       347 
352 
     | 
    
         
             
              XENDTRY
         
     | 
| 
       348 
353 
     | 
    
         
             
            }
         
     | 
| 
       349 
354 
     | 
    
         | 
| 
       350 
     | 
    
         
            -
            void dw_write_norms(DocumentWriter * 
     | 
| 
      
 355 
     | 
    
         
            +
            void dw_write_norms(DocumentWriter *self, char *segment)
         
     | 
| 
       351 
356 
     | 
    
         
             
            {
         
     | 
| 
       352 
357 
     | 
    
         
             
              int i;
         
     | 
| 
       353 
358 
     | 
    
         
             
              float norm;
         
     | 
| 
       354 
359 
     | 
    
         
             
              OutStream *norms_out;
         
     | 
| 
       355 
360 
     | 
    
         
             
              char fname[SEGMENT_NAME_MAX_LENGTH];
         
     | 
| 
       356 
     | 
    
         
            -
              FieldInfos *fis =  
     | 
| 
      
 361 
     | 
    
         
            +
              FieldInfos *fis = self->fis;
         
     | 
| 
       357 
362 
     | 
    
         
             
              FieldInfo *fi;
         
     | 
| 
       358 
363 
     | 
    
         | 
| 
       359 
364 
     | 
    
         
             
              for (i = 0; i < fis->fcnt; i++) {
         
     | 
| 
       360 
365 
     | 
    
         
             
                fi = fis->by_number[i];
         
     | 
| 
       361 
366 
     | 
    
         | 
| 
       362 
367 
     | 
    
         
             
                if (fi->is_indexed && !fi->omit_norms) {
         
     | 
| 
       363 
     | 
    
         
            -
                  norm =  
     | 
| 
      
 368 
     | 
    
         
            +
                  norm = self->field_boosts[i] *
         
     | 
| 
      
 369 
     | 
    
         
            +
                    sim_length_norm(self->similarity, fi->name, self->field_lengths[i]);
         
     | 
| 
       364 
370 
     | 
    
         
             
                  sprintf(fname, "%s.f%d", segment, i);
         
     | 
| 
       365 
     | 
    
         
            -
                  norms_out =  
     | 
| 
      
 371 
     | 
    
         
            +
                  norms_out = self->store->create_output(self->store, fname);
         
     | 
| 
       366 
372 
     | 
    
         
             
                  TRY
         
     | 
| 
       367 
     | 
    
         
            -
                    os_write_byte(norms_out, sim_encode_norm( 
     | 
| 
      
 373 
     | 
    
         
            +
                    os_write_byte(norms_out, sim_encode_norm(self->similarity, norm));
         
     | 
| 
       368 
374 
     | 
    
         
             
                  XFINALLY
         
     | 
| 
       369 
375 
     | 
    
         
             
                    os_close(norms_out);
         
     | 
| 
       370 
376 
     | 
    
         
             
                  XENDTRY
         
     | 
| 
         @@ -372,49 +378,54 @@ void dw_write_norms(DocumentWriter *dw, char *segment) 
     | 
|
| 
       372 
378 
     | 
    
         
             
              }
         
     | 
| 
       373 
379 
     | 
    
         
             
            }
         
     | 
| 
       374 
380 
     | 
    
         | 
| 
       375 
     | 
    
         
            -
            void dw_add_doc(DocumentWriter * 
     | 
| 
      
 381 
     | 
    
         
            +
            void dw_add_doc(DocumentWriter *self, char *segment, Document *doc)
         
     | 
| 
       376 
382 
     | 
    
         
             
            {
         
     | 
| 
      
 383 
     | 
    
         
            +
              Posting **postings;
         
     | 
| 
      
 384 
     | 
    
         
            +
              FieldsWriter *fw;
         
     | 
| 
       377 
385 
     | 
    
         
             
              int i;
         
     | 
| 
       378 
     | 
    
         
            -
              // write field names
         
     | 
| 
       379 
     | 
    
         
            -
              dw->fis = fis_create();
         
     | 
| 
       380 
     | 
    
         
            -
              fis_add_doc(dw->fis, doc);
         
     | 
| 
       381 
     | 
    
         
            -
              fis_write(dw->fis, dw->store, segment, ".fnm");
         
     | 
| 
       382 
386 
     | 
    
         | 
| 
       383 
     | 
    
         
            -
               
     | 
| 
       384 
     | 
    
         
            -
               
     | 
| 
      
 387 
     | 
    
         
            +
              /* write field names */
         
     | 
| 
      
 388 
     | 
    
         
            +
              self->fis = fis_create();
         
     | 
| 
      
 389 
     | 
    
         
            +
              fis_add_doc(self->fis, doc);
         
     | 
| 
      
 390 
     | 
    
         
            +
              fis_write(self->fis, self->store, segment, ".fnm");
         
     | 
| 
      
 391 
     | 
    
         
            +
             
     | 
| 
      
 392 
     | 
    
         
            +
              /* write field values */
         
     | 
| 
      
 393 
     | 
    
         
            +
              fw = fw_open(self->store, segment, self->fis);
         
     | 
| 
       385 
394 
     | 
    
         
             
              TRY
         
     | 
| 
       386 
395 
     | 
    
         
             
                fw_add_doc(fw, doc);
         
     | 
| 
       387 
396 
     | 
    
         
             
              XFINALLY
         
     | 
| 
       388 
397 
     | 
    
         
             
                fw_close(fw);
         
     | 
| 
       389 
398 
     | 
    
         
             
              XENDTRY
         
     | 
| 
       390 
399 
     | 
    
         | 
| 
       391 
     | 
    
         
            -
               
     | 
| 
       392 
     | 
    
         
            -
              h_clear(dw->postingtable);        // clear posting_table
         
     | 
| 
      
 400 
     | 
    
         
            +
              /* invert doc into posting_table */
         
     | 
| 
       393 
401 
     | 
    
         | 
| 
       394 
     | 
    
         
            -
               
     | 
| 
       395 
     | 
    
         
            -
              dw->field_lengths = ALLOC_AND_ZERO_N(int, dw->fis->fcnt);
         
     | 
| 
       396 
     | 
    
         
            -
              dw->field_offsets = ALLOC_AND_ZERO_N(int, dw->fis->fcnt);
         
     | 
| 
       397 
     | 
    
         
            -
              dw->field_positions = ALLOC_AND_ZERO_N(int, dw->fis->fcnt);
         
     | 
| 
      
 402 
     | 
    
         
            +
              h_clear(self->postingtable); /* clear posting_table */
         
     | 
| 
       398 
403 
     | 
    
         | 
| 
       399 
     | 
    
         
            -
               
     | 
| 
       400 
     | 
    
         
            -
             
     | 
| 
      
 404 
     | 
    
         
            +
              self->field_boosts = ALLOC_N(float, self->fis->fcnt);
         
     | 
| 
      
 405 
     | 
    
         
            +
              self->field_lengths = ALLOC_AND_ZERO_N(int, self->fis->fcnt);
         
     | 
| 
      
 406 
     | 
    
         
            +
              self->field_offsets = ALLOC_AND_ZERO_N(int, self->fis->fcnt);
         
     | 
| 
      
 407 
     | 
    
         
            +
              self->field_positions = ALLOC_AND_ZERO_N(int, self->fis->fcnt);
         
     | 
| 
       401 
408 
     | 
    
         | 
| 
       402 
     | 
    
         
            -
               
     | 
| 
      
 409 
     | 
    
         
            +
              for (i = 0; i < self->fis->fcnt; i++) {
         
     | 
| 
      
 410 
     | 
    
         
            +
                self->field_boosts[i] = doc->boost;
         
     | 
| 
      
 411 
     | 
    
         
            +
              }
         
     | 
| 
      
 412 
     | 
    
         
            +
             
     | 
| 
      
 413 
     | 
    
         
            +
              dw_invert_doc(self, doc);
         
     | 
| 
       403 
414 
     | 
    
         | 
| 
       404 
     | 
    
         
            -
               
     | 
| 
       405 
     | 
    
         
            -
               
     | 
| 
      
 415 
     | 
    
         
            +
              /* sort posting_table into an array */
         
     | 
| 
      
 416 
     | 
    
         
            +
              postings = dw_sort_posting_table(self);
         
     | 
| 
       406 
417 
     | 
    
         | 
| 
       407 
     | 
    
         
            -
               
     | 
| 
       408 
     | 
    
         
            -
              dw_write_postings( 
     | 
| 
      
 418 
     | 
    
         
            +
              /* write postings */
         
     | 
| 
      
 419 
     | 
    
         
            +
              dw_write_postings(self, postings, segment);
         
     | 
| 
       409 
420 
     | 
    
         
             
              free(postings);
         
     | 
| 
       410 
421 
     | 
    
         | 
| 
       411 
     | 
    
         
            -
               
     | 
| 
       412 
     | 
    
         
            -
              dw_write_norms( 
     | 
| 
      
 422 
     | 
    
         
            +
              /* write norms of indexed fields */
         
     | 
| 
      
 423 
     | 
    
         
            +
              dw_write_norms(self, segment);
         
     | 
| 
       413 
424 
     | 
    
         | 
| 
       414 
     | 
    
         
            -
              free( 
     | 
| 
       415 
     | 
    
         
            -
              free( 
     | 
| 
       416 
     | 
    
         
            -
              free( 
     | 
| 
       417 
     | 
    
         
            -
              free( 
     | 
| 
      
 425 
     | 
    
         
            +
              free(self->field_boosts);
         
     | 
| 
      
 426 
     | 
    
         
            +
              free(self->field_lengths);
         
     | 
| 
      
 427 
     | 
    
         
            +
              free(self->field_offsets);
         
     | 
| 
      
 428 
     | 
    
         
            +
              free(self->field_positions);
         
     | 
| 
       418 
429 
     | 
    
         
             
            }
         
     | 
| 
       419 
430 
     | 
    
         | 
| 
       420 
431 
     | 
    
         
             
            /****************************************************************************
         
     | 
| 
         @@ -432,9 +443,8 @@ SegmentInfo *si_create(char *name, int doc_cnt, Store *store) 
     | 
|
| 
       432 
443 
     | 
    
         
             
              return si;
         
     | 
| 
       433 
444 
     | 
    
         
             
            }
         
     | 
| 
       434 
445 
     | 
    
         | 
| 
       435 
     | 
    
         
            -
            void si_destroy( 
     | 
| 
      
 446 
     | 
    
         
            +
            void si_destroy(SegmentInfo *si)
         
     | 
| 
       436 
447 
     | 
    
         
             
            {
         
     | 
| 
       437 
     | 
    
         
            -
              SegmentInfo *si = (SegmentInfo *)p;
         
     | 
| 
       438 
448 
     | 
    
         
             
              free(si->name);
         
     | 
| 
       439 
449 
     | 
    
         
             
              free(si);
         
     | 
| 
       440 
450 
     | 
    
         
             
            }
         
     | 
| 
         @@ -501,21 +511,19 @@ SegmentInfos *sis_create() 
     | 
|
| 
       501 
511 
     | 
    
         
             
              return sis;
         
     | 
| 
       502 
512 
     | 
    
         
             
            }
         
     | 
| 
       503 
513 
     | 
    
         | 
| 
       504 
     | 
    
         
            -
            void sis_destroy_not_infos( 
     | 
| 
      
 514 
     | 
    
         
            +
            void sis_destroy_not_infos(SegmentInfos *sis)
         
     | 
| 
       505 
515 
     | 
    
         
             
            {
         
     | 
| 
       506 
     | 
    
         
            -
              SegmentInfos *sis = (SegmentInfos *)p;
         
     | 
| 
       507 
516 
     | 
    
         
             
              free(sis->segs);
         
     | 
| 
       508 
     | 
    
         
            -
              free( 
     | 
| 
      
 517 
     | 
    
         
            +
              free(sis);
         
     | 
| 
       509 
518 
     | 
    
         
             
            }
         
     | 
| 
       510 
519 
     | 
    
         | 
| 
       511 
     | 
    
         
            -
            void sis_destroy( 
     | 
| 
      
 520 
     | 
    
         
            +
            void sis_destroy(SegmentInfos *sis)
         
     | 
| 
       512 
521 
     | 
    
         
             
            {
         
     | 
| 
       513 
522 
     | 
    
         
             
              int i;
         
     | 
| 
       514 
     | 
    
         
            -
              SegmentInfos *sis = (SegmentInfos *)p;
         
     | 
| 
       515 
523 
     | 
    
         
             
              for (i = 0; i < sis->scnt; i++)
         
     | 
| 
       516 
524 
     | 
    
         
             
                si_destroy(sis->segs[i]);
         
     | 
| 
       517 
525 
     | 
    
         
             
              free(sis->segs);
         
     | 
| 
       518 
     | 
    
         
            -
              free( 
     | 
| 
      
 526 
     | 
    
         
            +
              free(sis);
         
     | 
| 
       519 
527 
     | 
    
         
             
            }
         
     | 
| 
       520 
528 
     | 
    
         | 
| 
       521 
529 
     | 
    
         
             
            void sis_add_si(SegmentInfos *sis, SegmentInfo *si)
         
     | 
| 
         @@ -533,8 +541,9 @@ void sis_del_at(SegmentInfos *sis, int at) 
     | 
|
| 
       533 
541 
     | 
    
         
             
              int i;
         
     | 
| 
       534 
542 
     | 
    
         
             
              si_destroy(sis->segs[at]);
         
     | 
| 
       535 
543 
     | 
    
         
             
              sis->scnt--;
         
     | 
| 
       536 
     | 
    
         
            -
              for (i = at; i < sis->scnt; i++)
         
     | 
| 
      
 544 
     | 
    
         
            +
              for (i = at; i < sis->scnt; i++) {
         
     | 
| 
       537 
545 
     | 
    
         
             
                sis->segs[i] = sis->segs[i+1];
         
     | 
| 
      
 546 
     | 
    
         
            +
              }
         
     | 
| 
       538 
547 
     | 
    
         
             
            }
         
     | 
| 
       539 
548 
     | 
    
         | 
| 
       540 
549 
     | 
    
         
             
            void sis_del_from_to(SegmentInfos *sis, int from, int to)
         
     | 
| 
         @@ -561,24 +570,25 @@ void sis_clear(SegmentInfos *sis) 
     | 
|
| 
       561 
570 
     | 
    
         
             
            void sis_read(SegmentInfos *sis, Store *store)
         
     | 
| 
       562 
571 
     | 
    
         
             
            {
         
     | 
| 
       563 
572 
     | 
    
         
             
              int doc_cnt;
         
     | 
| 
      
 573 
     | 
    
         
            +
              int seg_count;
         
     | 
| 
      
 574 
     | 
    
         
            +
              int i;
         
     | 
| 
       564 
575 
     | 
    
         
             
              char *name;
         
     | 
| 
       565 
576 
     | 
    
         
             
              InStream *is = store->open_input(store, SEGMENT_FILENAME);
         
     | 
| 
       566 
577 
     | 
    
         | 
| 
       567 
578 
     | 
    
         
             
              TRY
         
     | 
| 
       568 
579 
     | 
    
         | 
| 
       569 
580 
     | 
    
         
             
                sis->format = is_read_int(is);
         
     | 
| 
       570 
     | 
    
         
            -
                if (sis->format < 0) {  
     | 
| 
       571 
     | 
    
         
            -
                   
     | 
| 
      
 581 
     | 
    
         
            +
                if (sis->format < 0) { /* file contains explicit format info */
         
     | 
| 
      
 582 
     | 
    
         
            +
                  /* check that it is a format we can understand */
         
     | 
| 
       572 
583 
     | 
    
         
             
                  if (sis->format < FORMAT)
         
     | 
| 
       573 
     | 
    
         
            -
                    RAISE( 
     | 
| 
       574 
     | 
    
         
            -
                  sis->version = is_read_long(is);
         
     | 
| 
       575 
     | 
    
         
            -
                  sis->counter = is_read_int(is);
         
     | 
| 
       576 
     | 
    
         
            -
                } else {  
     | 
| 
      
 584 
     | 
    
         
            +
                    RAISE(EXCEPTION, FORMAT_VERSION_ERROR_MSG);
         
     | 
| 
      
 585 
     | 
    
         
            +
                  sis->version = (uint)is_read_long(is);
         
     | 
| 
      
 586 
     | 
    
         
            +
                  sis->counter = (int)is_read_int(is);
         
     | 
| 
      
 587 
     | 
    
         
            +
                } else { /* file is in old format without explicit format info */
         
     | 
| 
       577 
588 
     | 
    
         
             
                  sis->counter = sis->format;
         
     | 
| 
       578 
589 
     | 
    
         
             
                }
         
     | 
| 
       579 
590 
     | 
    
         | 
| 
       580 
     | 
    
         
            -
                 
     | 
| 
       581 
     | 
    
         
            -
                int i;
         
     | 
| 
      
 591 
     | 
    
         
            +
                seg_count = is_read_int(is);
         
     | 
| 
       582 
592 
     | 
    
         
             
                for (i = 0; i < seg_count; i++) {
         
     | 
| 
       583 
593 
     | 
    
         
             
                  name = is_read_string(is);
         
     | 
| 
       584 
594 
     | 
    
         
             
                  doc_cnt = is_read_int(is);
         
     | 
| 
         @@ -586,11 +596,12 @@ void sis_read(SegmentInfos *sis, Store *store) 
     | 
|
| 
       586 
596 
     | 
    
         
             
                }
         
     | 
| 
       587 
597 
     | 
    
         | 
| 
       588 
598 
     | 
    
         
             
                if (sis->format >= 0) {
         
     | 
| 
       589 
     | 
    
         
            -
                   
     | 
| 
       590 
     | 
    
         
            -
                  if (is_pos(is) >= is_length(is))
         
     | 
| 
       591 
     | 
    
         
            -
                    sis->version = 0;  
     | 
| 
       592 
     | 
    
         
            -
                  else
         
     | 
| 
       593 
     | 
    
         
            -
                    sis->version = is_read_long(is);  
     | 
| 
      
 599 
     | 
    
         
            +
                  /* in old format the version number may be at the end of the file */
         
     | 
| 
      
 600 
     | 
    
         
            +
                  if (is_pos(is) >= is_length(is)) {
         
     | 
| 
      
 601 
     | 
    
         
            +
                    sis->version = 0; /* old file format without version number */
         
     | 
| 
      
 602 
     | 
    
         
            +
                  } else {
         
     | 
| 
      
 603 
     | 
    
         
            +
                    sis->version = (int)is_read_long(is); /* read version */
         
     | 
| 
      
 604 
     | 
    
         
            +
                  }
         
     | 
| 
       594 
605 
     | 
    
         
             
                }
         
     | 
| 
       595 
606 
     | 
    
         
             
              XFINALLY
         
     | 
| 
       596 
607 
     | 
    
         
             
                is_close(is);
         
     | 
| 
         @@ -604,7 +615,7 @@ void sis_write(SegmentInfos *sis, Store *store) 
     | 
|
| 
       604 
615 
     | 
    
         
             
              OutStream *os = store->create_output(store, TEMPORARY_SEGMENT_FILENAME);
         
     | 
| 
       605 
616 
     | 
    
         
             
              TRY
         
     | 
| 
       606 
617 
     | 
    
         
             
                os_write_int(os, FORMAT);
         
     | 
| 
       607 
     | 
    
         
            -
                os_write_long(os, ++(sis->version));  
     | 
| 
      
 618 
     | 
    
         
            +
                os_write_long(os, ++(sis->version)); /* every write changes the index */
         
     | 
| 
       608 
619 
     | 
    
         
             
                os_write_int(os, sis->counter);
         
     | 
| 
       609 
620 
     | 
    
         
             
                os_write_int(os, sis->scnt); 
         
     | 
| 
       610 
621 
     | 
    
         
             
                for (i = 0; i < sis->scnt; i++) {
         
     | 
| 
         @@ -617,24 +628,27 @@ void sis_write(SegmentInfos *sis, Store *store) 
     | 
|
| 
       617 
628 
     | 
    
         
             
                os_close(os);
         
     | 
| 
       618 
629 
     | 
    
         
             
              XENDTRY
         
     | 
| 
       619 
630 
     | 
    
         | 
| 
       620 
     | 
    
         
            -
               
     | 
| 
      
 631 
     | 
    
         
            +
              /* install new segment info */
         
     | 
| 
       621 
632 
     | 
    
         
             
              store->rename(store, TEMPORARY_SEGMENT_FILENAME, SEGMENT_FILENAME);
         
     | 
| 
       622 
633 
     | 
    
         
             
            }
         
     | 
| 
       623 
634 
     | 
    
         | 
| 
       624 
635 
     | 
    
         
             
            int sis_read_current_version(Store *store)
         
     | 
| 
       625 
636 
     | 
    
         
             
            {
         
     | 
| 
       626 
     | 
    
         
            -
               
     | 
| 
       627 
     | 
    
         
            -
             
     | 
| 
       628 
     | 
    
         
            -
              InStream *is = store->open_input(store, SEGMENT_FILENAME);
         
     | 
| 
      
 637 
     | 
    
         
            +
              InStream *is;
         
     | 
| 
      
 638 
     | 
    
         
            +
              SegmentInfos *sis;
         
     | 
| 
       629 
639 
     | 
    
         
             
              int format = 0;
         
     | 
| 
       630 
640 
     | 
    
         
             
              int version = 0;
         
     | 
| 
       631 
641 
     | 
    
         | 
| 
      
 642 
     | 
    
         
            +
              if (!store->exists(store, SEGMENT_FILENAME))
         
     | 
| 
      
 643 
     | 
    
         
            +
                return 0;
         
     | 
| 
      
 644 
     | 
    
         
            +
              is = store->open_input(store, SEGMENT_FILENAME);
         
     | 
| 
      
 645 
     | 
    
         
            +
             
     | 
| 
       632 
646 
     | 
    
         
             
              TRY
         
     | 
| 
       633 
647 
     | 
    
         
             
                format = is_read_int(is);
         
     | 
| 
       634 
648 
     | 
    
         
             
                if (format < 0) {
         
     | 
| 
       635 
649 
     | 
    
         
             
                  if (format < FORMAT) 
         
     | 
| 
       636 
     | 
    
         
            -
                    RAISE( 
     | 
| 
       637 
     | 
    
         
            -
                  version = is_read_long(is);
         
     | 
| 
      
 650 
     | 
    
         
            +
                    RAISE(EXCEPTION, FORMAT_VERSION_ERROR_MSG);
         
     | 
| 
      
 651 
     | 
    
         
            +
                  version = (int)is_read_long(is);
         
     | 
| 
       638 
652 
     | 
    
         
             
                }
         
     | 
| 
       639 
653 
     | 
    
         
             
              XFINALLY
         
     | 
| 
       640 
654 
     | 
    
         
             
                is_close(is);
         
     | 
| 
         @@ -643,11 +657,11 @@ int sis_read_current_version(Store *store) 
     | 
|
| 
       643 
657 
     | 
    
         
             
              if (format < 0)
         
     | 
| 
       644 
658 
     | 
    
         
             
                return version;
         
     | 
| 
       645 
659 
     | 
    
         | 
| 
       646 
     | 
    
         
            -
               
     | 
| 
       647 
     | 
    
         
            -
             
     | 
| 
       648 
     | 
    
         
            -
             
     | 
| 
      
 660 
     | 
    
         
            +
              /* We cannot be sure about the format of the file.
         
     | 
| 
      
 661 
     | 
    
         
            +
               * Therefore we have to read the whole file and cannot simply
         
     | 
| 
      
 662 
     | 
    
         
            +
               * seek to the version entry. */
         
     | 
| 
       649 
663 
     | 
    
         | 
| 
       650 
     | 
    
         
            -
               
     | 
| 
      
 664 
     | 
    
         
            +
              sis = sis_create();
         
     | 
| 
       651 
665 
     | 
    
         
             
              sis_read(sis, store);
         
     | 
| 
       652 
666 
     | 
    
         
             
              version = sis->version;
         
     | 
| 
       653 
667 
     | 
    
         
             
              sis_destroy(sis);
         
     | 
| 
         @@ -660,8 +674,10 @@ int sis_read_current_version(Store *store) 
     | 
|
| 
       660 
674 
     | 
    
         
             
             *
         
     | 
| 
       661 
675 
     | 
    
         
             
             ****************************************************************************/
         
     | 
| 
       662 
676 
     | 
    
         | 
| 
       663 
     | 
    
         
            -
             
     | 
| 
       664 
     | 
    
         
            -
             
     | 
| 
      
 677 
     | 
    
         
            +
            /**
         
     | 
| 
      
 678 
     | 
    
         
            +
             * Deletes the analyzer by default but leaves the store by default
         
     | 
| 
      
 679 
     | 
    
         
            +
             */
         
     | 
| 
      
 680 
     | 
    
         
            +
            IndexWriter *iw_open(Store *store, Analyzer *analyzer, bool create)
         
     | 
| 
       665 
681 
     | 
    
         
             
            {
         
     | 
| 
       666 
682 
     | 
    
         
             
              IndexWriter *iw = ALLOC(IndexWriter);
         
     | 
| 
       667 
683 
     | 
    
         
             
              if (create)
         
     | 
| 
         @@ -674,15 +690,14 @@ IndexWriter *iw_open(Store *store, Analyzer *analyzer, 
     | 
|
| 
       674 
690 
     | 
    
         
             
              iw->term_index_interval = config.term_index_interval;
         
     | 
| 
       675 
691 
     | 
    
         
             
              iw->use_compound_file = true;
         
     | 
| 
       676 
692 
     | 
    
         
             
              iw->store = store;
         
     | 
| 
       677 
     | 
    
         
            -
               
     | 
| 
       678 
     | 
    
         
            -
              iw->close_analyzer = close_analyzer;
         
     | 
| 
      
 693 
     | 
    
         
            +
              ref(store);
         
     | 
| 
       679 
694 
     | 
    
         
             
              iw->analyzer = analyzer;
         
     | 
| 
       680 
695 
     | 
    
         
             
              iw->sis = sis_create();
         
     | 
| 
       681 
696 
     | 
    
         
             
              iw->similarity = sim_create_default();
         
     | 
| 
       682 
697 
     | 
    
         
             
              iw->ram_store = open_ram_store();
         
     | 
| 
       683 
698 
     | 
    
         | 
| 
       684 
699 
     | 
    
         
             
              mutex_lock(&store->mutex);
         
     | 
| 
       685 
     | 
    
         
            -
               
     | 
| 
      
 700 
     | 
    
         
            +
              /* keep the write_lock obtained until the IndexWriter is closed. */
         
     | 
| 
       686 
701 
     | 
    
         
             
              iw->write_lock = store->open_lock(store, WRITE_LOCK_NAME);
         
     | 
| 
       687 
702 
     | 
    
         
             
              if (!iw->write_lock->obtain(iw->write_lock)) {
         
     | 
| 
       688 
703 
     | 
    
         
             
                RAISE(STATE_ERROR, WRITE_LOCK_ERROR_MSG);
         
     | 
| 
         @@ -695,7 +710,7 @@ IndexWriter *iw_open(Store *store, Analyzer *analyzer, 
     | 
|
| 
       695 
710 
     | 
    
         
             
                  RAISE(STATE_ERROR, COMMIT_LOCK_ERROR_MSG);
         
     | 
| 
       696 
711 
     | 
    
         
             
                }
         
     | 
| 
       697 
712 
     | 
    
         
             
                TRY
         
     | 
| 
       698 
     | 
    
         
            -
                   
     | 
| 
      
 713 
     | 
    
         
            +
                  /* commit the index */
         
     | 
| 
       699 
714 
     | 
    
         
             
                  store->clear(store);
         
     | 
| 
       700 
715 
     | 
    
         
             
                  sis_write(iw->sis, store);
         
     | 
| 
       701 
716 
     | 
    
         
             
                XFINALLY
         
     | 
| 
         @@ -714,8 +729,9 @@ const char base36_digitmap[] = "0123456789abcdefghijklmnopqrstuvwxyz"; 
     | 
|
| 
       714 
729 
     | 
    
         
             
            char *new_segment_name(int counter) 
         
     | 
| 
       715 
730 
     | 
    
         
             
            {
         
     | 
| 
       716 
731 
     | 
    
         
             
              char buf[SEGMENT_NAME_MAX_LENGTH];
         
     | 
| 
       717 
     | 
    
         
            -
              buf[SEGMENT_NAME_MAX_LENGTH - 1] = '\0';
         
     | 
| 
       718 
732 
     | 
    
         
             
              int i;
         
     | 
| 
      
 733 
     | 
    
         
            +
             
     | 
| 
      
 734 
     | 
    
         
            +
              buf[SEGMENT_NAME_MAX_LENGTH - 1] = '\0';
         
     | 
| 
       719 
735 
     | 
    
         
             
              for (i = SEGMENT_NAME_MAX_LENGTH - 2; ; i--) {
         
     | 
| 
       720 
736 
     | 
    
         
             
                buf[i] = base36_digitmap[counter%36];
         
     | 
| 
       721 
737 
     | 
    
         
             
                counter /= 36;
         
     | 
| 
         @@ -749,8 +765,8 @@ void delete_files(Array *file_names, Store *store) 
     | 
|
| 
       749 
765 
     | 
    
         
             
            Array *sr_file_names(IndexReader *ir);
         
     | 
| 
       750 
766 
     | 
    
         
             
            void iw_delete_segments(IndexWriter *iw, IndexReader **segment_readers, int del_cnt)
         
     | 
| 
       751 
767 
     | 
    
         
             
            {
         
     | 
| 
       752 
     | 
    
         
            -
               
     | 
| 
       753 
     | 
    
         
            -
             
     | 
| 
      
 768 
     | 
    
         
            +
              /* The java version keeps a record of files that it couldn't delete. This
         
     | 
| 
      
 769 
     | 
    
         
            +
               * shouldn't be a problem on linux I hope. */
         
     | 
| 
       754 
770 
     | 
    
         
             
              IndexReader *ir;
         
     | 
| 
       755 
771 
     | 
    
         
             
              int i;
         
     | 
| 
       756 
772 
     | 
    
         
             
              for (i = 0; i < del_cnt; i++) {
         
     | 
| 
         @@ -761,22 +777,25 @@ void iw_delete_segments(IndexWriter *iw, IndexReader **segment_readers, int del_ 
     | 
|
| 
       761 
777 
     | 
    
         | 
| 
       762 
778 
     | 
    
         
             
            void make_compound_file(IndexWriter *iw, char *merged_name, SegmentMerger *merger)
         
     | 
| 
       763 
779 
     | 
    
         
             
            {
         
     | 
| 
      
 780 
     | 
    
         
            +
              Array *files_to_delete;
         
     | 
| 
      
 781 
     | 
    
         
            +
              Lock *commit_lock;
         
     | 
| 
       764 
782 
     | 
    
         
             
              char merged_tmp[SEGMENT_NAME_MAX_LENGTH], merged_cfs[SEGMENT_NAME_MAX_LENGTH];
         
     | 
| 
       765 
783 
     | 
    
         | 
| 
       766 
784 
     | 
    
         
             
              mutex_lock(&iw->store->mutex);
         
     | 
| 
       767 
785 
     | 
    
         
             
              sprintf(merged_tmp, "%s.tmp", merged_name);
         
     | 
| 
       768 
786 
     | 
    
         
             
              sprintf(merged_cfs, "%s.cfs", merged_name);
         
     | 
| 
       769 
787 
     | 
    
         | 
| 
       770 
     | 
    
         
            -
               
     | 
| 
       771 
     | 
    
         
            -
               
     | 
| 
      
 788 
     | 
    
         
            +
              files_to_delete = sm_create_compound_file(merger, merged_tmp);
         
     | 
| 
      
 789 
     | 
    
         
            +
              commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
         
     | 
| 
       772 
790 
     | 
    
         | 
| 
       773 
791 
     | 
    
         
             
              if (!commit_lock->obtain(commit_lock)) {
         
     | 
| 
       774 
792 
     | 
    
         
             
                RAISE(STATE_ERROR, COMMIT_LOCK_ERROR_MSG);
         
     | 
| 
       775 
793 
     | 
    
         
             
              }
         
     | 
| 
       776 
794 
     | 
    
         | 
| 
       777 
     | 
    
         
            -
               
     | 
| 
      
 795 
     | 
    
         
            +
              /* make compound file visible for SegmentReaders */
         
     | 
| 
       778 
796 
     | 
    
         
             
              iw->store->rename(iw->store, merged_tmp, merged_cfs);
         
     | 
| 
       779 
     | 
    
         
            -
             
     | 
| 
      
 797 
     | 
    
         
            +
             
     | 
| 
      
 798 
     | 
    
         
            +
              /* delete now unused files of segment */
         
     | 
| 
       780 
799 
     | 
    
         
             
              delete_files(files_to_delete, iw->store);
         
     | 
| 
       781 
800 
     | 
    
         | 
| 
       782 
801 
     | 
    
         
             
              commit_lock->release(commit_lock);
         
     | 
| 
         @@ -787,7 +806,9 @@ void make_compound_file(IndexWriter *iw, char *merged_name, SegmentMerger *merge 
     | 
|
| 
       787 
806 
     | 
    
         
             
            void iw_merge_segments_with_max(IndexWriter *iw, int min_segment, int max_segment)
         
     | 
| 
       788 
807 
     | 
    
         
             
            {
         
     | 
| 
       789 
808 
     | 
    
         
             
              int i;
         
     | 
| 
       790 
     | 
    
         
            -
               
     | 
| 
      
 809 
     | 
    
         
            +
              int merged_doc_count;
         
     | 
| 
      
 810 
     | 
    
         
            +
              Lock *commit_lock;
         
     | 
| 
      
 811 
     | 
    
         
            +
              IndexReader **segments_to_delete = ALLOC_N(IndexReader *, max_segment - min_segment);
         
     | 
| 
       791 
812 
     | 
    
         
             
              int del_cnt = 0;
         
     | 
| 
       792 
813 
     | 
    
         | 
| 
       793 
814 
     | 
    
         
             
              char *merged_name = new_segment_name(iw->sis->counter++);
         
     | 
| 
         @@ -797,31 +818,31 @@ void iw_merge_segments_with_max(IndexWriter *iw, int min_segment, int max_segmen 
     | 
|
| 
       797 
818 
     | 
    
         | 
| 
       798 
819 
     | 
    
         | 
| 
       799 
820 
     | 
    
         
             
              for (i = min_segment; i < max_segment; i++) {
         
     | 
| 
       800 
     | 
    
         
            -
                reader = sr_open(iw->sis, i, false 
     | 
| 
      
 821 
     | 
    
         
            +
                reader = sr_open(iw->sis, i, false);
         
     | 
| 
       801 
822 
     | 
    
         
             
                sm_add(merger, reader);
         
     | 
| 
       802 
     | 
    
         
            -
                if ((reader->store == iw->store) ||  
     | 
| 
      
 823 
     | 
    
         
            +
                if ((reader->store == iw->store) ||         /* if we own the directory */
         
     | 
| 
       803 
824 
     | 
    
         
             
                    (reader->store == iw->ram_store)) {
         
     | 
| 
       804 
     | 
    
         
            -
                  segments_to_delete[del_cnt++] = reader;    
     | 
| 
      
 825 
     | 
    
         
            +
                  segments_to_delete[del_cnt++] = reader;   /* queue segment for deletion */
         
     | 
| 
       805 
826 
     | 
    
         
             
                }
         
     | 
| 
       806 
827 
     | 
    
         
             
              }
         
     | 
| 
       807 
828 
     | 
    
         | 
| 
       808 
     | 
    
         
            -
               
     | 
| 
      
 829 
     | 
    
         
            +
              merged_doc_count = sm_merge(merger);
         
     | 
| 
       809 
830 
     | 
    
         | 
| 
       810 
831 
     | 
    
         
             
              sis_del_from_to(iw->sis, min_segment, max_segment);
         
     | 
| 
       811 
832 
     | 
    
         | 
| 
       812 
833 
     | 
    
         
             
              sis_add_si(iw->sis, si_create(merged_name, merged_doc_count, iw->store));
         
     | 
| 
       813 
834 
     | 
    
         | 
| 
       814 
     | 
    
         
            -
               
     | 
| 
      
 835 
     | 
    
         
            +
              /* close readers before we attempt to delete now-obsolete segments */
         
     | 
| 
       815 
836 
     | 
    
         | 
| 
       816 
837 
     | 
    
         
             
              mutex_lock(&iw->store->mutex);
         
     | 
| 
       817 
     | 
    
         
            -
               
     | 
| 
      
 838 
     | 
    
         
            +
              commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
         
     | 
| 
       818 
839 
     | 
    
         
             
              if (!commit_lock->obtain(commit_lock)) {
         
     | 
| 
       819 
840 
     | 
    
         
             
                RAISE(STATE_ERROR, COMMIT_LOCK_ERROR_MSG);
         
     | 
| 
       820 
841 
     | 
    
         
             
              }
         
     | 
| 
       821 
     | 
    
         
            -
               
     | 
| 
      
 842 
     | 
    
         
            +
              /* commit the index */
         
     | 
| 
       822 
843 
     | 
    
         
             
              sis_write(iw->sis, iw->store);
         
     | 
| 
       823 
844 
     | 
    
         
             
              iw_delete_segments(iw, segments_to_delete, del_cnt);
         
     | 
| 
       824 
     | 
    
         
            -
             
     | 
| 
      
 845 
     | 
    
         
            +
             
     | 
| 
       825 
846 
     | 
    
         
             
              commit_lock->release(commit_lock);
         
     | 
| 
       826 
847 
     | 
    
         
             
              iw->store->close_lock(commit_lock);
         
     | 
| 
       827 
848 
     | 
    
         
             
              mutex_unlock(&iw->store->mutex);
         
     | 
| 
         @@ -830,6 +851,7 @@ void iw_merge_segments_with_max(IndexWriter *iw, int min_segment, int max_segmen 
     | 
|
| 
       830 
851 
     | 
    
         
             
                make_compound_file(iw, merged_name, merger);
         
     | 
| 
       831 
852 
     | 
    
         
             
              }
         
     | 
| 
       832 
853 
     | 
    
         | 
| 
      
 854 
     | 
    
         
            +
              free(segments_to_delete);
         
     | 
| 
       833 
855 
     | 
    
         
             
              sm_destroy(merger);
         
     | 
| 
       834 
856 
     | 
    
         
             
            }
         
     | 
| 
       835 
857 
     | 
    
         | 
| 
         @@ -845,23 +867,25 @@ void iw_maybe_merge_segments(IndexWriter *iw) 
     | 
|
| 
       845 
867 
     | 
    
         
             
              SegmentInfo *si;
         
     | 
| 
       846 
868 
     | 
    
         | 
| 
       847 
869 
     | 
    
         
             
              while (target_merge_docs <= iw->max_merge_docs) {
         
     | 
| 
       848 
     | 
    
         
            -
                 
     | 
| 
      
 870 
     | 
    
         
            +
                /* find segments smaller than current target size */
         
     | 
| 
       849 
871 
     | 
    
         
             
                min_segment = iw->sis->scnt - 1;
         
     | 
| 
       850 
872 
     | 
    
         
             
                merge_docs = 0;
         
     | 
| 
       851 
873 
     | 
    
         
             
                while (min_segment >= 0) {
         
     | 
| 
       852 
874 
     | 
    
         
             
                  si = iw->sis->segs[min_segment];
         
     | 
| 
       853 
     | 
    
         
            -
                  if (si->doc_cnt >= target_merge_docs)
         
     | 
| 
      
 875 
     | 
    
         
            +
                  if (si->doc_cnt >= target_merge_docs) {
         
     | 
| 
       854 
876 
     | 
    
         
             
                    break;
         
     | 
| 
      
 877 
     | 
    
         
            +
                  }
         
     | 
| 
       855 
878 
     | 
    
         
             
                  merge_docs += si->doc_cnt;
         
     | 
| 
       856 
879 
     | 
    
         
             
                  min_segment -= 1;
         
     | 
| 
       857 
880 
     | 
    
         
             
                }
         
     | 
| 
       858 
881 
     | 
    
         | 
| 
       859 
     | 
    
         
            -
                if (merge_docs >= target_merge_docs) 
     | 
| 
      
 882 
     | 
    
         
            +
                if (merge_docs >= target_merge_docs) { /* found a merge to do */
         
     | 
| 
       860 
883 
     | 
    
         
             
                  iw_merge_segments(iw, min_segment + 1);
         
     | 
| 
       861 
     | 
    
         
            -
                else
         
     | 
| 
      
 884 
     | 
    
         
            +
                } else {
         
     | 
| 
       862 
885 
     | 
    
         
             
                  break;
         
     | 
| 
      
 886 
     | 
    
         
            +
                }
         
     | 
| 
       863 
887 
     | 
    
         | 
| 
       864 
     | 
    
         
            -
                target_merge_docs *= iw->merge_factor;  
     | 
| 
      
 888 
     | 
    
         
            +
                target_merge_docs *= iw->merge_factor; /* increase target size */
         
     | 
| 
       865 
889 
     | 
    
         
             
              }
         
     | 
| 
       866 
890 
     | 
    
         
             
            }
         
     | 
| 
       867 
891 
     | 
    
         | 
| 
         @@ -883,12 +907,14 @@ void iw_flush_ram_segments(IndexWriter *iw) 
     | 
|
| 
       883 
907 
     | 
    
         
             
               * that wasn't the ram segment. But if it fit's in with the merge
         
     | 
| 
       884 
908 
     | 
    
         
             
               * factor, why not merge it. Otherwise we leave it and increment min_seg
         
     | 
| 
       885 
909 
     | 
    
         
             
               */
         
     | 
| 
       886 
     | 
    
         
            -
              if (min_segment < 0 || 
     | 
| 
       887 
     | 
    
         
            -
                  (doc_count + segs[min_segment]->doc_cnt) > iw->merge_factor ||
         
     | 
| 
       888 
     | 
    
         
            -
                  (segs[iw->sis->scnt-1]->store != iw->ram_store))
         
     | 
| 
      
 910 
     | 
    
         
            +
              if ((min_segment < 0) ||                       /* add one FS segment? */
         
     | 
| 
      
 911 
     | 
    
         
            +
                  ((doc_count + segs[min_segment]->doc_cnt) > iw->merge_factor) ||
         
     | 
| 
      
 912 
     | 
    
         
            +
                  (segs[iw->sis->scnt - 1]->store != iw->ram_store)) {
         
     | 
| 
       889 
913 
     | 
    
         
             
                min_segment++;
         
     | 
| 
       890 
     | 
    
         
            -
               
     | 
| 
      
 914 
     | 
    
         
            +
              }
         
     | 
| 
      
 915 
     | 
    
         
            +
              if (min_segment >= iw->sis->scnt) {
         
     | 
| 
       891 
916 
     | 
    
         
             
                return;
         
     | 
| 
      
 917 
     | 
    
         
            +
              }
         
     | 
| 
       892 
918 
     | 
    
         
             
              iw_merge_segments(iw, min_segment);
         
     | 
| 
       893 
919 
     | 
    
         
             
            }
         
     | 
| 
       894 
920 
     | 
    
         | 
| 
         @@ -937,17 +963,16 @@ void iw_close(IndexWriter *iw) 
     | 
|
| 
       937 
963 
     | 
    
         
             
            {
         
     | 
| 
       938 
964 
     | 
    
         
             
              mutex_lock(&iw->mutex);
         
     | 
| 
       939 
965 
     | 
    
         
             
              iw_flush_ram_segments(iw);
         
     | 
| 
       940 
     | 
    
         
            -
               
     | 
| 
      
 966 
     | 
    
         
            +
              store_deref(iw->ram_store);
         
     | 
| 
       941 
967 
     | 
    
         
             
              sis_destroy(iw->sis);
         
     | 
| 
       942 
968 
     | 
    
         | 
| 
       943 
969 
     | 
    
         
             
              sim_destroy(iw->similarity);
         
     | 
| 
       944 
     | 
    
         
            -
               
     | 
| 
      
 970 
     | 
    
         
            +
              a_deref(iw->analyzer);
         
     | 
| 
       945 
971 
     | 
    
         | 
| 
       946 
972 
     | 
    
         
             
              iw->write_lock->release(iw->write_lock);
         
     | 
| 
       947 
973 
     | 
    
         
             
              iw->store->close_lock(iw->write_lock);
         
     | 
| 
       948 
974 
     | 
    
         | 
| 
       949 
     | 
    
         
            -
               
     | 
| 
       950 
     | 
    
         
            -
                store_close(iw->store);
         
     | 
| 
      
 975 
     | 
    
         
            +
              store_deref(iw->store);
         
     | 
| 
       951 
976 
     | 
    
         
             
              mutex_destroy(&iw->mutex);
         
     | 
| 
       952 
977 
     | 
    
         
             
              free(iw);
         
     | 
| 
       953 
978 
     | 
    
         
             
            }
         
     | 
| 
         @@ -957,13 +982,13 @@ void iw_add_indexes(IndexWriter *iw, Store **stores, int cnt) 
     | 
|
| 
       957 
982 
     | 
    
         
             
              int i, j, end, start;
         
     | 
| 
       958 
983 
     | 
    
         | 
| 
       959 
984 
     | 
    
         
             
              mutex_lock(&iw->mutex);
         
     | 
| 
       960 
     | 
    
         
            -
              iw_optimize_internal(iw);  
     | 
| 
      
 985 
     | 
    
         
            +
              iw_optimize_internal(iw); /* start with zero or 1 seg */
         
     | 
| 
       961 
986 
     | 
    
         | 
| 
       962 
987 
     | 
    
         
             
              start = iw->sis->scnt;
         
     | 
| 
       963 
988 
     | 
    
         | 
| 
       964 
989 
     | 
    
         
             
              for (i = 0; i < cnt; i++) {
         
     | 
| 
       965 
990 
     | 
    
         
             
                Store *store = stores[i];
         
     | 
| 
       966 
     | 
    
         
            -
                SegmentInfos *sis = sis_create();  
     | 
| 
      
 991 
     | 
    
         
            +
                SegmentInfos *sis = sis_create(); /* read infos from dir */
         
     | 
| 
       967 
992 
     | 
    
         
             
                sis_read(sis, store);
         
     | 
| 
       968 
993 
     | 
    
         | 
| 
       969 
994 
     | 
    
         
             
                for (j = 0; j < sis->scnt; j++) {
         
     | 
| 
         @@ -973,7 +998,7 @@ void iw_add_indexes(IndexWriter *iw, Store **stores, int cnt) 
     | 
|
| 
       973 
998 
     | 
    
         
             
                sis_destroy_not_infos(sis);
         
     | 
| 
       974 
999 
     | 
    
         
             
              }
         
     | 
| 
       975 
1000 
     | 
    
         | 
| 
       976 
     | 
    
         
            -
               
     | 
| 
      
 1001 
     | 
    
         
            +
              /* merge newly added segments in log(n) passes */
         
     | 
| 
       977 
1002 
     | 
    
         
             
              while (iw->sis->scnt > start + iw->merge_factor) {
         
     | 
| 
       978 
1003 
     | 
    
         
             
                for (i = start + 1; i < iw->sis->scnt; i++) {
         
     | 
| 
       979 
1004 
     | 
    
         
             
                  end = MIN(iw->sis->scnt, i + iw->merge_factor);
         
     | 
| 
         @@ -983,7 +1008,7 @@ void iw_add_indexes(IndexWriter *iw, Store **stores, int cnt) 
     | 
|
| 
       983 
1008 
     | 
    
         
             
                }
         
     | 
| 
       984 
1009 
     | 
    
         
             
              }
         
     | 
| 
       985 
1010 
     | 
    
         | 
| 
       986 
     | 
    
         
            -
               
     | 
| 
      
 1011 
     | 
    
         
            +
              /* final cleanup */
         
     | 
| 
       987 
1012 
     | 
    
         
             
              iw_optimize_internal(iw);
         
     | 
| 
       988 
1013 
     | 
    
         
             
              mutex_unlock(&iw->mutex);
         
     | 
| 
       989 
1014 
     | 
    
         
             
            }
         
     | 
| 
         @@ -996,16 +1021,20 @@ void iw_add_readers(IndexWriter *iw, IndexReader **irs, int cnt) 
     | 
|
| 
       996 
1021 
     | 
    
         
             
            {
         
     | 
| 
       997 
1022 
     | 
    
         
             
              IndexReader *ir = NULL;
         
     | 
| 
       998 
1023 
     | 
    
         
             
              int i, del_cnt = 0;
         
     | 
| 
       999 
     | 
    
         
            -
             
     | 
| 
      
 1024 
     | 
    
         
            +
              int doc_count;
         
     | 
| 
      
 1025 
     | 
    
         
            +
              char *merged_name;
         
     | 
| 
      
 1026 
     | 
    
         
            +
              SegmentMerger *merger;
         
     | 
| 
      
 1027 
     | 
    
         
            +
              Lock *commit_lock;
         
     | 
| 
      
 1028 
     | 
    
         
            +
              
         
     | 
| 
       1000 
1029 
     | 
    
         
             
              mutex_lock(&iw->mutex);
         
     | 
| 
       1001 
     | 
    
         
            -
              iw_optimize_internal(iw);  
     | 
| 
      
 1030 
     | 
    
         
            +
              iw_optimize_internal(iw); /* start with zero or 1 seg */
         
     | 
| 
       1002 
1031 
     | 
    
         | 
| 
       1003 
     | 
    
         
            -
               
     | 
| 
      
 1032 
     | 
    
         
            +
              merged_name = new_segment_name(iw->sis->counter++);
         
     | 
| 
       1004 
1033 
     | 
    
         | 
| 
       1005 
     | 
    
         
            -
               
     | 
| 
       1006 
     | 
    
         
            -
              merger->readers->free_elem = NULL;  
     | 
| 
      
 1034 
     | 
    
         
            +
              merger = sm_create(iw->store, merged_name, iw->term_index_interval);
         
     | 
| 
      
 1035 
     | 
    
         
            +
              merger->readers->free_elem = NULL; /* don't close readers */
         
     | 
| 
       1007 
1036 
     | 
    
         | 
| 
       1008 
     | 
    
         
            -
              if (iw->sis->scnt == 1) { 
     | 
| 
      
 1037 
     | 
    
         
            +
              if (iw->sis->scnt == 1) { /* add existing index, if any */
         
     | 
| 
       1009 
1038 
     | 
    
         
             
                ir = sr_open_si(iw->sis->segs[0]);
         
     | 
| 
       1010 
1039 
     | 
    
         
             
                sm_add(merger, ir);
         
     | 
| 
       1011 
1040 
     | 
    
         
             
                del_cnt = 1;
         
     | 
| 
         @@ -1015,18 +1044,19 @@ void iw_add_readers(IndexWriter *iw, IndexReader **irs, int cnt) 
     | 
|
| 
       1015 
1044 
     | 
    
         
             
                sm_add(merger, irs[i]);
         
     | 
| 
       1016 
1045 
     | 
    
         
             
              }
         
     | 
| 
       1017 
1046 
     | 
    
         | 
| 
       1018 
     | 
    
         
            -
               
     | 
| 
      
 1047 
     | 
    
         
            +
              doc_count = sm_merge(merger); /* merge 'em */
         
     | 
| 
       1019 
1048 
     | 
    
         | 
| 
       1020 
     | 
    
         
            -
               
     | 
| 
      
 1049 
     | 
    
         
            +
              /* pop old infos and add new ones. */
         
     | 
| 
       1021 
1050 
     | 
    
         
             
              sis_clear(iw->sis);
         
     | 
| 
       1022 
1051 
     | 
    
         
             
              sis_add_si(iw->sis, si_create(merged_name, doc_count, iw->store));
         
     | 
| 
       1023 
1052 
     | 
    
         | 
| 
       1024 
1053 
     | 
    
         | 
| 
       1025 
     | 
    
         
            -
               
     | 
| 
       1026 
     | 
    
         
            -
              if (!commit_lock->obtain(commit_lock))  
     | 
| 
      
 1054 
     | 
    
         
            +
              commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
         
     | 
| 
      
 1055 
     | 
    
         
            +
              if (!commit_lock->obtain(commit_lock)) { /* obtain write lock */
         
     | 
| 
       1027 
1056 
     | 
    
         
             
                RAISE(STATE_ERROR, COMMIT_LOCK_ERROR_MSG);
         
     | 
| 
      
 1057 
     | 
    
         
            +
              }
         
     | 
| 
       1028 
1058 
     | 
    
         | 
| 
       1029 
     | 
    
         
            -
              sis_write(iw->sis, iw->store);  
     | 
| 
      
 1059 
     | 
    
         
            +
              sis_write(iw->sis, iw->store); /* commit changes */
         
     | 
| 
       1030 
1060 
     | 
    
         
             
              iw_delete_segments(iw, &ir, del_cnt);
         
     | 
| 
       1031 
1061 
     | 
    
         
             
              if (ir) ir_close(ir);
         
     | 
| 
       1032 
1062 
     | 
    
         | 
| 
         @@ -1059,24 +1089,28 @@ Norm *norm_create(InStream *is, int field_num) 
     | 
|
| 
       1059 
1089 
     | 
    
         
             
              return norm;
         
     | 
| 
       1060 
1090 
     | 
    
         
             
            }
         
     | 
| 
       1061 
1091 
     | 
    
         | 
| 
       1062 
     | 
    
         
            -
            void norm_destroy( 
     | 
| 
      
 1092 
     | 
    
         
            +
            void norm_destroy(Norm *norm)
         
     | 
| 
       1063 
1093 
     | 
    
         
             
            {
         
     | 
| 
       1064 
     | 
    
         
            -
              Norm *norm = (Norm *)p;
         
     | 
| 
       1065 
1094 
     | 
    
         
             
              is_close(norm->is);
         
     | 
| 
       1066 
     | 
    
         
            -
              if (norm->bytes != NULL)  
     | 
| 
      
 1095 
     | 
    
         
            +
              if (norm->bytes != NULL) {
         
     | 
| 
      
 1096 
     | 
    
         
            +
                free(norm->bytes);
         
     | 
| 
      
 1097 
     | 
    
         
            +
              }
         
     | 
| 
       1067 
1098 
     | 
    
         
             
              free(norm);
         
     | 
| 
       1068 
1099 
     | 
    
         
             
            }
         
     | 
| 
       1069 
1100 
     | 
    
         | 
| 
       1070 
1101 
     | 
    
         
             
            void norm_rewrite(Norm *norm, Store *store, char *segment,
         
     | 
| 
       1071 
1102 
     | 
    
         
             
                int doc_count, Store *cfs_store)
         
     | 
| 
       1072 
1103 
     | 
    
         
             
            {
         
     | 
| 
       1073 
     | 
    
         
            -
               
     | 
| 
       1074 
     | 
    
         
            -
                return; // These norms do not need to be rewritten
         
     | 
| 
       1075 
     | 
    
         
            -
             
     | 
| 
      
 1104 
     | 
    
         
            +
              OutStream *os;
         
     | 
| 
       1076 
1105 
     | 
    
         
             
              char tmp_fname[SEGMENT_NAME_MAX_LENGTH];
         
     | 
| 
       1077 
1106 
     | 
    
         
             
              char norm_fname[SEGMENT_NAME_MAX_LENGTH];
         
     | 
| 
      
 1107 
     | 
    
         
            +
             
     | 
| 
      
 1108 
     | 
    
         
            +
              if (norm->bytes == NULL) {
         
     | 
| 
      
 1109 
     | 
    
         
            +
                return; /* These norms do not need to be rewritten */
         
     | 
| 
      
 1110 
     | 
    
         
            +
              }
         
     | 
| 
      
 1111 
     | 
    
         
            +
             
     | 
| 
       1078 
1112 
     | 
    
         
             
              sprintf(tmp_fname, "%s.tmp", segment);
         
     | 
| 
       1079 
     | 
    
         
            -
               
     | 
| 
      
 1113 
     | 
    
         
            +
              os = store->create_output(store, tmp_fname);
         
     | 
| 
       1080 
1114 
     | 
    
         
             
              TRY
         
     | 
| 
       1081 
1115 
     | 
    
         
             
                os_write_bytes(os, norm->bytes, doc_count);
         
     | 
| 
       1082 
1116 
     | 
    
         
             
              XFINALLY
         
     | 
| 
         @@ -1097,7 +1131,7 @@ void norm_rewrite(Norm *norm, Store *store, char *segment, 
     | 
|
| 
       1097 
1131 
     | 
    
         
             
             *
         
     | 
| 
       1098 
1132 
     | 
    
         
             
             ****************************************************************************/
         
     | 
| 
       1099 
1133 
     | 
    
         | 
| 
       1100 
     | 
    
         
            -
            #define GET_SR SegmentReader *sr = (SegmentReader *)ir->data 
     | 
| 
      
 1134 
     | 
    
         
            +
            #define GET_SR SegmentReader *sr = (SegmentReader *)ir->data
         
     | 
| 
       1101 
1135 
     | 
    
         | 
| 
       1102 
1136 
     | 
    
         
             
            int sr_max_doc(IndexReader *ir)
         
     | 
| 
       1103 
1137 
     | 
    
         
             
            {
         
     | 
| 
         @@ -1129,8 +1163,8 @@ void sr_close(IndexReader *ir) 
     | 
|
| 
       1129 
1163 
     | 
    
         | 
| 
       1130 
1164 
     | 
    
         
             
              if (sr->freq_in) is_close(sr->freq_in);
         
     | 
| 
       1131 
1165 
     | 
    
         
             
              if (sr->prox_in) is_close(sr->prox_in);
         
     | 
| 
      
 1166 
     | 
    
         
            +
              
         
     | 
| 
       1132 
1167 
     | 
    
         
             
              fis_destroy(sr->fis);
         
     | 
| 
       1133 
     | 
    
         
            -
             
     | 
| 
       1134 
1168 
     | 
    
         
             
              sr_close_norms(sr);
         
     | 
| 
       1135 
1169 
     | 
    
         | 
| 
       1136 
1170 
     | 
    
         
             
              if (sr->orig_tvr) {
         
     | 
| 
         @@ -1139,7 +1173,7 @@ void sr_close(IndexReader *ir) 
     | 
|
| 
       1139 
1173 
     | 
    
         
             
                ary_destroy(sr->tvr_bucket);
         
     | 
| 
       1140 
1174 
     | 
    
         
             
              }
         
     | 
| 
       1141 
1175 
     | 
    
         
             
              if (sr->deleted_docs) bv_destroy(sr->deleted_docs);
         
     | 
| 
       1142 
     | 
    
         
            -
              if (sr->cfs_store)  
     | 
| 
      
 1176 
     | 
    
         
            +
              if (sr->cfs_store) store_deref(sr->cfs_store);
         
     | 
| 
       1143 
1177 
     | 
    
         
             
              if (sr->fake_norms) free(sr->fake_norms);
         
     | 
| 
       1144 
1178 
     | 
    
         
             
              free(sr->segment);
         
     | 
| 
       1145 
1179 
     | 
    
         
             
              free(sr);
         
     | 
| 
         @@ -1175,8 +1209,8 @@ bool sr_is_deleted(IndexReader *ir, int doc_num) 
     | 
|
| 
       1175 
1209 
     | 
    
         | 
| 
       1176 
1210 
     | 
    
         
             
            bool sr_has_norms(IndexReader *ir, char *field)
         
     | 
| 
       1177 
1211 
     | 
    
         
             
            {
         
     | 
| 
       1178 
     | 
    
         
            -
              bool has_norms;
         
     | 
| 
       1179 
1212 
     | 
    
         
             
              GET_SR;
         
     | 
| 
      
 1213 
     | 
    
         
            +
              bool has_norms;
         
     | 
| 
       1180 
1214 
     | 
    
         
             
              mutex_lock(&ir->mutex);
         
     | 
| 
       1181 
1215 
     | 
    
         
             
              has_norms = h_has_key(sr->norms, field);
         
     | 
| 
       1182 
1216 
     | 
    
         
             
              mutex_unlock(&ir->mutex);
         
     | 
| 
         @@ -1215,13 +1249,13 @@ TermEnum *sr_terms_from(IndexReader *ir, Term *term) 
     | 
|
| 
       1215 
1249 
     | 
    
         | 
| 
       1216 
1250 
     | 
    
         
             
            Document *sr_get_doc(IndexReader *ir, int doc_num)
         
     | 
| 
       1217 
1251 
     | 
    
         
             
            {
         
     | 
| 
      
 1252 
     | 
    
         
            +
              GET_SR;
         
     | 
| 
       1218 
1253 
     | 
    
         
             
              Document *doc;
         
     | 
| 
       1219 
1254 
     | 
    
         
             
              mutex_lock(&ir->mutex);
         
     | 
| 
       1220 
1255 
     | 
    
         
             
              if (sr_is_deleted_internal(ir, doc_num)) {
         
     | 
| 
       1221 
1256 
     | 
    
         
             
                mutex_unlock(&ir->mutex);
         
     | 
| 
       1222 
1257 
     | 
    
         
             
                RAISE(STATE_ERROR, DELETED_DOC_ERROR_MSG);
         
     | 
| 
       1223 
1258 
     | 
    
         
             
              }
         
     | 
| 
       1224 
     | 
    
         
            -
              GET_SR;
         
     | 
| 
       1225 
1259 
     | 
    
         
             
              doc = fr_get_doc(sr->fr, doc_num);
         
     | 
| 
       1226 
1260 
     | 
    
         
             
              mutex_unlock(&ir->mutex);
         
     | 
| 
       1227 
1261 
     | 
    
         
             
              return doc;
         
     | 
| 
         @@ -1234,11 +1268,11 @@ sr_get_norms_into_internal(IndexReader *ir, char *field, uchar *buf, int offset) 
     | 
|
| 
       1234 
1268 
     | 
    
         
             
              Norm *norm = h_get(sr->norms, field);
         
     | 
| 
       1235 
1269 
     | 
    
         
             
              if (norm == NULL) {
         
     | 
| 
       1236 
1270 
     | 
    
         
             
                memset(buf + offset*sizeof(uchar), 0, sr_max_doc(ir)*sizeof(uchar));
         
     | 
| 
       1237 
     | 
    
         
            -
              } else if (norm->bytes != NULL) {  
     | 
| 
      
 1271 
     | 
    
         
            +
              } else if (norm->bytes != NULL) { /* can copy from cache */
         
     | 
| 
       1238 
1272 
     | 
    
         
             
                memcpy(buf + offset*sizeof(uchar), norm->bytes, sr_max_doc(ir)*sizeof(uchar));
         
     | 
| 
       1239 
1273 
     | 
    
         
             
              } else {
         
     | 
| 
       1240 
1274 
     | 
    
         
             
                InStream *norm_in = is_clone(norm->is);
         
     | 
| 
       1241 
     | 
    
         
            -
                 
     | 
| 
      
 1275 
     | 
    
         
            +
                /* read from disk */
         
     | 
| 
       1242 
1276 
     | 
    
         
             
                is_seek(norm_in, 0);
         
     | 
| 
       1243 
1277 
     | 
    
         
             
                is_read_bytes(norm_in, buf, offset, sr_max_doc(ir));
         
     | 
| 
       1244 
1278 
     | 
    
         
             
                is_close(norm_in);
         
     | 
| 
         @@ -1256,13 +1290,14 @@ static inline uchar *sr_get_norms_internal(IndexReader *ir, char *field) 
     | 
|
| 
       1256 
1290 
     | 
    
         
             
            {
         
     | 
| 
       1257 
1291 
     | 
    
         
             
              GET_SR;
         
     | 
| 
       1258 
1292 
     | 
    
         
             
              Norm *norm = h_get(sr->norms, field);
         
     | 
| 
       1259 
     | 
    
         
            -
              if (norm == NULL) 
     | 
| 
      
 1293 
     | 
    
         
            +
              if (norm == NULL) {                           /* not an indexed field */
         
     | 
| 
       1260 
1294 
     | 
    
         
             
                return NULL;
         
     | 
| 
      
 1295 
     | 
    
         
            +
              }
         
     | 
| 
       1261 
1296 
     | 
    
         | 
| 
       1262 
     | 
    
         
            -
              if (norm->bytes == NULL) {                     
     | 
| 
      
 1297 
     | 
    
         
            +
              if (norm->bytes == NULL) {                    /* value not yet read */
         
     | 
| 
       1263 
1298 
     | 
    
         
             
                uchar *bytes = ALLOC_N(uchar, ir->max_doc(ir));
         
     | 
| 
       1264 
1299 
     | 
    
         
             
                sr_get_norms_into_internal(ir, field, bytes, 0);
         
     | 
| 
       1265 
     | 
    
         
            -
                norm->bytes = bytes;                         
     | 
| 
      
 1300 
     | 
    
         
            +
                norm->bytes = bytes;                        /* cache it */
         
     | 
| 
       1266 
1301 
     | 
    
         
             
              }
         
     | 
| 
       1267 
1302 
     | 
    
         
             
              return norm->bytes;
         
     | 
| 
       1268 
1303 
     | 
    
         
             
            }
         
     | 
| 
         @@ -1278,8 +1313,8 @@ uchar *sr_get_norms(IndexReader *ir, char *field) 
     | 
|
| 
       1278 
1313 
     | 
    
         | 
| 
       1279 
1314 
     | 
    
         
             
            static inline uchar *sr_get_norms_always(IndexReader *ir, char *field)
         
     | 
| 
       1280 
1315 
     | 
    
         
             
            {
         
     | 
| 
       1281 
     | 
    
         
            -
              uchar *bytes;
         
     | 
| 
       1282 
1316 
     | 
    
         
             
              GET_SR;
         
     | 
| 
      
 1317 
     | 
    
         
            +
              uchar *bytes;
         
     | 
| 
       1283 
1318 
     | 
    
         
             
              mutex_lock(&ir->mutex);
         
     | 
| 
       1284 
1319 
     | 
    
         | 
| 
       1285 
1320 
     | 
    
         
             
              bytes = sr_get_norms_internal(ir, field);
         
     | 
| 
         @@ -1303,7 +1338,7 @@ void sr_set_norm(IndexReader *ir, int doc_num, char *field, uchar val) 
     | 
|
| 
       1303 
1338 
     | 
    
         | 
| 
       1304 
1339 
     | 
    
         
             
              norm = h_get(sr->norms, field);
         
     | 
| 
       1305 
1340 
     | 
    
         
             
              if (norm != NULL) { /* an indexed field */
         
     | 
| 
       1306 
     | 
    
         
            -
                norm->is_dirty = true;                            
     | 
| 
      
 1341 
     | 
    
         
            +
                norm->is_dirty = true;                           /* mark it dirty */
         
     | 
| 
       1307 
1342 
     | 
    
         
             
                sr->norms_dirty = true;
         
     | 
| 
       1308 
1343 
     | 
    
         | 
| 
       1309 
1344 
     | 
    
         
             
                sr_get_norms_internal(ir, field)[doc_num] = val;
         
     | 
| 
         @@ -1318,13 +1353,15 @@ int sr_doc_freq(IndexReader *ir, Term *t) 
     | 
|
| 
       1318 
1353 
     | 
    
         
             
                int df = ti->doc_freq;
         
     | 
| 
       1319 
1354 
     | 
    
         
             
                ti_destroy(ti);
         
     | 
| 
       1320 
1355 
     | 
    
         
             
                return df;
         
     | 
| 
       1321 
     | 
    
         
            -
              } else  
     | 
| 
      
 1356 
     | 
    
         
            +
              } else {
         
     | 
| 
      
 1357 
     | 
    
         
            +
                return 0;
         
     | 
| 
      
 1358 
     | 
    
         
            +
              }
         
     | 
| 
       1322 
1359 
     | 
    
         
             
            }
         
     | 
| 
       1323 
1360 
     | 
    
         | 
| 
       1324 
1361 
     | 
    
         
             
            Array *sr_file_names(IndexReader *ir)
         
     | 
| 
       1325 
1362 
     | 
    
         
             
            {
         
     | 
| 
       1326 
1363 
     | 
    
         
             
              GET_SR;
         
     | 
| 
       1327 
     | 
    
         
            -
              Array *file_names = ary_create(0, & 
     | 
| 
      
 1364 
     | 
    
         
            +
              Array *file_names = ary_create(0, &free);
         
     | 
| 
       1328 
1365 
     | 
    
         
             
              FieldInfo *fi;
         
     | 
| 
       1329 
1366 
     | 
    
         
             
              int i;
         
     | 
| 
       1330 
1367 
     | 
    
         
             
              char fname[SEGMENT_NAME_MAX_LENGTH];
         
     | 
| 
         @@ -1352,8 +1389,8 @@ Array *sr_file_names(IndexReader *ir) 
     | 
|
| 
       1352 
1389 
     | 
    
         | 
| 
       1353 
1390 
     | 
    
         
             
            HashSet *sr_get_field_names(IndexReader *ir, int field_type)
         
     | 
| 
       1354 
1391 
     | 
    
         
             
            {
         
     | 
| 
       1355 
     | 
    
         
            -
              int i;
         
     | 
| 
       1356 
1392 
     | 
    
         
             
              GET_SR;
         
     | 
| 
      
 1393 
     | 
    
         
            +
              int i;
         
     | 
| 
       1357 
1394 
     | 
    
         
             
              HashSet *field_set = hs_str_create(NULL);
         
     | 
| 
       1358 
1395 
     | 
    
         
             
              FieldInfo *fi;
         
     | 
| 
       1359 
1396 
     | 
    
         
             
              for (i = 0; i < sr->fis->fcnt; i++) {
         
     | 
| 
         @@ -1396,9 +1433,10 @@ HashSet *sr_get_field_names(IndexReader *ir, int field_type) 
     | 
|
| 
       1396 
1433 
     | 
    
         
             
            int sr_num_docs(IndexReader *ir)
         
     | 
| 
       1397 
1434 
     | 
    
         
             
            {
         
     | 
| 
       1398 
1435 
     | 
    
         
             
              GET_SR;
         
     | 
| 
       1399 
     | 
    
         
            -
             
     | 
| 
      
 1436 
     | 
    
         
            +
              int num_docs;
         
     | 
| 
      
 1437 
     | 
    
         
            +
              
         
     | 
| 
       1400 
1438 
     | 
    
         
             
              mutex_lock(&ir->mutex);
         
     | 
| 
       1401 
     | 
    
         
            -
               
     | 
| 
      
 1439 
     | 
    
         
            +
              num_docs = sr_max_doc(ir);
         
     | 
| 
       1402 
1440 
     | 
    
         
             
              if (sr->deleted_docs != NULL)
         
     | 
| 
       1403 
1441 
     | 
    
         
             
                num_docs -= sr->deleted_docs->count;
         
     | 
| 
       1404 
1442 
     | 
    
         
             
              mutex_unlock(&ir->mutex);
         
     | 
| 
         @@ -1444,8 +1482,9 @@ TermVector *sr_get_term_vector(IndexReader *ir, int doc_num, char *field) 
     | 
|
| 
       1444 
1482 
     | 
    
         
             
              FieldInfo *fi = (FieldInfo *)ht_get(sr->fis->by_name, field);
         
     | 
| 
       1445 
1483 
     | 
    
         
             
              TermVectorsReader *tvr;
         
     | 
| 
       1446 
1484 
     | 
    
         | 
| 
       1447 
     | 
    
         
            -
              if (fi == NULL || !fi->store_tv || !sr->orig_tvr || !(tvr = sr_tvr(sr)))
         
     | 
| 
      
 1485 
     | 
    
         
            +
              if (fi == NULL || !fi->store_tv || !sr->orig_tvr || !(tvr = sr_tvr(sr))) {
         
     | 
| 
       1448 
1486 
     | 
    
         
             
                return NULL;
         
     | 
| 
      
 1487 
     | 
    
         
            +
              }
         
     | 
| 
       1449 
1488 
     | 
    
         | 
| 
       1450 
1489 
     | 
    
         
             
              return tvr_get_field_tv(tvr, doc_num, field);
         
     | 
| 
       1451 
1490 
     | 
    
         
             
            }
         
     | 
| 
         @@ -1454,8 +1493,9 @@ Array *sr_get_term_vectors(IndexReader *ir, int doc_num) 
     | 
|
| 
       1454 
1493 
     | 
    
         
             
            {
         
     | 
| 
       1455 
1494 
     | 
    
         
             
              GET_SR;
         
     | 
| 
       1456 
1495 
     | 
    
         
             
              TermVectorsReader *tvr;
         
     | 
| 
       1457 
     | 
    
         
            -
              if (sr->orig_tvr == NULL || (tvr = sr_tvr(sr)) == NULL)
         
     | 
| 
      
 1496 
     | 
    
         
            +
              if (sr->orig_tvr == NULL || (tvr = sr_tvr(sr)) == NULL) {
         
     | 
| 
       1458 
1497 
     | 
    
         
             
                return NULL;
         
     | 
| 
      
 1498 
     | 
    
         
            +
              }
         
     | 
| 
       1459 
1499 
     | 
    
         | 
| 
       1460 
1500 
     | 
    
         
             
              return tvr_get_tv(tvr, doc_num);
         
     | 
| 
       1461 
1501 
     | 
    
         
             
            }
         
     | 
| 
         @@ -1465,16 +1505,17 @@ void sr_commit(IndexReader *ir) 
     | 
|
| 
       1465 
1505 
     | 
    
         
             
              GET_SR;
         
     | 
| 
       1466 
1506 
     | 
    
         
             
              char tmp_fname[SEGMENT_NAME_MAX_LENGTH];
         
     | 
| 
       1467 
1507 
     | 
    
         
             
              char del_fname[SEGMENT_NAME_MAX_LENGTH];
         
     | 
| 
      
 1508 
     | 
    
         
            +
             
     | 
| 
       1468 
1509 
     | 
    
         
             
              sprintf(del_fname, "%s.del", sr->segment);
         
     | 
| 
       1469 
1510 
     | 
    
         | 
| 
       1470 
     | 
    
         
            -
              if (sr->deleted_docs_dirty) {  
     | 
| 
      
 1511 
     | 
    
         
            +
              if (sr->deleted_docs_dirty) { /* re-write deleted */
         
     | 
| 
       1471 
1512 
     | 
    
         
             
                sprintf(tmp_fname, "%s.tmp", sr->segment);
         
     | 
| 
       1472 
1513 
     | 
    
         
             
                bv_write(sr->deleted_docs, ir->store, tmp_fname);
         
     | 
| 
       1473 
1514 
     | 
    
         
             
                ir->store->rename(ir->store, tmp_fname, del_fname);
         
     | 
| 
       1474 
1515 
     | 
    
         
             
              }
         
     | 
| 
       1475 
1516 
     | 
    
         
             
              if (sr->undelete_all && ir->store->exists(ir->store, del_fname))
         
     | 
| 
       1476 
1517 
     | 
    
         
             
                ir->store->remove(ir->store, del_fname);
         
     | 
| 
       1477 
     | 
    
         
            -
              if (sr->norms_dirty) { 
     | 
| 
      
 1518 
     | 
    
         
            +
              if (sr->norms_dirty) {/* re-write norms */
         
     | 
| 
       1478 
1519 
     | 
    
         
             
                int i;
         
     | 
| 
       1479 
1520 
     | 
    
         
             
                FieldInfo *fi;
         
     | 
| 
       1480 
1521 
     | 
    
         
             
                for (i = 0; i < sr->fis->fcnt; i++) {
         
     | 
| 
         @@ -1494,6 +1535,8 @@ IndexReader *sr_open_internal(IndexReader *ir, SegmentInfo *si) 
     | 
|
| 
       1494 
1535 
     | 
    
         
             
            {
         
     | 
| 
       1495 
1536 
     | 
    
         
             
              Store *store = si->store;
         
     | 
| 
       1496 
1537 
     | 
    
         
             
              SegmentReader *sr = ALLOC(SegmentReader);
         
     | 
| 
      
 1538 
     | 
    
         
            +
              char fname[SEGMENT_NAME_MAX_LENGTH];
         
     | 
| 
      
 1539 
     | 
    
         
            +
              
         
     | 
| 
       1497 
1540 
     | 
    
         
             
              ir->get_term_vector = &sr_get_term_vector;
         
     | 
| 
       1498 
1541 
     | 
    
         
             
              ir->get_term_vectors = &sr_get_term_vectors;
         
     | 
| 
       1499 
1542 
     | 
    
         
             
              ir->num_docs = &sr_num_docs;
         
     | 
| 
         @@ -1518,7 +1561,6 @@ IndexReader *sr_open_internal(IndexReader *ir, SegmentInfo *si) 
     | 
|
| 
       1518 
1561 
     | 
    
         
             
              ir->do_close = &sr_close;
         
     | 
| 
       1519 
1562 
     | 
    
         
             
              ir->data = sr;
         
     | 
| 
       1520 
1563 
     | 
    
         
             
              sr->segment = estrdup(si->name);
         
     | 
| 
       1521 
     | 
    
         
            -
              char fname[SEGMENT_NAME_MAX_LENGTH];
         
     | 
| 
       1522 
1564 
     | 
    
         
             
              sr->cfs_store = NULL;
         
     | 
| 
       1523 
1565 
     | 
    
         
             
              sr->fake_norms = NULL;
         
     | 
| 
       1524 
1566 
     | 
    
         
             
              sprintf(fname, "%s.cfs", sr->segment);
         
     | 
| 
         @@ -1545,13 +1587,13 @@ IndexReader *sr_open_internal(IndexReader *ir, SegmentInfo *si) 
     | 
|
| 
       1545 
1587 
     | 
    
         
             
              sr->freq_in = store->open_input(store, fname);
         
     | 
| 
       1546 
1588 
     | 
    
         
             
              sprintf(fname, "%s.prx", sr->segment);
         
     | 
| 
       1547 
1589 
     | 
    
         
             
              sr->prox_in = store->open_input(store, fname);
         
     | 
| 
       1548 
     | 
    
         
            -
              sr->norms = h_new_str(NULL, &norm_destroy);
         
     | 
| 
      
 1590 
     | 
    
         
            +
              sr->norms = h_new_str((free_ft)NULL, (free_ft)&norm_destroy);
         
     | 
| 
       1549 
1591 
     | 
    
         
             
              sr_open_norms(ir, store);
         
     | 
| 
       1550 
1592 
     | 
    
         | 
| 
       1551 
1593 
     | 
    
         
             
              if (fis_has_vectors(sr->fis)) {
         
     | 
| 
       1552 
1594 
     | 
    
         
             
                sr->orig_tvr = tvr_open(store, sr->segment, sr->fis);
         
     | 
| 
       1553 
1595 
     | 
    
         
             
                thread_key_create(&sr->thread_tvr, NULL);
         
     | 
| 
       1554 
     | 
    
         
            -
                sr->tvr_bucket = ary_create(1, ( 
     | 
| 
      
 1596 
     | 
    
         
            +
                sr->tvr_bucket = ary_create(1, (free_ft)&tvr_close);
         
     | 
| 
       1555 
1597 
     | 
    
         
             
              } else {
         
     | 
| 
       1556 
1598 
     | 
    
         
             
                sr->orig_tvr = NULL;
         
     | 
| 
       1557 
1599 
     | 
    
         
             
              }
         
     | 
| 
         @@ -1560,16 +1602,19 @@ IndexReader *sr_open_internal(IndexReader *ir, SegmentInfo *si) 
     | 
|
| 
       1560 
1602 
     | 
    
         | 
| 
       1561 
1603 
     | 
    
         
             
            IndexReader *sr_open_si(SegmentInfo *si)
         
     | 
| 
       1562 
1604 
     | 
    
         
             
            {
         
     | 
| 
       1563 
     | 
    
         
            -
              IndexReader *ir = ir_create(si->store, NULL, false 
     | 
| 
      
 1605 
     | 
    
         
            +
              IndexReader *ir = ir_create(si->store, NULL, false);
         
     | 
| 
      
 1606 
     | 
    
         
            +
              ref(si->store);
         
     | 
| 
       1564 
1607 
     | 
    
         
             
              return sr_open_internal(ir, si);
         
     | 
| 
       1565 
1608 
     | 
    
         
             
            }
         
     | 
| 
       1566 
1609 
     | 
    
         | 
| 
       1567 
     | 
    
         
            -
            IndexReader *sr_open(SegmentInfos *sis, int si_num,  
     | 
| 
      
 1610 
     | 
    
         
            +
            IndexReader *sr_open(SegmentInfos *sis, int si_num, bool is_owner)
         
     | 
| 
       1568 
1611 
     | 
    
         
             
            {
         
     | 
| 
       1569 
1612 
     | 
    
         
             
              SegmentInfo *si = sis->segs[si_num];
         
     | 
| 
       1570 
     | 
    
         
            -
              IndexReader *ir = ir_create(si->store, sis, is_owner 
     | 
| 
      
 1613 
     | 
    
         
            +
              IndexReader *ir = ir_create(si->store, sis, is_owner);
         
     | 
| 
      
 1614 
     | 
    
         
            +
              ref(si->store);
         
     | 
| 
       1571 
1615 
     | 
    
         
             
              return sr_open_internal(ir, si);
         
     | 
| 
       1572 
1616 
     | 
    
         
             
            }
         
     | 
| 
      
 1617 
     | 
    
         
            +
             
     | 
| 
       1573 
1618 
     | 
    
         
             
            /****************************************************************************
         
     | 
| 
       1574 
1619 
     | 
    
         
             
             *
         
     | 
| 
       1575 
1620 
     | 
    
         
             
             * MultiReader
         
     | 
| 
         @@ -1579,14 +1624,14 @@ IndexReader *sr_open(SegmentInfos *sis, int si_num, int is_owner, int close_stor 
     | 
|
| 
       1579 
1624 
     | 
    
         
             
            #define GET_MR MultiReader *mr = (MultiReader *)ir->data
         
     | 
| 
       1580 
1625 
     | 
    
         
             
            #define GET_READER(doc_num) MultiReader *mr = (MultiReader *)ir->data;\
         
     | 
| 
       1581 
1626 
     | 
    
         
             
              int i = mr_reader_index(mr, doc_num);\
         
     | 
| 
       1582 
     | 
    
         
            -
              IndexReader *reader = mr->sub_readers[i] 
     | 
| 
      
 1627 
     | 
    
         
            +
              IndexReader *reader = mr->sub_readers[i]
         
     | 
| 
       1583 
1628 
     | 
    
         | 
| 
       1584 
1629 
     | 
    
         | 
| 
       1585 
1630 
     | 
    
         | 
| 
       1586 
1631 
     | 
    
         
             
            int mr_reader_index(MultiReader *mr, int doc_num)
         
     | 
| 
       1587 
1632 
     | 
    
         
             
            {
         
     | 
| 
       1588 
     | 
    
         
            -
              int lo = 0;                        
     | 
| 
       1589 
     | 
    
         
            -
              int hi = mr->rcnt - 1;             
     | 
| 
      
 1633 
     | 
    
         
            +
              int lo = 0;                       /* search @starts array */
         
     | 
| 
      
 1634 
     | 
    
         
            +
              int hi = mr->rcnt - 1;            /* for first element less */
         
     | 
| 
       1590 
1635 
     | 
    
         
             
              int mid;
         
     | 
| 
       1591 
1636 
     | 
    
         
             
              int mid_value;
         
     | 
| 
       1592 
1637 
     | 
    
         | 
| 
         @@ -1597,9 +1642,9 @@ int mr_reader_index(MultiReader *mr, int doc_num) 
     | 
|
| 
       1597 
1642 
     | 
    
         
             
                  hi = mid - 1;
         
     | 
| 
       1598 
1643 
     | 
    
         
             
                } else if (doc_num > mid_value) {
         
     | 
| 
       1599 
1644 
     | 
    
         
             
                  lo = mid + 1;
         
     | 
| 
       1600 
     | 
    
         
            -
                } else {  
     | 
| 
      
 1645 
     | 
    
         
            +
                } else { /* found a match */
         
     | 
| 
       1601 
1646 
     | 
    
         
             
                  while ((mid+1 < mr->rcnt) && (mr->starts[mid+1] == mid_value))
         
     | 
| 
       1602 
     | 
    
         
            -
                    mid += 1;  
     | 
| 
      
 1647 
     | 
    
         
            +
                    mid += 1; /* scan to last match in case we have empty segments */
         
     | 
| 
       1603 
1648 
     | 
    
         
             
                  return mid;
         
     | 
| 
       1604 
1649 
     | 
    
         
             
                }
         
     | 
| 
       1605 
1650 
     | 
    
         
             
              }
         
     | 
| 
         @@ -1652,10 +1697,11 @@ Document *mr_get_doc(IndexReader *ir, int doc_num) 
     | 
|
| 
       1652 
1697 
     | 
    
         
             
            void mr_get_norms_into(IndexReader *ir, char *field, uchar *buf, int offset)
         
     | 
| 
       1653 
1698 
     | 
    
         
             
            {
         
     | 
| 
       1654 
1699 
     | 
    
         
             
              int i;
         
     | 
| 
      
 1700 
     | 
    
         
            +
              uchar *bytes;
         
     | 
| 
       1655 
1701 
     | 
    
         
             
              GET_MR;
         
     | 
| 
       1656 
1702 
     | 
    
         | 
| 
       1657 
1703 
     | 
    
         
             
              mutex_lock(&ir->mutex);
         
     | 
| 
       1658 
     | 
    
         
            -
               
     | 
| 
      
 1704 
     | 
    
         
            +
              bytes = h_get(mr->norms_cache, field);
         
     | 
| 
       1659 
1705 
     | 
    
         
             
              if (bytes != NULL) {
         
     | 
| 
       1660 
1706 
     | 
    
         
             
                memcpy(buf + offset, bytes, mr->max_doc);
         
     | 
| 
       1661 
1707 
     | 
    
         
             
              } else {
         
     | 
| 
         @@ -1684,7 +1730,7 @@ uchar *mr_get_norms(IndexReader *ir, char *field) 
     | 
|
| 
       1684 
1730 
     | 
    
         
             
                  reader = mr->sub_readers[i];
         
     | 
| 
       1685 
1731 
     | 
    
         
             
                  reader->get_norms_into(reader, field, bytes, mr->starts[i]);
         
     | 
| 
       1686 
1732 
     | 
    
         
             
                }
         
     | 
| 
       1687 
     | 
    
         
            -
                h_set(mr->norms_cache, field, bytes);  
     | 
| 
      
 1733 
     | 
    
         
            +
                h_set(mr->norms_cache, field, bytes); /* update cache */
         
     | 
| 
       1688 
1734 
     | 
    
         
             
              }
         
     | 
| 
       1689 
1735 
     | 
    
         
             
              mutex_unlock(&ir->mutex);
         
     | 
| 
       1690 
1736 
     | 
    
         | 
| 
         @@ -1694,7 +1740,7 @@ uchar *mr_get_norms(IndexReader *ir, char *field) 
     | 
|
| 
       1694 
1740 
     | 
    
         
             
            void mr_set_norm(IndexReader *ir, int doc_num, char *field, uchar val)
         
     | 
| 
       1695 
1741 
     | 
    
         
             
            {
         
     | 
| 
       1696 
1742 
     | 
    
         
             
              GET_READER(doc_num);
         
     | 
| 
       1697 
     | 
    
         
            -
              h_del(mr->norms_cache, field);                
     | 
| 
      
 1743 
     | 
    
         
            +
              h_del(mr->norms_cache, field);               /* clear cache */
         
     | 
| 
       1698 
1744 
     | 
    
         
             
              ir_set_norm(reader, doc_num - mr->starts[i], field, val);
         
     | 
| 
       1699 
1745 
     | 
    
         
             
            }
         
     | 
| 
       1700 
1746 
     | 
    
         | 
| 
         @@ -1712,7 +1758,7 @@ TermEnum *mr_terms_from(IndexReader *ir, Term *term) 
     | 
|
| 
       1712 
1758 
     | 
    
         | 
| 
       1713 
1759 
     | 
    
         
             
            int mr_doc_freq(IndexReader *ir, Term *t)
         
     | 
| 
       1714 
1760 
     | 
    
         
             
            {
         
     | 
| 
       1715 
     | 
    
         
            -
              int total = 0, i;           
     | 
| 
      
 1761 
     | 
    
         
            +
              int total = 0, i;          /* sum freqs in segments */
         
     | 
| 
       1716 
1762 
     | 
    
         
             
              GET_MR;
         
     | 
| 
       1717 
1763 
     | 
    
         | 
| 
       1718 
1764 
     | 
    
         
             
              IndexReader *reader;
         
     | 
| 
         @@ -1738,9 +1784,10 @@ TermDocEnum *mr_term_positions(IndexReader *ir) 
     | 
|
| 
       1738 
1784 
     | 
    
         
             
            void mr_delete_doc(IndexReader *ir, int doc_num)
         
     | 
| 
       1739 
1785 
     | 
    
         
             
            {
         
     | 
| 
       1740 
1786 
     | 
    
         
             
              GET_READER(doc_num);
         
     | 
| 
       1741 
     | 
    
         
            -
              mr->num_docs_cache = -1; 
     | 
| 
      
 1787 
     | 
    
         
            +
              mr->num_docs_cache = -1; /* invalidate cache */
         
     | 
| 
       1742 
1788 
     | 
    
         | 
| 
       1743 
     | 
    
         
            -
               
     | 
| 
      
 1789 
     | 
    
         
            +
              /* dispatch to segment reader */
         
     | 
| 
      
 1790 
     | 
    
         
            +
              reader->do_delete_doc(reader, doc_num - mr->starts[i]);
         
     | 
| 
       1744 
1791 
     | 
    
         
             
              mr->has_deletions = true;
         
     | 
| 
       1745 
1792 
     | 
    
         
             
            }
         
     | 
| 
       1746 
1793 
     | 
    
         | 
| 
         @@ -1778,8 +1825,9 @@ void mr_undelete_all(IndexReader *ir) 
     | 
|
| 
       1778 
1825 
     | 
    
         
             
            {
         
     | 
| 
       1779 
1826 
     | 
    
         
             
              int i;
         
     | 
| 
       1780 
1827 
     | 
    
         
             
              GET_MR;
         
     | 
| 
       1781 
     | 
    
         
            -
              mr->num_docs_cache = -1;                     // invalidate cache
         
     | 
| 
       1782 
1828 
     | 
    
         
             
              IndexReader *reader;
         
     | 
| 
      
 1829 
     | 
    
         
            +
             
     | 
| 
      
 1830 
     | 
    
         
            +
              mr->num_docs_cache = -1;                     /* invalidate cache */
         
     | 
| 
       1783 
1831 
     | 
    
         
             
              for (i = 0; i < mr->rcnt; i++) {
         
     | 
| 
       1784 
1832 
     | 
    
         
             
                reader = mr->sub_readers[i];
         
     | 
| 
       1785 
1833 
     | 
    
         
             
                reader->do_undelete_all(reader);
         
     | 
| 
         @@ -1829,12 +1877,12 @@ void mr_close(IndexReader *ir) 
     | 
|
| 
       1829 
1877 
     | 
    
         
             
            IndexReader *mr_open(Store *store,
         
     | 
| 
       1830 
1878 
     | 
    
         
             
                SegmentInfos *sis,
         
     | 
| 
       1831 
1879 
     | 
    
         
             
                IndexReader **sub_readers,
         
     | 
| 
       1832 
     | 
    
         
            -
                int rcnt 
     | 
| 
       1833 
     | 
    
         
            -
                int close_store)
         
     | 
| 
      
 1880 
     | 
    
         
            +
                int rcnt)
         
     | 
| 
       1834 
1881 
     | 
    
         
             
            {
         
     | 
| 
       1835 
1882 
     | 
    
         
             
              int i;
         
     | 
| 
       1836 
1883 
     | 
    
         
             
              MultiReader *mr = ALLOC(MultiReader);
         
     | 
| 
       1837 
1884 
     | 
    
         
             
              IndexReader *sub_reader;
         
     | 
| 
      
 1885 
     | 
    
         
            +
              IndexReader *ir;
         
     | 
| 
       1838 
1886 
     | 
    
         
             
              mr->sub_readers = sub_readers;
         
     | 
| 
       1839 
1887 
     | 
    
         
             
              mr->rcnt = rcnt;
         
     | 
| 
       1840 
1888 
     | 
    
         | 
| 
         @@ -1846,15 +1894,16 @@ IndexReader *mr_open(Store *store, 
     | 
|
| 
       1846 
1894 
     | 
    
         
             
              for (i = 0; i < rcnt; i++) {
         
     | 
| 
       1847 
1895 
     | 
    
         
             
                sub_reader = sub_readers[i];
         
     | 
| 
       1848 
1896 
     | 
    
         
             
                mr->starts[i] = mr->max_doc;
         
     | 
| 
       1849 
     | 
    
         
            -
                mr->max_doc += sub_reader->max_doc(sub_reader);  
     | 
| 
      
 1897 
     | 
    
         
            +
                mr->max_doc += sub_reader->max_doc(sub_reader); /* compute max_docs */
         
     | 
| 
       1850 
1898 
     | 
    
         | 
| 
       1851 
     | 
    
         
            -
                if (sub_reader->has_deletions(sub_reader))
         
     | 
| 
      
 1899 
     | 
    
         
            +
                if (sub_reader->has_deletions(sub_reader)) {
         
     | 
| 
       1852 
1900 
     | 
    
         
             
                  mr->has_deletions = true;
         
     | 
| 
      
 1901 
     | 
    
         
            +
                }
         
     | 
| 
       1853 
1902 
     | 
    
         
             
              }
         
     | 
| 
       1854 
1903 
     | 
    
         
             
              mr->starts[rcnt] = mr->max_doc;
         
     | 
| 
       1855 
     | 
    
         
            -
              mr->norms_cache = h_new_str(NULL, & 
     | 
| 
      
 1904 
     | 
    
         
            +
              mr->norms_cache = h_new_str(NULL, &free);
         
     | 
| 
       1856 
1905 
     | 
    
         | 
| 
       1857 
     | 
    
         
            -
               
     | 
| 
      
 1906 
     | 
    
         
            +
              ir = ir_create(store, sis, true);
         
     | 
| 
       1858 
1907 
     | 
    
         
             
              ir->get_term_vector = &mr_get_term_vector;
         
     | 
| 
       1859 
1908 
     | 
    
         
             
              ir->get_term_vectors = &mr_get_term_vectors;
         
     | 
| 
       1860 
1909 
     | 
    
         
             
              ir->num_docs = &mr_num_docs;
         
     | 
| 
         @@ -1888,11 +1937,8 @@ IndexReader *mr_open(Store *store, 
     | 
|
| 
       1888 
1937 
     | 
    
         
             
             *
         
     | 
| 
       1889 
1938 
     | 
    
         
             
             ****************************************************************************/
         
     | 
| 
       1890 
1939 
     | 
    
         | 
| 
       1891 
     | 
    
         
            -
            bool smi_lt( 
     | 
| 
      
 1940 
     | 
    
         
            +
            bool smi_lt(SegmentMergeInfo *smi1, SegmentMergeInfo *smi2)
         
     | 
| 
       1892 
1941 
     | 
    
         
             
            {
         
     | 
| 
       1893 
     | 
    
         
            -
              SegmentMergeInfo *smi1 = (SegmentMergeInfo *)p1;
         
     | 
| 
       1894 
     | 
    
         
            -
              SegmentMergeInfo *smi2 = (SegmentMergeInfo *)p2;
         
     | 
| 
       1895 
     | 
    
         
            -
             
     | 
| 
       1896 
1942 
     | 
    
         
             
              int cmpres = tb_cmp(smi1->tb, smi2->tb);
         
     | 
| 
       1897 
1943 
     | 
    
         
             
              if (cmpres == 0) {
         
     | 
| 
       1898 
1944 
     | 
    
         
             
                return smi1->base < smi2->base;
         
     | 
| 
         @@ -1906,8 +1952,9 @@ int *smi_load_doc_map(SegmentMergeInfo *smi) 
     | 
|
| 
       1906 
1952 
     | 
    
         
             
              IndexReader *ir = smi->ir;
         
     | 
| 
       1907 
1953 
     | 
    
         
             
              if (ir->has_deletions(ir) && (smi->doc_map == NULL)) {
         
     | 
| 
       1908 
1954 
     | 
    
         
             
                int max_doc = ir->max_doc(ir);
         
     | 
| 
       1909 
     | 
    
         
            -
                smi->doc_map = ALLOC_N(int, max_doc);
         
     | 
| 
       1910 
1955 
     | 
    
         
             
                int j = 0, i;
         
     | 
| 
      
 1956 
     | 
    
         
            +
             
     | 
| 
      
 1957 
     | 
    
         
            +
                smi->doc_map = ALLOC_N(int, max_doc);
         
     | 
| 
       1911 
1958 
     | 
    
         
             
                for (i = 0; i < max_doc; i++) {
         
     | 
| 
       1912 
1959 
     | 
    
         
             
                  if (ir->is_deleted(ir, i)) {
         
     | 
| 
       1913 
1960 
     | 
    
         
             
                    smi->doc_map[i] = -1;
         
     | 
| 
         @@ -1931,9 +1978,8 @@ SegmentMergeInfo *smi_create(int base, TermEnum *te, IndexReader *ir) 
     | 
|
| 
       1931 
1978 
     | 
    
         
             
              return smi;
         
     | 
| 
       1932 
1979 
     | 
    
         
             
            }
         
     | 
| 
       1933 
1980 
     | 
    
         | 
| 
       1934 
     | 
    
         
            -
            void smi_destroy( 
     | 
| 
      
 1981 
     | 
    
         
            +
            void smi_destroy(SegmentMergeInfo *smi)
         
     | 
| 
       1935 
1982 
     | 
    
         
             
            {
         
     | 
| 
       1936 
     | 
    
         
            -
              SegmentMergeInfo *smi = (SegmentMergeInfo *)p;
         
     | 
| 
       1937 
1983 
     | 
    
         
             
              smi->postings->close(smi->postings);
         
     | 
| 
       1938 
1984 
     | 
    
         
             
              smi->te->close(smi->te);
         
     | 
| 
       1939 
1985 
     | 
    
         
             
              if (smi->doc_map != NULL)
         
     | 
| 
         @@ -1957,7 +2003,7 @@ SegmentMerger *sm_create(Store *store, char *name, int term_index_interval) 
     | 
|
| 
       1957 
2003 
     | 
    
         
             
              SegmentMerger *sm = ALLOC(SegmentMerger);
         
     | 
| 
       1958 
2004 
     | 
    
         
             
              sm->store = store;
         
     | 
| 
       1959 
2005 
     | 
    
         
             
              sm->name = estrdup(name);
         
     | 
| 
       1960 
     | 
    
         
            -
              sm->readers = ary_create(config.merge_factor, & 
     | 
| 
      
 2006 
     | 
    
         
            +
              sm->readers = ary_create(config.merge_factor, (free_ft)&ir_close);
         
     | 
| 
       1961 
2007 
     | 
    
         
             
              sm->fis = NULL;
         
     | 
| 
       1962 
2008 
     | 
    
         
             
              sm->freq_out = NULL;
         
     | 
| 
       1963 
2009 
     | 
    
         
             
              sm->prox_out = NULL;
         
     | 
| 
         @@ -1976,8 +2022,9 @@ void sm_close(SegmentMerger *sm) 
     | 
|
| 
       1976 
2022 
     | 
    
         
             
              if (sm->freq_out != NULL) os_close(sm->freq_out);
         
     | 
| 
       1977 
2023 
     | 
    
         
             
              if (sm->prox_out != NULL) os_close(sm->prox_out);
         
     | 
| 
       1978 
2024 
     | 
    
         
             
              if (sm->tiw != NULL) {
         
     | 
| 
       1979 
     | 
    
         
            -
                for (i = 0; i < sm->terms_buf_size; i++)
         
     | 
| 
      
 2025 
     | 
    
         
            +
                for (i = 0; i < sm->terms_buf_size; i++) {
         
     | 
| 
       1980 
2026 
     | 
    
         
             
                  free(sm->terms_buf[i].text);
         
     | 
| 
      
 2027 
     | 
    
         
            +
                }
         
     | 
| 
       1981 
2028 
     | 
    
         
             
                free(sm->terms_buf);
         
     | 
| 
       1982 
2029 
     | 
    
         
             
                tiw_close(sm->tiw);
         
     | 
| 
       1983 
2030 
     | 
    
         
             
              }
         
     | 
| 
         @@ -1988,9 +2035,8 @@ void sm_close(SegmentMerger *sm) 
     | 
|
| 
       1988 
2035 
     | 
    
         
             
              sm->queue = NULL;
         
     | 
| 
       1989 
2036 
     | 
    
         
             
            }
         
     | 
| 
       1990 
2037 
     | 
    
         | 
| 
       1991 
     | 
    
         
            -
            void sm_destroy( 
     | 
| 
      
 2038 
     | 
    
         
            +
            void sm_destroy(SegmentMerger *sm)
         
     | 
| 
       1992 
2039 
     | 
    
         
             
            {
         
     | 
| 
       1993 
     | 
    
         
            -
              SegmentMerger *sm = (SegmentMerger *)p;
         
     | 
| 
       1994 
2040 
     | 
    
         
             
              if (sm->fis != NULL) fis_destroy(sm->fis);
         
     | 
| 
       1995 
2041 
     | 
    
         
             
              ary_destroy(sm->readers);
         
     | 
| 
       1996 
2042 
     | 
    
         
             
              sm_close(sm);
         
     | 
| 
         @@ -2028,6 +2074,8 @@ int sm_merge_fields(SegmentMerger *sm) 
     | 
|
| 
       2028 
2074 
     | 
    
         
             
              FieldInfos *fis = sm->fis = fis_create();
         
     | 
| 
       2029 
2075 
     | 
    
         
             
              int doc_count = 0;
         
     | 
| 
       2030 
2076 
     | 
    
         
             
              Document *doc;
         
     | 
| 
      
 2077 
     | 
    
         
            +
              FieldsWriter *fw;
         
     | 
| 
      
 2078 
     | 
    
         
            +
              
         
     | 
| 
       2031 
2079 
     | 
    
         
             
              for (i = 0; i < sm->readers->size; i++) {
         
     | 
| 
       2032 
2080 
     | 
    
         
             
                IndexReader *ir = sm->readers->elems[i];
         
     | 
| 
       2033 
2081 
     | 
    
         | 
| 
         @@ -2049,15 +2097,15 @@ int sm_merge_fields(SegmentMerger *sm) 
     | 
|
| 
       2049 
2097 
     | 
    
         
             
              }
         
     | 
| 
       2050 
2098 
     | 
    
         
             
              fis_write(fis, sm->store, sm->name, ".fnm");
         
     | 
| 
       2051 
2099 
     | 
    
         | 
| 
       2052 
     | 
    
         
            -
               
     | 
| 
       2053 
     | 
    
         
            -
               
     | 
| 
      
 2100 
     | 
    
         
            +
              /* merge field values */
         
     | 
| 
      
 2101 
     | 
    
         
            +
              fw = fw_open(sm->store, sm->name, fis);
         
     | 
| 
       2054 
2102 
     | 
    
         | 
| 
       2055 
2103 
     | 
    
         
             
              TRY
         
     | 
| 
       2056 
2104 
     | 
    
         
             
                for (i = 0; i < sm->readers->size; i++) {
         
     | 
| 
       2057 
2105 
     | 
    
         
             
                  IndexReader *ir = sm->readers->elems[i];
         
     | 
| 
       2058 
2106 
     | 
    
         
             
                  maxdoc = ir->max_doc(ir);
         
     | 
| 
       2059 
2107 
     | 
    
         
             
                  for (j = 0; j < maxdoc; j++) {
         
     | 
| 
       2060 
     | 
    
         
            -
                    if (!ir->is_deleted(ir, j)) {  
     | 
| 
      
 2108 
     | 
    
         
            +
                    if (!ir->is_deleted(ir, j)) { /* skip deleted docs */
         
     | 
| 
       2061 
2109 
     | 
    
         
             
                      doc = ir->get_doc(ir, j);
         
     | 
| 
       2062 
2110 
     | 
    
         
             
                      fw_add_doc(fw, doc);
         
     | 
| 
       2063 
2111 
     | 
    
         
             
                      doc_destroy(doc);
         
     | 
| 
         @@ -2098,7 +2146,7 @@ int sm_append_postings(SegmentMerger *sm, SegmentMergeInfo **smis, int cnt) 
     | 
|
| 
       2098 
2146 
     | 
    
         
             
              int i, j;
         
     | 
| 
       2099 
2147 
     | 
    
         
             
              int last_doc = 0, base, doc, doc_code, freq, last_position, position;
         
     | 
| 
       2100 
2148 
     | 
    
         
             
              int *doc_map = NULL;
         
     | 
| 
       2101 
     | 
    
         
            -
              int df = 0;             
     | 
| 
      
 2149 
     | 
    
         
            +
              int df = 0;            /* number of docs w/ term */
         
     | 
| 
       2102 
2150 
     | 
    
         
             
              TermDocEnum *postings;
         
     | 
| 
       2103 
2151 
     | 
    
         
             
              SegmentMergeInfo *smi;
         
     | 
| 
       2104 
2152 
     | 
    
         
             
              sm_reset_skip(sm);
         
     | 
| 
         @@ -2111,31 +2159,34 @@ int sm_append_postings(SegmentMerger *sm, SegmentMergeInfo **smis, int cnt) 
     | 
|
| 
       2111 
2159 
     | 
    
         
             
                stde_seek_ti(postings, smi->te->ti_curr);
         
     | 
| 
       2112 
2160 
     | 
    
         
             
                while (postings->next(postings)) {
         
     | 
| 
       2113 
2161 
     | 
    
         
             
                  doc = postings->doc_num(postings);
         
     | 
| 
       2114 
     | 
    
         
            -
                  if (doc_map != NULL) 
         
     | 
| 
       2115 
     | 
    
         
            -
                    doc = doc_map[doc];  
     | 
| 
       2116 
     | 
    
         
            -
                   
     | 
| 
      
 2162 
     | 
    
         
            +
                  if (doc_map != NULL) {
         
     | 
| 
      
 2163 
     | 
    
         
            +
                    doc = doc_map[doc]; /* work around deletions */
         
     | 
| 
      
 2164 
     | 
    
         
            +
                  }
         
     | 
| 
      
 2165 
     | 
    
         
            +
                  doc += base;          /* convert to merged space */
         
     | 
| 
       2117 
2166 
     | 
    
         | 
| 
       2118 
     | 
    
         
            -
                  if (doc < last_doc)
         
     | 
| 
      
 2167 
     | 
    
         
            +
                  if (doc < last_doc) {
         
     | 
| 
       2119 
2168 
     | 
    
         
             
                    RAISE(STATE_ERROR, DOC_ORDER_ERROR_MSG);
         
     | 
| 
      
 2169 
     | 
    
         
            +
                  }
         
     | 
| 
       2120 
2170 
     | 
    
         | 
| 
       2121 
2171 
     | 
    
         
             
                  df++;
         
     | 
| 
       2122 
2172 
     | 
    
         | 
| 
       2123 
     | 
    
         
            -
                  if ((df % sm->skip_interval) == 0) 
         
     | 
| 
      
 2173 
     | 
    
         
            +
                  if ((df % sm->skip_interval) == 0) {
         
     | 
| 
       2124 
2174 
     | 
    
         
             
                    sm_buffer_skip(sm, last_doc);
         
     | 
| 
      
 2175 
     | 
    
         
            +
                  }
         
     | 
| 
       2125 
2176 
     | 
    
         | 
| 
       2126 
     | 
    
         
            -
                  doc_code = (doc - last_doc) << 1;     
     | 
| 
      
 2177 
     | 
    
         
            +
                  doc_code = (doc - last_doc) << 1;    /* use low bit to flag freq=1 */
         
     | 
| 
       2127 
2178 
     | 
    
         
             
                  last_doc = doc;
         
     | 
| 
       2128 
2179 
     | 
    
         | 
| 
       2129 
2180 
     | 
    
         
             
                  freq = postings->freq(postings);
         
     | 
| 
       2130 
2181 
     | 
    
         
             
                  if (freq == 1) {
         
     | 
| 
       2131 
     | 
    
         
            -
                    os_write_vint(sm->freq_out, doc_code | 1);  
     | 
| 
      
 2182 
     | 
    
         
            +
                    os_write_vint(sm->freq_out, doc_code | 1); /* write doc & freq=1 */
         
     | 
| 
       2132 
2183 
     | 
    
         
             
                  } else {
         
     | 
| 
       2133 
     | 
    
         
            -
                    os_write_vint(sm->freq_out, doc_code);  
     | 
| 
       2134 
     | 
    
         
            -
                    os_write_vint(sm->freq_out, freq);      
     | 
| 
      
 2184 
     | 
    
         
            +
                    os_write_vint(sm->freq_out, doc_code); /* write doc */
         
     | 
| 
      
 2185 
     | 
    
         
            +
                    os_write_vint(sm->freq_out, freq);     /* write freqency in doc */
         
     | 
| 
       2135 
2186 
     | 
    
         
             
                  }
         
     | 
| 
       2136 
2187 
     | 
    
         | 
| 
       2137 
2188 
     | 
    
         | 
| 
       2138 
     | 
    
         
            -
                  last_position = 0;         
     | 
| 
      
 2189 
     | 
    
         
            +
                  last_position = 0;        /* write position deltas */
         
     | 
| 
       2139 
2190 
     | 
    
         
             
                  for (j = 0; j < freq; j++) {
         
     | 
| 
       2140 
2191 
     | 
    
         
             
                    position = postings->next_position(postings);
         
     | 
| 
       2141 
2192 
     | 
    
         
             
                    os_write_vint(sm->prox_out, position - last_position);
         
     | 
| 
         @@ -2167,12 +2218,12 @@ void sm_merge_term_info(SegmentMerger *sm, SegmentMergeInfo **smis, int cnt) 
     | 
|
| 
       2167 
2218 
     | 
    
         
             
              int freq_pointer = os_pos(sm->freq_out);
         
     | 
| 
       2168 
2219 
     | 
    
         
             
              int prox_pointer = os_pos(sm->prox_out);
         
     | 
| 
       2169 
2220 
     | 
    
         | 
| 
       2170 
     | 
    
         
            -
              int df = sm_append_postings(sm, smis, cnt);       
     | 
| 
      
 2221 
     | 
    
         
            +
              int df = sm_append_postings(sm, smis, cnt);      /* append posting data */
         
     | 
| 
       2171 
2222 
     | 
    
         | 
| 
       2172 
2223 
     | 
    
         
             
              int skip_pointer = sm_write_skip(sm);
         
     | 
| 
       2173 
2224 
     | 
    
         | 
| 
       2174 
2225 
     | 
    
         
             
              if (df > 0) {
         
     | 
| 
       2175 
     | 
    
         
            -
                 
     | 
| 
      
 2226 
     | 
    
         
            +
                /* add an entry to the dictionary with pointers to prox and freq files */
         
     | 
| 
       2176 
2227 
     | 
    
         
             
                ti_set(sm->ti, df, freq_pointer, prox_pointer, (skip_pointer - freq_pointer));
         
     | 
| 
       2177 
2228 
     | 
    
         
             
                tiw_add(sm->tiw, sm_tb_to_term(sm, smis[0]->tb), sm->ti);
         
     | 
| 
       2178 
2229 
     | 
    
         
             
              }
         
     | 
| 
         @@ -2184,7 +2235,7 @@ void sm_merge_term_infos(SegmentMerger *sm) 
     | 
|
| 
       2184 
2235 
     | 
    
         
             
              int i, match_size;
         
     | 
| 
       2185 
2236 
     | 
    
         
             
              IndexReader *ir;
         
     | 
| 
       2186 
2237 
     | 
    
         
             
              TermEnum *te;
         
     | 
| 
       2187 
     | 
    
         
            -
              SegmentMergeInfo *smi, *top;
         
     | 
| 
      
 2238 
     | 
    
         
            +
              SegmentMergeInfo *smi, *top, **match;
         
     | 
| 
       2188 
2239 
     | 
    
         
             
              TermBuffer *tb;
         
     | 
| 
       2189 
2240 
     | 
    
         | 
| 
       2190 
2241 
     | 
    
         
             
              for (i = 0; i < sm->readers->size; i++) {
         
     | 
| 
         @@ -2192,20 +2243,23 @@ void sm_merge_term_infos(SegmentMerger *sm) 
     | 
|
| 
       2192 
2243 
     | 
    
         
             
                te = ir->terms(ir);
         
     | 
| 
       2193 
2244 
     | 
    
         
             
                smi = smi_create(base, te, ir);
         
     | 
| 
       2194 
2245 
     | 
    
         
             
                base += ir->num_docs(ir);
         
     | 
| 
       2195 
     | 
    
         
            -
                if (smi_next(smi) != NULL)
         
     | 
| 
       2196 
     | 
    
         
            -
                  pq_push(sm->queue, smi);  
     | 
| 
       2197 
     | 
    
         
            -
                else
         
     | 
| 
      
 2246 
     | 
    
         
            +
                if (smi_next(smi) != NULL) {
         
     | 
| 
      
 2247 
     | 
    
         
            +
                  pq_push(sm->queue, smi); /* initialize @queue */
         
     | 
| 
      
 2248 
     | 
    
         
            +
                } else {
         
     | 
| 
       2198 
2249 
     | 
    
         
             
                  smi_destroy(smi);
         
     | 
| 
      
 2250 
     | 
    
         
            +
                }
         
     | 
| 
       2199 
2251 
     | 
    
         
             
              }
         
     | 
| 
       2200 
2252 
     | 
    
         | 
| 
       2201 
     | 
    
         
            -
               
     | 
| 
      
 2253 
     | 
    
         
            +
              match = ALLOC_N(SegmentMergeInfo *, sm->readers->size);
         
     | 
| 
       2202 
2254 
     | 
    
         | 
| 
       2203 
2255 
     | 
    
         
             
              while (sm->queue->count > 0) {
         
     | 
| 
       2204 
     | 
    
         
            -
                 
     | 
| 
       2205 
     | 
    
         
            -
                 
     | 
| 
       2206 
     | 
    
         
            -
             
     | 
| 
       2207 
     | 
    
         
            -
             
     | 
| 
       2208 
     | 
    
         
            -
                 
     | 
| 
      
 2256 
     | 
    
         
            +
                /*
         
     | 
| 
      
 2257 
     | 
    
         
            +
                for (i = 1; i <= sm->queue->count; i++) {
         
     | 
| 
      
 2258 
     | 
    
         
            +
                  printf("<{%s:%s}>", ((SegmentMergeInfo *)sm->queue->heap[i])->tb->field,
         
     | 
| 
      
 2259 
     | 
    
         
            +
                                      ((SegmentMergeInfo *)sm->queue->heap[i])->tb->text);
         
     | 
| 
      
 2260 
     | 
    
         
            +
                }printf("\n\n");
         
     | 
| 
      
 2261 
     | 
    
         
            +
                */
         
     | 
| 
      
 2262 
     | 
    
         
            +
                match_size = 0;     /* pop matching terms */
         
     | 
| 
       2209 
2263 
     | 
    
         
             
                match[match_size] = pq_pop(sm->queue);
         
     | 
| 
       2210 
2264 
     | 
    
         
             
                match_size++;
         
     | 
| 
       2211 
2265 
     | 
    
         
             
                tb = match[0]->tb;
         
     | 
| 
         @@ -2216,16 +2270,17 @@ void sm_merge_term_infos(SegmentMerger *sm) 
     | 
|
| 
       2216 
2270 
     | 
    
         
             
                  top = pq_top(sm->queue);
         
     | 
| 
       2217 
2271 
     | 
    
         
             
                }
         
     | 
| 
       2218 
2272 
     | 
    
         | 
| 
       2219 
     | 
    
         
            -
                 
     | 
| 
       2220 
     | 
    
         
            -
                sm_merge_term_info(sm, match, match_size);       
     | 
| 
      
 2273 
     | 
    
         
            +
                /* printf(">%s:%s<\n", match[0]->tb->field, match[0]->tb->text); */
         
     | 
| 
      
 2274 
     | 
    
         
            +
                sm_merge_term_info(sm, match, match_size);      /* add new TermInfo */
         
     | 
| 
       2221 
2275 
     | 
    
         | 
| 
       2222 
2276 
     | 
    
         
             
                while (match_size > 0) {
         
     | 
| 
       2223 
2277 
     | 
    
         
             
                  match_size--;
         
     | 
| 
       2224 
2278 
     | 
    
         
             
                  smi = match[match_size];
         
     | 
| 
       2225 
     | 
    
         
            -
                  if (smi_next(smi) != NULL)
         
     | 
| 
       2226 
     | 
    
         
            -
                    pq_push(sm->queue, smi);  
     | 
| 
       2227 
     | 
    
         
            -
                  else
         
     | 
| 
       2228 
     | 
    
         
            -
                    smi_destroy(smi); 
     | 
| 
      
 2279 
     | 
    
         
            +
                  if (smi_next(smi) != NULL) {
         
     | 
| 
      
 2280 
     | 
    
         
            +
                    pq_push(sm->queue, smi); /* restore queue */
         
     | 
| 
      
 2281 
     | 
    
         
            +
                  } else {
         
     | 
| 
      
 2282 
     | 
    
         
            +
                    smi_destroy(smi);        /* done with a segment */
         
     | 
| 
      
 2283 
     | 
    
         
            +
                  }
         
     | 
| 
       2229 
2284 
     | 
    
         
             
                }
         
     | 
| 
       2230 
2285 
     | 
    
         
             
              }
         
     | 
| 
       2231 
2286 
     | 
    
         
             
              free(match);
         
     | 
| 
         @@ -2242,10 +2297,10 @@ void sm_merge_terms(SegmentMerger *sm) 
     | 
|
| 
       2242 
2297 
     | 
    
         
             
                sprintf(fname, "%s.prx", sm->name);
         
     | 
| 
       2243 
2298 
     | 
    
         
             
                sm->prox_out = sm->store->create_output(sm->store, fname);
         
     | 
| 
       2244 
2299 
     | 
    
         
             
                sm->tiw = tiw_open(sm->store, sm->name, sm->fis, sm->term_index_interval);
         
     | 
| 
       2245 
     | 
    
         
            -
                 
     | 
| 
       2246 
     | 
    
         
            -
             
     | 
| 
       2247 
     | 
    
         
            -
             
     | 
| 
       2248 
     | 
    
         
            -
             
     | 
| 
      
 2300 
     | 
    
         
            +
                /* terms_buf_pointer holds a buffer of terms since the TermInfosWriter needs
         
     | 
| 
      
 2301 
     | 
    
         
            +
                 * to keep the last index_interval terms so that it can compare the last term
         
     | 
| 
      
 2302 
     | 
    
         
            +
                 * put in the index with the next one. So the size of the buffer must by 
         
     | 
| 
      
 2303 
     | 
    
         
            +
                 * index_interval + 2. */
         
     | 
| 
       2249 
2304 
     | 
    
         
             
                sm->terms_buf_pointer = 0;
         
     | 
| 
       2250 
2305 
     | 
    
         
             
                sm->terms_buf_size = sm->tiw->index_interval + 2;
         
     | 
| 
       2251 
2306 
     | 
    
         
             
                sm->terms_buf = ALLOC_N(Term, sm->terms_buf_size);
         
     | 
| 
         @@ -2254,7 +2309,7 @@ void sm_merge_terms(SegmentMerger *sm) 
     | 
|
| 
       2254 
2309 
     | 
    
         
             
                  sm->terms_buf[i].text = ALLOC_N(char, MAX_WORD_SIZE);
         
     | 
| 
       2255 
2310 
     | 
    
         
             
                }
         
     | 
| 
       2256 
2311 
     | 
    
         
             
                sm->skip_interval = sm->tiw->skip_interval;
         
     | 
| 
       2257 
     | 
    
         
            -
                sm->queue = pq_create(sm->readers->size, &smi_lt);
         
     | 
| 
      
 2312 
     | 
    
         
            +
                sm->queue = pq_create(sm->readers->size, (lt_ft)&smi_lt);
         
     | 
| 
       2258 
2313 
     | 
    
         | 
| 
       2259 
2314 
     | 
    
         
             
                sm_merge_term_infos(sm);
         
     | 
| 
       2260 
2315 
     | 
    
         | 
| 
         @@ -2308,11 +2363,13 @@ void sm_merge_vectors(SegmentMerger *sm) 
     | 
|
| 
       2308 
2363 
     | 
    
         
             
                  ir = sm->readers->elems[i];
         
     | 
| 
       2309 
2364 
     | 
    
         
             
                  max_doc = ir->max_doc(ir);
         
     | 
| 
       2310 
2365 
     | 
    
         
             
                  for (j = 0; j < max_doc; j++) {
         
     | 
| 
       2311 
     | 
    
         
            -
                     
     | 
| 
      
 2366 
     | 
    
         
            +
                    /* skip deleted docs */
         
     | 
| 
       2312 
2367 
     | 
    
         
             
                    if (! ir->is_deleted(ir, j)) {
         
     | 
| 
       2313 
2368 
     | 
    
         
             
                      tvs = ir->get_term_vectors(ir, j);
         
     | 
| 
       2314 
     | 
    
         
            -
                       
     | 
| 
       2315 
     | 
    
         
            -
             
     | 
| 
      
 2369 
     | 
    
         
            +
                      if (tvs) {
         
     | 
| 
      
 2370 
     | 
    
         
            +
                        tvw_add_all_doc_vectors(tvw, tvs);
         
     | 
| 
      
 2371 
     | 
    
         
            +
                        ary_destroy(tvs);
         
     | 
| 
      
 2372 
     | 
    
         
            +
                      }
         
     | 
| 
       2316 
2373 
     | 
    
         
             
                    }
         
     | 
| 
       2317 
2374 
     | 
    
         
             
                  }
         
     | 
| 
       2318 
2375 
     | 
    
         
             
                }
         
     | 
| 
         @@ -2333,7 +2390,7 @@ int sm_merge(SegmentMerger *sm) 
     | 
|
| 
       2333 
2390 
     | 
    
         | 
| 
       2334 
2391 
     | 
    
         
             
            Array *sm_create_compound_file(SegmentMerger *sm, char *file_name)
         
     | 
| 
       2335 
2392 
     | 
    
         
             
            {
         
     | 
| 
       2336 
     | 
    
         
            -
              Array *files = ary_create(0, & 
     | 
| 
      
 2393 
     | 
    
         
            +
              Array *files = ary_create(0, &free);
         
     | 
| 
       2337 
2394 
     | 
    
         
             
              CompoundWriter *cw = open_cw(sm->store, file_name);
         
     | 
| 
       2338 
2395 
     | 
    
         
             
              FieldInfo *fi;
         
     | 
| 
       2339 
2396 
     | 
    
         
             
              char fname[SEGMENT_NAME_MAX_LENGTH];
         
     | 
| 
         @@ -2344,7 +2401,7 @@ Array *sm_create_compound_file(SegmentMerger *sm, char *file_name) 
     | 
|
| 
       2344 
2401 
     | 
    
         
             
                ary_append(files, estrdup(fname));
         
     | 
| 
       2345 
2402 
     | 
    
         
             
              }
         
     | 
| 
       2346 
2403 
     | 
    
         | 
| 
       2347 
     | 
    
         
            -
               
     | 
| 
      
 2404 
     | 
    
         
            +
              /* Field norm files */
         
     | 
| 
       2348 
2405 
     | 
    
         
             
              for (i = 0; i < sm->fis->fcnt; i++) {
         
     | 
| 
       2349 
2406 
     | 
    
         
             
                fi = sm->fis->by_number[i];
         
     | 
| 
       2350 
2407 
     | 
    
         
             
                if (fi->is_indexed && !fi->omit_norms) {
         
     | 
| 
         @@ -2353,7 +2410,7 @@ Array *sm_create_compound_file(SegmentMerger *sm, char *file_name) 
     | 
|
| 
       2353 
2410 
     | 
    
         
             
                }
         
     | 
| 
       2354 
2411 
     | 
    
         
             
              }
         
     | 
| 
       2355 
2412 
     | 
    
         | 
| 
       2356 
     | 
    
         
            -
               
     | 
| 
      
 2413 
     | 
    
         
            +
              /* Vector files */
         
     | 
| 
       2357 
2414 
     | 
    
         
             
              if (fis_has_vectors(sm->fis)) {
         
     | 
| 
       2358 
2415 
     | 
    
         
             
                for (i = 0; i < NELEMS(VECTOR_EXTENSIONS); i++) {
         
     | 
| 
       2359 
2416 
     | 
    
         
             
                  sprintf(fname, "%s.%s", sm->name, VECTOR_EXTENSIONS[i]);
         
     | 
| 
         @@ -2361,12 +2418,12 @@ Array *sm_create_compound_file(SegmentMerger *sm, char *file_name) 
     | 
|
| 
       2361 
2418 
     | 
    
         
             
                }
         
     | 
| 
       2362 
2419 
     | 
    
         
             
              }
         
     | 
| 
       2363 
2420 
     | 
    
         | 
| 
       2364 
     | 
    
         
            -
               
     | 
| 
      
 2421 
     | 
    
         
            +
              /* Now merge all added files */
         
     | 
| 
       2365 
2422 
     | 
    
         
             
              for (i = 0; i < files->size; i++) {
         
     | 
| 
       2366 
2423 
     | 
    
         
             
                cw_add_file(cw, (char *)files->elems[i]);
         
     | 
| 
       2367 
2424 
     | 
    
         
             
              }
         
     | 
| 
       2368 
2425 
     | 
    
         | 
| 
       2369 
     | 
    
         
            -
               
     | 
| 
      
 2426 
     | 
    
         
            +
              /* Perform the merge */
         
     | 
| 
       2370 
2427 
     | 
    
         
             
              cw_close(cw);
         
     | 
| 
       2371 
2428 
     | 
    
         | 
| 
       2372 
2429 
     | 
    
         
             
              return files;
         
     | 
| 
         @@ -2386,11 +2443,11 @@ void ir_acquire_write_lock(IndexReader *ir) 
     | 
|
| 
       2386 
2443 
     | 
    
         | 
| 
       2387 
2444 
     | 
    
         
             
              if (ir->write_lock == NULL) {
         
     | 
| 
       2388 
2445 
     | 
    
         
             
                ir->write_lock = ir->store->open_lock(ir->store, WRITE_LOCK_NAME);
         
     | 
| 
       2389 
     | 
    
         
            -
                if (!ir->write_lock->obtain(ir->write_lock))  
     | 
| 
      
 2446 
     | 
    
         
            +
                if (!ir->write_lock->obtain(ir->write_lock)) /* obtain write lock */
         
     | 
| 
       2390 
2447 
     | 
    
         
             
                  RAISE(STATE_ERROR, WRITE_LOCK_ERROR_MSG);
         
     | 
| 
       2391 
2448 
     | 
    
         | 
| 
       2392 
     | 
    
         
            -
                 
     | 
| 
       2393 
     | 
    
         
            -
             
     | 
| 
      
 2449 
     | 
    
         
            +
                /* we have to check whether index has changed since this reader was opened.
         
     | 
| 
      
 2450 
     | 
    
         
            +
                 * if so, this reader is no longer valid for deletion */
         
     | 
| 
       2394 
2451 
     | 
    
         
             
                if (sis_read_current_version(ir->store) > ir->sis->version) {
         
     | 
| 
       2395 
2452 
     | 
    
         
             
                  ir->is_stale = true;
         
     | 
| 
       2396 
2453 
     | 
    
         
             
                  ir->write_lock->release(ir->write_lock);
         
     | 
| 
         @@ -2401,7 +2458,7 @@ void ir_acquire_write_lock(IndexReader *ir) 
     | 
|
| 
       2401 
2458 
     | 
    
         
             
              }
         
     | 
| 
       2402 
2459 
     | 
    
         
             
            }
         
     | 
| 
       2403 
2460 
     | 
    
         | 
| 
       2404 
     | 
    
         
            -
            IndexReader *ir_create(Store *store, SegmentInfos *sis, int is_owner 
     | 
| 
      
 2461 
     | 
    
         
            +
            IndexReader *ir_create(Store *store, SegmentInfos *sis, int is_owner)
         
     | 
| 
       2405 
2462 
     | 
    
         
             
            {
         
     | 
| 
       2406 
2463 
     | 
    
         
             
              IndexReader *ir = ALLOC(IndexReader);
         
     | 
| 
       2407 
2464 
     | 
    
         | 
| 
         @@ -2414,7 +2471,6 @@ IndexReader *ir_create(Store *store, SegmentInfos *sis, int is_owner, int close_ 
     | 
|
| 
       2414 
2471 
     | 
    
         
             
              }
         
     | 
| 
       2415 
2472 
     | 
    
         | 
| 
       2416 
2473 
     | 
    
         
             
              ir->store = store;
         
     | 
| 
       2417 
     | 
    
         
            -
              ir->close_store = close_store;
         
     | 
| 
       2418 
2474 
     | 
    
         
             
              ir->sis = sis;
         
     | 
| 
       2419 
2475 
     | 
    
         
             
              ir->has_changes = false;
         
     | 
| 
       2420 
2476 
     | 
    
         
             
              ir->is_stale = false;
         
     | 
| 
         @@ -2424,7 +2480,11 @@ IndexReader *ir_create(Store *store, SegmentInfos *sis, int is_owner, int close_ 
     | 
|
| 
       2424 
2480 
     | 
    
         
             
              return ir;
         
     | 
| 
       2425 
2481 
     | 
    
         
             
            }
         
     | 
| 
       2426 
2482 
     | 
    
         | 
| 
       2427 
     | 
    
         
            -
             
     | 
| 
      
 2483 
     | 
    
         
            +
            /**
         
     | 
| 
      
 2484 
     | 
    
         
            +
             * Will keep a reference to the store. To let this method delete the store
         
     | 
| 
      
 2485 
     | 
    
         
            +
             * make sure you deref the store that you pass to it
         
     | 
| 
      
 2486 
     | 
    
         
            +
             */
         
     | 
| 
      
 2487 
     | 
    
         
            +
            IndexReader *ir_open(Store *store)
         
     | 
| 
       2428 
2488 
     | 
    
         
             
            {
         
     | 
| 
       2429 
2489 
     | 
    
         
             
              int i;
         
     | 
| 
       2430 
2490 
     | 
    
         
             
              IndexReader *ir;
         
     | 
| 
         @@ -2434,13 +2494,14 @@ IndexReader *ir_open(Store *store, int close_store) 
     | 
|
| 
       2434 
2494 
     | 
    
         
             
              sis = sis_create();
         
     | 
| 
       2435 
2495 
     | 
    
         
             
              sis_read(sis, store);
         
     | 
| 
       2436 
2496 
     | 
    
         
             
              if (sis->scnt == 1) {
         
     | 
| 
       2437 
     | 
    
         
            -
                ir = sr_open(sis, 0, true 
     | 
| 
      
 2497 
     | 
    
         
            +
                ir = sr_open(sis, 0, true);
         
     | 
| 
       2438 
2498 
     | 
    
         
             
              } else {
         
     | 
| 
       2439 
2499 
     | 
    
         
             
                IndexReader **readers = ALLOC_N(IndexReader *, sis->scnt);
         
     | 
| 
       2440 
2500 
     | 
    
         
             
                for (i = 0; i < sis->scnt; i++) {
         
     | 
| 
       2441 
     | 
    
         
            -
                  readers[i] = sr_open(sis, i, false 
     | 
| 
      
 2501 
     | 
    
         
            +
                  readers[i] = sr_open(sis, i, false);
         
     | 
| 
       2442 
2502 
     | 
    
         
             
                }
         
     | 
| 
       2443 
     | 
    
         
            -
                 
     | 
| 
      
 2503 
     | 
    
         
            +
                ref(store);
         
     | 
| 
      
 2504 
     | 
    
         
            +
                ir = mr_open(store, sis, readers, sis->scnt);
         
     | 
| 
       2444 
2505 
     | 
    
         
             
              }
         
     | 
| 
       2445 
2506 
     | 
    
         
             
              mutex_unlock(&store->mutex);
         
     | 
| 
       2446 
2507 
     | 
    
         
             
              return ir;
         
     | 
| 
         @@ -2481,11 +2542,13 @@ void ir_delete_doc(IndexReader *ir, int doc_num) 
     | 
|
| 
       2481 
2542 
     | 
    
         
             
            Document *ir_get_doc_with_term(IndexReader *ir, Term *term)
         
     | 
| 
       2482 
2543 
     | 
    
         
             
            {
         
     | 
| 
       2483 
2544 
     | 
    
         
             
              TermDocEnum *tde = ir_term_docs_for(ir, term);
         
     | 
| 
      
 2545 
     | 
    
         
            +
              Document *doc = NULL;
         
     | 
| 
      
 2546 
     | 
    
         
            +
             
     | 
| 
       2484 
2547 
     | 
    
         
             
              if (!tde) return NULL;
         
     | 
| 
       2485 
2548 
     | 
    
         | 
| 
       2486 
     | 
    
         
            -
               
     | 
| 
       2487 
     | 
    
         
            -
              if (tde->next(tde))
         
     | 
| 
      
 2549 
     | 
    
         
            +
              if (tde->next(tde)) {
         
     | 
| 
       2488 
2550 
     | 
    
         
             
                doc = ir->get_doc(ir, tde->doc_num(tde));
         
     | 
| 
      
 2551 
     | 
    
         
            +
              }
         
     | 
| 
       2489 
2552 
     | 
    
         
             
              tde->close(tde);
         
     | 
| 
       2490 
2553 
     | 
    
         
             
              return doc;
         
     | 
| 
       2491 
2554 
     | 
    
         
             
            }
         
     | 
| 
         @@ -2508,11 +2571,13 @@ void ir_commit_internal(IndexReader *ir) 
     | 
|
| 
       2508 
2571 
     | 
    
         
             
            {
         
     | 
| 
       2509 
2572 
     | 
    
         
             
              if (ir->has_changes) {
         
     | 
| 
       2510 
2573 
     | 
    
         
             
                if (ir->is_owner) {
         
     | 
| 
      
 2574 
     | 
    
         
            +
                  Lock *commit_lock;
         
     | 
| 
       2511 
2575 
     | 
    
         | 
| 
       2512 
2576 
     | 
    
         
             
                  mutex_lock(&ir->store->mutex);
         
     | 
| 
       2513 
     | 
    
         
            -
                   
     | 
| 
       2514 
     | 
    
         
            -
                  if (!commit_lock->obtain(commit_lock))  
     | 
| 
      
 2577 
     | 
    
         
            +
                  commit_lock = ir->store->open_lock(ir->store, COMMIT_LOCK_NAME);
         
     | 
| 
      
 2578 
     | 
    
         
            +
                  if (!commit_lock->obtain(commit_lock)) { /* obtain write lock */
         
     | 
| 
       2515 
2579 
     | 
    
         
             
                    RAISE(STATE_ERROR, COMMIT_LOCK_ERROR_MSG);
         
     | 
| 
      
 2580 
     | 
    
         
            +
                  }
         
     | 
| 
       2516 
2581 
     | 
    
         | 
| 
       2517 
2582 
     | 
    
         
             
                  ir->do_commit(ir);
         
     | 
| 
       2518 
2583 
     | 
    
         
             
                  sis_write(ir->sis, ir->store);
         
     | 
| 
         @@ -2522,7 +2587,7 @@ void ir_commit_internal(IndexReader *ir) 
     | 
|
| 
       2522 
2587 
     | 
    
         
             
                  mutex_unlock(&ir->store->mutex);
         
     | 
| 
       2523 
2588 
     | 
    
         | 
| 
       2524 
2589 
     | 
    
         
             
                  if (ir->write_lock != NULL) {
         
     | 
| 
       2525 
     | 
    
         
            -
                    ir->write_lock->release(ir->write_lock);   
     | 
| 
      
 2590 
     | 
    
         
            +
                    ir->write_lock->release(ir->write_lock);  /* release write lock */
         
     | 
| 
       2526 
2591 
     | 
    
         
             
                    ir->store->close_lock(ir->write_lock);
         
     | 
| 
       2527 
2592 
     | 
    
         
             
                    ir->write_lock = NULL;
         
     | 
| 
       2528 
2593 
     | 
    
         
             
                  }
         
     | 
| 
         @@ -2545,9 +2610,7 @@ void ir_close(IndexReader *ir) 
     | 
|
| 
       2545 
2610 
     | 
    
         
             
              mutex_lock(&ir->mutex);
         
     | 
| 
       2546 
2611 
     | 
    
         
             
              ir_commit_internal(ir);
         
     | 
| 
       2547 
2612 
     | 
    
         
             
              ir->do_close(ir);
         
     | 
| 
       2548 
     | 
    
         
            -
               
     | 
| 
       2549 
     | 
    
         
            -
                ir->store->close(ir->store);
         
     | 
| 
       2550 
     | 
    
         
            -
              }
         
     | 
| 
      
 2613 
     | 
    
         
            +
              store_deref(ir->store);
         
     | 
| 
       2551 
2614 
     | 
    
         
             
              if (ir->is_owner) {
         
     | 
| 
       2552 
2615 
     | 
    
         
             
                sis_destroy(ir->sis);
         
     | 
| 
       2553 
2616 
     | 
    
         
             
              }
         
     | 
| 
         @@ -2562,12 +2625,6 @@ void ir_close(IndexReader *ir) 
     | 
|
| 
       2562 
2625 
     | 
    
         
             
              free(ir);
         
     | 
| 
       2563 
2626 
     | 
    
         
             
            }
         
     | 
| 
       2564 
2627 
     | 
    
         | 
| 
       2565 
     | 
    
         
            -
            void ir_destroy(void *p)
         
     | 
| 
       2566 
     | 
    
         
            -
            {
         
     | 
| 
       2567 
     | 
    
         
            -
              IndexReader *ir = (IndexReader *)p;
         
     | 
| 
       2568 
     | 
    
         
            -
              ir_close(ir);
         
     | 
| 
       2569 
     | 
    
         
            -
            }
         
     | 
| 
       2570 
     | 
    
         
            -
             
     | 
| 
       2571 
2628 
     | 
    
         
             
            /**
         
     | 
| 
       2572 
2629 
     | 
    
         
             
             * Don't call this method if the cache already exists
         
     | 
| 
       2573 
2630 
     | 
    
         
             
             **/
         
     |