ferret 0.9.6 → 0.10.0

data/ext/index_io.c

@@ -1,382 +0,0 @@
-#include <store.h>
-#include <string.h>
-
-static char * const STORE_EOF_ERROR_MSG = "EOF Error when trying to refill";
-
-Buffer *buf_create()
-{
-  Buffer *buf = ALLOC(Buffer);
-  buf->start = 0;
-  buf->pos = 0;
-  buf->len = 0;
-  return buf;
-}
-
-void buf_destroy(Buffer *buf)
-{
-  free(buf);
-}
-
-OutStream *os_create()
-{
-  OutStream *os = ALLOC(OutStream);
-  os->buf.start = 0;
-  os->buf.pos = 0;
-  os->buf.len = 0;
-  return os;
-}
-
-inline void os_flush(OutStream *os)
-{
-  os->flush_internal(os, os->buf.buf, os->buf.pos);
-  os->buf.start += os->buf.pos;
-  os->buf.pos = 0;
-}
-
-void os_close(OutStream *os)
-{
-  os_flush(os);
-  os->close_internal(os);
-  free(os);
-}
-
-int os_pos(OutStream *os)
-{
-  return os->buf.start + os->buf.pos;
-}
-
-void os_seek(OutStream *os, int new_pos)
-{
-  os_flush(os);
-  os->buf.start = new_pos;
-  os->seek_internal(os, new_pos);
-}
-
-#define write_byte(os, b) os->buf.buf[os->buf.pos++] = b
-
-inline void os_write_byte(OutStream *os, uchar b)
-{
-  if (os->buf.pos >= BUFFER_SIZE) {
-    os_flush(os);
-  }
-  write_byte(os, b);
-}
-
-void os_write_bytes(OutStream *os, uchar *b, int len)
-{
-  if (os->buf.pos > 0) { /* flush buffer */
-    os_flush(os);
-  }
-
-  if (len < BUFFER_SIZE) {
-    os->flush_internal(os, b, len);
-    os->buf.start += len;
-  } else {
-    int pos = 0;
-    int size;
-    while (pos < len) {
-      if (len - pos < BUFFER_SIZE) {
-        size = len - pos;
-      } else {
-        size = BUFFER_SIZE;
-      }
-      os->flush_internal(os, b + pos, size);
-      pos += size;
-      os->buf.start += size;
-    }
-  }
-}
-
-InStream *is_create()
-{
-  InStream *is = ALLOC(InStream);
-  is->buf.start = 0;
-  is->buf.pos = 0;
-  is->buf.len = 0;
-  return is;
-}
-
-void is_refill(InStream *is)
-{
-  int start = is->buf.start + is->buf.pos;
-  int last = start + BUFFER_SIZE;
-  int flen = is->length_internal(is);
-  if (last > flen) { /* don't read past EOF */
-    last = flen;
-  }
-
-  is->buf.len = last - start;
-  if (is->buf.len <= 0) {
-    RAISE(EOF_ERROR, STORE_EOF_ERROR_MSG);
-  }
-
-  is->read_internal(is, is->buf.buf, 0, is->buf.len);
-
-  is->buf.start = start;
-  is->buf.pos = 0;
-}
-
-#define read_byte(is) is->buf.buf[is->buf.pos++]
-inline uchar is_read_byte(InStream *is)
-{
-  if (is->buf.pos >= is->buf.len) {
-    is_refill(is);
-  }
-
-  return read_byte(is);
-}
-
-int is_pos(InStream *is)
-{
-  return is->buf.start + is->buf.pos;
-}
-
-uchar *is_read_bytes(InStream *is, uchar *b, int offset, int len)
-{
-  int i, start;
-  if ((offset + len) < BUFFER_SIZE) {
-    for (i = offset; i < offset + len; i++) {
-      b[i] = is_read_byte(is);
-    }
-  } else {                           // read all-at-once
-    start = is_pos(is);
-    is->seek_internal(is, start);
-    is->read_internal(is, b, offset, len);
-
-    is->buf.start = start + len;        // adjust stream variables
-    is->buf.pos = 0;
-    is->buf.len = 0;                    // trigger refill on read
-  }
-  return b;
-}
-
-void is_seek(InStream *is, int pos)
-{
-  if (pos >= is->buf.start && pos < (is->buf.start + is->buf.len)) {
-    is->buf.pos = pos - is->buf.start;  // seek within buffer
-  } else {
-    is->buf.start = pos;
-    is->buf.pos = 0;
-    is->buf.len = 0;               // trigger refill() on read()
-    is->seek_internal(is, pos);
-  }
-}
-
-void is_close(InStream *is)
-{
-  is->close_internal(is);
-  free(is);
-}
-
-InStream *is_clone(InStream *is) 
-{
-  InStream *new_index_i = ALLOC(InStream);
-  memcpy(new_index_i, is, sizeof(InStream));
-  new_index_i->is_clone = true;
-  is->clone_internal(is, new_index_i);
-  return new_index_i;
-}
-
-int
-is_read_int(InStream *is)
-{
-  return ((int)is_read_byte(is) << 24) |
-         ((int)is_read_byte(is) << 16) |
-         ((int)is_read_byte(is) <<  8) |
-          (int)is_read_byte(is);
-}
-
-llong
-is_read_long(InStream *is)
-{
-  return ((llong)is_read_byte(is) << 56) |
-         ((llong)is_read_byte(is) << 48) |
-         ((llong)is_read_byte(is) << 40) |
-         ((llong)is_read_byte(is) << 32) |
-         ((llong)is_read_byte(is) << 24) |
-         ((llong)is_read_byte(is) << 16) |
-         ((llong)is_read_byte(is) <<  8) |
-          (llong)is_read_byte(is);
-}
-
-unsigned int
-is_read_uint(InStream *is)
-{
-  return ((unsigned int)is_read_byte(is) << 24) |
-         ((unsigned int)is_read_byte(is) << 16) |
-         ((unsigned int)is_read_byte(is) <<  8) |
-          (unsigned int)is_read_byte(is);
-}
-
-ullong
-is_read_ulong(InStream *is)
-{
-  return ((ullong)is_read_byte(is) << 56) |
-         ((ullong)is_read_byte(is) << 48) |
-         ((ullong)is_read_byte(is) << 40) |
-         ((ullong)is_read_byte(is) << 32) |
-         ((ullong)is_read_byte(is) << 24) |
-         ((ullong)is_read_byte(is) << 16) |
-         ((ullong)is_read_byte(is) <<  8) |
-          (ullong)is_read_byte(is);
-}
-
-/* optimized to use unchecked read_byte if there is definitely space */
-inline ullong
-is_read_vint(InStream *is)
-{
-  register ullong res, b;
-  register int shift = 7;
-
-  if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
-    b = is_read_byte(is);
-    res = b & 0x7F; // 0x7F = 0b01111111
-    
-    while ((b & 0x80) != 0) {// 0x80 = 0b10000000
-      b = is_read_byte(is);
-      res |= (b & 0x7F) << shift;
-      shift += 7;
-    }
-  } else { // unchecked
-    b = read_byte(is);
-    res = b & 0x7F; // 0x7F = 0b01111111
-    
-    while ((b & 0x80) != 0) {// 0x80 = 0b10000000
-      b = read_byte(is);
-      res |= (b & 0x7F) << shift;
-      shift += 7;
-    }
-  }
-  
-  return res;
-}
-
-inline void
-is_skip_vints(InStream *is, register int cnt)
-{
-  for (; cnt > 0; cnt--) {
-    while ((is_read_byte(is) & 0x80) != 0) {
-    }
-  }
-}
-
-inline void
-is_read_chars(InStream *is, char* buffer, int off, int len)
-{
-  int end, i;
-  
-  end = off + len;
-  
-  for(i = off; i < end; i++) {
-	buffer[i] = is_read_byte(is);
-  }
-}
-
-char *
-is_read_string(InStream *is)
-{
-  register int length = (int)is_read_vint(is);
-  char *str = ALLOC_N(char, length + 1);
-  str[length] = '\0';
-
-  if (is->buf.pos > (is->buf.len - length)) {
-    register int i;
-    for(i = 0; i < length; i++) {
-      str[i] = is_read_byte(is);
-    }
-  } else { // unchecked
-    memcpy(str, is->buf.buf + is->buf.pos, length);
-    is->buf.pos += length;
-  }
-  //is_read_chars(is, str, 0, length);
-
-  return str;
-}
-
-void
-os_write_int(OutStream *os, int l)
-{
-  os_write_byte(os, (uchar)((l >> 24) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 16) & 0xFF));
-  os_write_byte(os, (uchar)((l >>  8) & 0xFF));
-  os_write_byte(os, (uchar)(l & 0xFF));
-}
-
-void
-os_write_long(OutStream *os, llong l)
-{
-  os_write_byte(os, (uchar)((l >> 56) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 48) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 40) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 32) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 24) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 16) & 0xFF));
-  os_write_byte(os, (uchar)((l >>  8) & 0xFF));
-  os_write_byte(os, (uchar)(l & 0xFF));
-}
-
-void
-os_write_uint(OutStream *os, unsigned int l)
-{
-  os_write_byte(os, (uchar)((l >> 24) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 16) & 0xFF));
-  os_write_byte(os, (uchar)((l >>  8) & 0xFF));
-  os_write_byte(os, (uchar)(l & 0xFF));
-}
-
-void
-os_write_ulong(OutStream *os, ullong l)
-{
-  os_write_byte(os, (uchar)((l >> 56) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 48) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 40) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 32) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 24) & 0xFF));
-  os_write_byte(os, (uchar)((l >> 16) & 0xFF));
-  os_write_byte(os, (uchar)((l >>  8) & 0xFF));
-  os_write_byte(os, (uchar)(l & 0xFF));
-}
-
-/* optimized to use an unchecked write if there is space */
-inline void
-os_write_vint(OutStream *os, register ullong i)
-{
-  if (os->buf.pos > VINT_END) {
-    while (i > 127) {
-      os_write_byte(os, (uchar)((i & 0x7f) | 0x80));
-      i >>= 7;
-    }
-    os_write_byte(os, (uchar)(i));
-  } else {
-    while (i > 127) {
-      write_byte(os, (uchar)((i & 0x7f) | 0x80));
-      i >>= 7;
-    }
-    write_byte(os, (uchar)(i));
-  }
-}
-
-void
-os_write_chars(OutStream *os, char *buf, int start, int length)
-{
-  int i;
-
-  for (i = start; i < start + length; i++) {
-    os_write_byte(os, buf[i]);
-  }
-}
-
-void
-os_write_string(OutStream *os, char *str)
-{
-  int len = (int)strlen(str);
-  os_write_vint(os, len);
-
-  os_write_chars(os, str, 0, len);
-}
-
-int file_is_lock(char *filename)
-{
-  int start = (int)strlen(filename) - 4;
-  return ((start > 0) && (strcmp(".lck", &filename[start]) == 0));
-}

data/ext/index_rw.c

@@ -1,2658 +0,0 @@
-#include "index.h"
-#include <stdlib.h>
-#include <string.h>
-#include <array.h>
-static char * const FORMAT_VERSION_ERROR_MSG = "Unknown format version";
-static char * const WRITE_LOCK_ERROR_MSG = "Could not obtain write lock when trying to write index";
-static char * const COMMIT_LOCK_ERROR_MSG = "Could not obtain commit lock when trying to write index";
-static char * const DELETED_DOC_ERROR_MSG = "Tried to get doc that has already been deleted";
-static char * const INVALID_FIELD_TYPE_MSG = "Invalid field-type";
-static char * const DOC_ORDER_ERROR_MSG = "docs out of order curent";
-static char * const STALE_READER_ERROR_MSG = "IndexReader out of date and no longer valid for delete, undelete, or set_norm operations";
-
-const char *INDEX_EXTENSIONS[] = {
-    "cfs", "fnm", "fdx", "fdt", "tii", "tis", "frq", "prx", "del",
-    "tvx", "tvd", "tvf", "tvp"
-};
-
-const char *COMPOUND_EXTENSIONS[] = {
-        "fnm", "frq", "prx", "fdx", "fdt", "tii", "tis"
-};
-      
-const char *VECTOR_EXTENSIONS[] = {
-        "tvx", "tvd", "tvf"
-};
-      
-FerretConfig config = {
-  10,       /* default merge_factor */
-  10,       /* default min_merge_docs */
-  INT_MAX,  /* default max_merge_docs */
-  10000,    /* default max_field_length */
-  128       /* default term_index_interval */
-};
-
-/***************************************************************************
- *
- * CacheObject
- *
- ***************************************************************************/
-
-unsigned int co_hash(const void *key)
-{
-  return (unsigned int)key;
-}
-
-int co_eq(const void *key1, const void *key2)
-{
-  return (key1 == key2);
-}
-
-void co_destroy(CacheObject *self)
-{
-  h_rem(self->ref_tab1, self->ref2, false);
-  h_rem(self->ref_tab2, self->ref1, false);
-  self->destroy(self->obj);
-  free(self);
-}
-
-CacheObject *co_create(HshTable *ref_tab1, HshTable *ref_tab2,
-    void *ref1, void *ref2, free_ft destroy, void *obj)
-{
-  CacheObject *self = ALLOC(CacheObject);
-  h_set(ref_tab1, ref2, self);
-  h_set(ref_tab2, ref1, self);
-  self->ref_tab1 = ref_tab1;
-  self->ref_tab2 = ref_tab2;
-  self->ref1 = ref1;
-  self->ref2 = ref2;
-  self->destroy = destroy;
-  self->obj = obj;
-  return self;
-}
-
-HshTable *co_hsh_create()
-{
-  return h_new(&co_hash, &co_eq, (free_ft)NULL, (free_ft)&co_destroy);
-}
-
-/***************************************************************************
- *
- * Posting
- *
- ***************************************************************************/
-
-Posting *p_create(Term *term, int position, TVOffsetInfo *offset)
-{
-  Posting *self = ALLOC(Posting);
-  self->freq = 1;
-  self->size = 1;
-  self->term = term;
-  self->positions = ALLOC(int);
-  self->positions[0] = position;
-  self->offsets = ALLOC(TVOffsetInfo *);
-  self->offsets[0] = offset;
-  return self;
-}
-
-void p_destroy(Posting *self)
-{
-  /* the positions and offsets will be put in a TVTerm so no need to free */
-  int i;
-  free(self->positions);
-  for (i = 0; i < self->freq; i++)
-    tvoi_destroy(self->offsets[i]);
-  free(self->offsets);
-  free(self);
-}
-
-void p_add_occurance(Posting *self, int position, TVOffsetInfo *offset)
-{
-  if (self->freq >= self->size) {
-    self->size *= 2;
-    REALLOC_N(self->positions, int, self->size);
-    REALLOC_N(self->offsets, TVOffsetInfo *, self->size);
-  }
-  self->positions[self->freq] = position;
-  self->offsets[self->freq] = offset;
-  self->freq++;
-}
-
-inline int p_cmp(const void *const p1, const void *const p2)
-{
-  Term *t1 = (*(Posting **)p1)->term;
-  Term *t2 = (*(Posting **)p2)->term;
-  int res = strcmp(t1->field, t2->field);
-  if (res != 0) {
-    return res;
-  } else {
-    return strcmp(t1->text, t2->text);
-  }
-}
-
-DocumentWriter *dw_open(Store *store,
-                        Analyzer *analyzer,
-                        Similarity *similarity,
-                        int max_field_length,
-                        int term_index_interval)
-{
-  DocumentWriter *self = ALLOC(DocumentWriter);
-  self->store = store;
-  self->analyzer = analyzer;
-  self->similarity = similarity;
-  self->fis = NULL;
-  self->postingtable = h_new(&term_hash, &term_eq,
-                             (free_ft)&term_destroy,
-                             (free_ft)&p_destroy);
-  self->max_field_length = max_field_length;
-  self->term_index_interval = term_index_interval;
-  return self;
-}
-
-void dw_close(DocumentWriter *self)
-{
-  if (self->fis) fis_destroy(self->fis);
-  h_destroy(self->postingtable);
-  free(self);
-}
-
-void dw_add_position(DocumentWriter *self, char *field, char *text,
-    int position, TVOffsetInfo *offset) 
-{
-  Term termbuf = {field, text}, *term;
-  Posting *p = (Posting *)h_get(self->postingtable, &termbuf);
-
-  if (p) { /* word seen before */
-    if (p->freq >= p->size) {
-      /* double size of posting to make room for more posts. */
-      p->size <<= 1;
-      REALLOC_N(p->positions, int, p->size);
-      p->offsets = REALLOC_N(p->offsets, TVOffsetInfo *, p->size);
-    }
-    p->positions[p->freq] = position;    /* add new position */
-    p->offsets[p->freq] = offset;        /* add new offset */
-    p->freq++;                           /* update frequency */
-  } else { /* word not seen before */
-    term = term_create(field, text);
-    h_set(self->postingtable, term, p_create(term, position, offset));
-  }
-}
-
-void dw_invert_doc(DocumentWriter *self, Document *doc)
-{
-  int i;
-  int dfcnt = doc->dfcnt;
-  char *field_name, *text;
-  int field_number, length, position, offset, slen;
-  TokenStream *stream;
-  Token *token;
-  FieldInfo *fi;
-  char text_buf[MAX_WORD_SIZE];
-  text_buf[MAX_WORD_SIZE - 1] = '\0';
-
-  DocField **fields = doc->df_arr, *field;
-  for (i = 0; i < dfcnt; i++) {
-    field = fields[i];
-    field_name = field->name;
-    fi = ((FieldInfo *)ht_get(self->fis->by_name, field_name));
-    field_number = fi->number;
-
-    length = self->field_lengths[field_number];
-    offset = self->field_offsets[field_number];
-    position = self->field_positions[field_number];
-
-    if (fi->is_indexed) {
-      if (!field->is_tokenized) { /* un-tokenized field */
-        text = field->data;
-        slen = (int)strlen(text);
-        if (slen >= MAX_WORD_SIZE) {
-            slen = MAX_WORD_SIZE - 1;
-            text = strncpy(text_buf, text, MAX_WORD_SIZE - 1);
-        }
-        if (fi->store_offset) {
-          dw_add_position(self, field_name, text, position,
-              tvoi_create(offset, offset+slen));
-        } else {
-          dw_add_position(self, field_name, text, position, NULL);
-        }
-        offset += slen;
-        length++;
-      } else {
-
-        /* Tokenize field and add to posting_table */
-        stream = a_get_ts(self->analyzer, field_name, field->data);
-
-        while ((token = ts_next(stream)) != NULL) {
-          position += (token->pos_inc - 1);
-
-          if (fi->store_offset) {
-            dw_add_position(self, 
-                field_name,
-                token->text,
-                position,
-                tvoi_create(offset + token->start, offset + token->end));
-            position++;
-          } else {
-            dw_add_position(self, field_name, token->text, position, NULL);
-            position++;
-          }
-
-          length++;
-          /* stop if we reach the max field length */
-          if (length > self->max_field_length) {
-            break;
-          }
-        }
-
-        if (token) {
-          offset += token->end + 1;
-        }
-      }
-      self->field_lengths[field_number] = length;
-      self->field_offsets[field_number] = offset;
-      self->field_positions[field_number] = position;
-      self->field_boosts[field_number] *= field->boost;
-    }
-  }
-}
-
-Posting **dw_sort_posting_table(DocumentWriter *self)
-{
-  HshTable *ht = self->postingtable;
-  HshEntry *he = ht->table;
-  Posting **postings;
-  int i;
-
-  self->pcnt = i = ht->used;
-  postings = ALLOC_N(Posting *, i);
-
-  while (i > 0) {
-    if (he->value != NULL) {
-      i--;
-      postings[i] = (Posting *)he->value;
-    }
-    he++;
-  }
-  qsort(postings, self->pcnt, sizeof(Posting *), &p_cmp);
-  return postings;
-}
-
-void dw_write_postings(DocumentWriter *self, Posting **postings, char *segment)
-{
-  OutStream * volatile freq_out = NULL, * volatile prox_out = NULL;
-  TermInfosWriter * volatile tiw = NULL;
-  TermVectorsWriter * volatile tvw = NULL;
-  Store *store = self->store;
-  TermInfo * volatile ti = NULL;
-  Posting *posting;
-  int i, j, posting_freq, position, last_position;
-  char fname[SEGMENT_NAME_MAX_LENGTH], *curr_field = NULL, *term_field;
-  strcpy(fname, segment);
-
-  TRY
-    /* open files for inverse index storage */
-    sprintf(fname, "%s.frq", segment);
-    freq_out = store->create_output(store, fname);
-    sprintf(fname, "%s.prx", segment);
-    prox_out = store->create_output(store, fname);
-    tiw = tiw_open(store, segment, self->fis, self->term_index_interval);
-    ti = ti_create(0, 0, 0, 0);
-
-    for (i = 0; i < self->pcnt; i++) {
-      posting = postings[i];
-
-      /* add an entry to dictionary with pointers to prox and freq_out files */
-      ti_set(ti, 1, os_pos(freq_out), os_pos(prox_out), -1);
-      tiw_add(tiw, posting->term, ti);
-
-      /* add an entry to the freq_out file */
-      posting_freq = posting->freq;
-      if (posting_freq == 1) {                 /* optimize freq=1 */
-        os_write_vint(freq_out, 1);            /* set low bit of doc num */
-      } else {
-        os_write_vint(freq_out, 0);            /* the doc number */
-        os_write_vint(freq_out, posting_freq); /* frequency in doc */
-      }
-
-      last_position = 0;               /* write positions */
-
-      for (j = 0; j < posting_freq; j++) {
-        position = posting->positions[j];
-        os_write_vint(prox_out, position - last_position);
-        last_position = position;
-      }
-
-      /* check to see if we switched to a new field */
-      term_field = posting->term->field;
-      if (curr_field != term_field)  {
-        FieldInfo *fi;
-        /* changing field - see if there is something to save */
-        curr_field = term_field;
-        fi = (FieldInfo *)ht_get(self->fis->by_name, curr_field);
-        if (fi->store_tv) {
-          if (tvw == NULL) {
-            tvw = tvw_open(store, segment, self->fis);
-            tvw_open_doc(tvw);
-          }
-          tvw_open_field(tvw, curr_field);
-
-        } else if (tvw != NULL) {
-          tvw_close_field(tvw);
-        }
-      }
-      /* tvw->curr_field != NULL implies field is still open */
-      if (tvw != NULL && tvw->curr_field != NULL) {
-        tvw_add_term(tvw, posting->term->text, posting_freq, posting->positions, posting->offsets);
-      }
-    }
-  XFINALLY
-    if (tvw) {
-      tvw_close_doc(tvw);
-      tvw_close(tvw);
-    }
-    /* make an effort to close all streams we can but remember and re-raise
-     * the last exception encountered in this process */
-    if (freq_out) os_close(freq_out);
-    if (prox_out) os_close(prox_out);
-    if (tiw) tiw_close(tiw);
-    if (ti) ti_destroy(ti);
-  XENDTRY
-}
-
-void dw_write_norms(DocumentWriter *self, char *segment)
-{
-  int i;
-  float norm;
-  OutStream *norms_out;
-  char fname[SEGMENT_NAME_MAX_LENGTH];
-  FieldInfos *fis = self->fis;
-  FieldInfo *fi;
-
-  for (i = 0; i < fis->fcnt; i++) {
-    fi = fis->by_number[i];
-    
-    if (fi->is_indexed && !fi->omit_norms) {
-      norm = self->field_boosts[i] *
-        sim_length_norm(self->similarity, fi->name, self->field_lengths[i]);
-      sprintf(fname, "%s.f%d", segment, i);
-      norms_out = self->store->create_output(self->store, fname);
-      TRY
-        os_write_byte(norms_out, sim_encode_norm(self->similarity, norm));
-      XFINALLY
-        os_close(norms_out);
-      XENDTRY
-    }
-  }
-}
-
-void dw_add_doc(DocumentWriter *self, char *segment, Document *doc)
-{
-  Posting **postings;
-  FieldsWriter *fw;
-  int i;
-
-  /* write field names */
-  self->fis = fis_create();
-  fis_add_doc(self->fis, doc);
-  fis_write(self->fis, self->store, segment, ".fnm");
-
-  /* write field values */
-  fw = fw_open(self->store, segment, self->fis);
-  TRY
-    fw_add_doc(fw, doc);
-  XFINALLY
-    fw_close(fw);
-  XENDTRY
-
-  /* invert doc into posting_table */
-
-  h_clear(self->postingtable); /* clear posting_table */
-
-  self->field_boosts = ALLOC_N(float, self->fis->fcnt);
-  self->field_lengths = ALLOC_AND_ZERO_N(int, self->fis->fcnt);
-  self->field_offsets = ALLOC_AND_ZERO_N(int, self->fis->fcnt);
-  self->field_positions = ALLOC_AND_ZERO_N(int, self->fis->fcnt);
-
-  for (i = 0; i < self->fis->fcnt; i++) {
-    self->field_boosts[i] = doc->boost;
-  }
-
-  dw_invert_doc(self, doc);
-
-  /* sort posting_table into an array */
-  postings = dw_sort_posting_table(self);
-
-  /* write postings */
-  dw_write_postings(self, postings, segment);
-  free(postings);
-
-  /* write norms of indexed fields */
-  dw_write_norms(self, segment);
-
-  free(self->field_boosts);
-  free(self->field_lengths);
-  free(self->field_offsets);
-  free(self->field_positions);
-}
-
-/****************************************************************************
- *
- * SegmentInfo
- *
- ****************************************************************************/
-
-SegmentInfo *si_create(char *name, int doc_cnt, Store *store)
-{
-  SegmentInfo *si = ALLOC(SegmentInfo);
-  si->name = name;
-  si->doc_cnt = doc_cnt;
-  si->store = store;
-  return si;
-}
-
-void si_destroy(SegmentInfo *si)
-{
-  free(si->name);
-  free(si);
-}
-
-bool si_has_deletions(SegmentInfo *si)
-{
-  char del_file_name[SEGMENT_NAME_MAX_LENGTH];
-  sprintf(del_file_name, "%s.del", si->name);
-  return si->store->exists(si->store, del_file_name);
-}
-
-bool si_uses_compound_file(SegmentInfo *si)
-{
-  char compound_file_name[SEGMENT_NAME_MAX_LENGTH];
-  sprintf(compound_file_name, "%s.cfs", si->name);
-  return si->store->exists(si->store, compound_file_name);
-}
-
-struct NormTester {
-  bool has_norm_file;
-  char *segment_name;
-};
-void is_norm_file(char *fname, void *arg)
-{
-  struct NormTester *nt = (struct NormTester *)arg;
-  char norm_file_pattern[SEGMENT_NAME_MAX_LENGTH];
-  sprintf(norm_file_pattern, "%s.s", nt->segment_name);
-  if (strncmp(fname, norm_file_pattern, strlen(norm_file_pattern)) == 0) {
-    nt->has_norm_file = true;
-  }
-}
-
-bool si_has_separate_norms(SegmentInfo *si)
-{
-  struct NormTester nt;
-  nt.segment_name = si->name;
-  nt.has_norm_file = false;
-  si->store->each(si->store, &is_norm_file, &nt);
-
-  return nt.has_norm_file;
-}
-
-
-/****************************************************************************
- *
- * SegmentInfos
- *
- ****************************************************************************/
-
-#include <time.h>
-#define FORMAT -1
-#define SEGMENT_FILENAME "segments"
-#define TEMPORARY_SEGMENT_FILENAME "segments.new"
-
-SegmentInfos *sis_create()
-{
-  SegmentInfos *sis = ALLOC(SegmentInfos);
-  sis->format = FORMAT;
-  sis->version = (unsigned int)time(NULL);
-  sis->scnt = 0;
-  sis->counter = 0;
-  sis->size = 4;
-  sis->segs = ALLOC_N(SegmentInfo *, sis->size);
-  return sis;
-}
-
-void sis_destroy_not_infos(SegmentInfos *sis)
-{
-  free(sis->segs);
-  free(sis);
-}
-
-void sis_destroy(SegmentInfos *sis)
-{
-  int i;
-  for (i = 0; i < sis->scnt; i++)
-    si_destroy(sis->segs[i]);
-  free(sis->segs);
-  free(sis);
-}
-
-void sis_add_si(SegmentInfos *sis, SegmentInfo *si)
-{
-  if (sis->scnt >= sis->size) {
-    sis->size = sis->scnt * 2;
-    REALLOC_N(sis->segs, SegmentInfo *, sis->size);
-  }
-  sis->segs[sis->scnt] = si;
-  sis->scnt++;
-}
-
-void sis_del_at(SegmentInfos *sis, int at)
-{
-  int i;
-  si_destroy(sis->segs[at]);
-  sis->scnt--;
-  for (i = at; i < sis->scnt; i++) {
-    sis->segs[i] = sis->segs[i+1];
-  }
-}
-
-void sis_del_from_to(SegmentInfos *sis, int from, int to)
-{
-  int i, num_to_del = to - from;
-  sis->scnt -= num_to_del;
-  for (i = from; i < to; i++) {
-    si_destroy(sis->segs[i]);
-  }
-  for (i = from; i < sis->scnt; i++) {
-    sis->segs[i] = sis->segs[i+num_to_del];
-  }
-}
-
-void sis_clear(SegmentInfos *sis)
-{
-  int i;
-  for (i = 0; i < sis->scnt; i++) {
-    si_destroy(sis->segs[i]);
-  }
-  sis->scnt = 0;
-}
-
-void sis_read(SegmentInfos *sis, Store *store)
-{
-  int doc_cnt;
-  int seg_count;
-  int i;
-  char *name;
-  InStream *is = store->open_input(store, SEGMENT_FILENAME);
-
-  TRY
-
-    sis->format = is_read_int(is);
-    if (sis->format < 0) { /* file contains explicit format info */
-      /* check that it is a format we can understand */
-      if (sis->format < FORMAT)
-        RAISE(EXCEPTION, FORMAT_VERSION_ERROR_MSG);
-      sis->version = (uint)is_read_long(is);
-      sis->counter = (int)is_read_int(is);
-    } else { /* file is in old format without explicit format info */
-      sis->counter = sis->format;
-    }
-      
-    seg_count = is_read_int(is);
-    for (i = 0; i < seg_count; i++) {
-      name = is_read_string(is);
-      doc_cnt = is_read_int(is);
-      sis_add_si(sis, si_create(name, doc_cnt, store));
-    }
-   
-    if (sis->format >= 0) {
-      /* in old format the version number may be at the end of the file */
-      if (is_pos(is) >= is_length(is)) {
-        sis->version = 0; /* old file format without version number */
-      } else {
-        sis->version = (int)is_read_long(is); /* read version */
-      }
-    }
-  XFINALLY
-    is_close(is);
-  XENDTRY
-}
-
-void sis_write(SegmentInfos *sis, Store *store)
-{
-  int i;
-  SegmentInfo *si;
-  OutStream *os = store->create_output(store, TEMPORARY_SEGMENT_FILENAME);
-  TRY
-    os_write_int(os, FORMAT);
-    os_write_long(os, ++(sis->version)); /* every write changes the index */
-    os_write_int(os, sis->counter);
-    os_write_int(os, sis->scnt); 
-    for (i = 0; i < sis->scnt; i++) {
-      si = sis->segs[i];
-      os_write_string(os, si->name);
-      os_write_int(os, si->doc_cnt);
-    }
-  
-  XFINALLY
-    os_close(os);
-  XENDTRY
-
-  /* install new segment info */
-  store->rename(store, TEMPORARY_SEGMENT_FILENAME, SEGMENT_FILENAME);
-}
-
-int sis_read_current_version(Store *store)
-{
-  InStream *is;
-  SegmentInfos *sis;
-  int format = 0;
-  int version = 0;
-
-  if (!store->exists(store, SEGMENT_FILENAME))
-    return 0;
-  is = store->open_input(store, SEGMENT_FILENAME);
-
-  TRY
-    format = is_read_int(is);
-    if (format < 0) {
-      if (format < FORMAT) 
-        RAISE(EXCEPTION, FORMAT_VERSION_ERROR_MSG);
-      version = (int)is_read_long(is);
-    }
-  XFINALLY
-    is_close(is);
-  XENDTRY
-   
-  if (format < 0)
-    return version;
-
-  /* We cannot be sure about the format of the file.
-   * Therefore we have to read the whole file and cannot simply
-   * seek to the version entry. */
-
-  sis = sis_create();
-  sis_read(sis, store);
-  version = sis->version;
-  sis_destroy(sis);
-  return version;
-}
-
-/****************************************************************************
- *
- * IndexWriter
- *
- ****************************************************************************/
-
-/**
- * Deletes the analyzer by default but leaves the store by default
- */
-IndexWriter *iw_open(Store *store, Analyzer *analyzer, bool create)
-{
-  IndexWriter *iw = ALLOC(IndexWriter);
-  if (create)
-    store->clear_all(store);
-  mutex_init(&iw->mutex, NULL);
-  iw->merge_factor = config.merge_factor;
-  iw->min_merge_docs = config.min_merge_docs;
-  iw->max_merge_docs = config.max_merge_docs;
-  iw->max_field_length = config.max_field_length;
-  iw->term_index_interval = config.term_index_interval;
-  iw->use_compound_file = true;
-  iw->store = store;
-  ref(store);
-  iw->analyzer = analyzer;
-  iw->sis = sis_create();
-  iw->similarity = sim_create_default();
-  iw->ram_store = open_ram_store();
-
-  mutex_lock(&store->mutex);
-  /* keep the write_lock obtained until the IndexWriter is closed. */
-  iw->write_lock = store->open_lock(store, WRITE_LOCK_NAME);
-  if (!iw->write_lock->obtain(iw->write_lock)) {
-    RAISE(STATE_ERROR, WRITE_LOCK_ERROR_MSG);
-  }
-
-  if (create) {
-    Lock *commit_lock = store->open_lock(store, COMMIT_LOCK_NAME);
-    if (!commit_lock->obtain(commit_lock)) {
-      store->close_lock(commit_lock);
-      RAISE(STATE_ERROR, COMMIT_LOCK_ERROR_MSG);
-    }
-    TRY
-      /* commit the index */
-      store->clear(store);
-      sis_write(iw->sis, store);
-    XFINALLY
-      commit_lock->release(commit_lock);
-      store->close_lock(commit_lock);
-    XENDTRY
-  } else {
-    sis_read(iw->sis, store);
-  }
-  mutex_unlock(&store->mutex);
-  return iw;
-}
-
-const char base36_digitmap[] = "0123456789abcdefghijklmnopqrstuvwxyz";
-
-char *new_segment_name(int counter) 
-{
-  char buf[SEGMENT_NAME_MAX_LENGTH];
-  int i;
-
-  buf[SEGMENT_NAME_MAX_LENGTH - 1] = '\0';
-  for (i = SEGMENT_NAME_MAX_LENGTH - 2; ; i--) {
-    buf[i] = base36_digitmap[counter%36];
-    counter /= 36;
-    if (counter == 0) break;
-  }
-  i--;
-  buf[i] = '_';
-  return estrdup(&buf[i]);
-}
-
-int iw_doc_count(IndexWriter *iw)
-{
-  int i, doc_cnt = 0;
-  mutex_lock(&iw->mutex);
-  for (i = 0; i < iw->sis->scnt; i++)
-    doc_cnt += iw->sis->segs[i]->doc_cnt;
-  mutex_unlock(&iw->mutex);
-  return doc_cnt;
-}
-
-void delete_files(Array *file_names, Store *store)
-{
-  int i;
-  for (i = 0; i < file_names->size; i++) {
-    store->remove(store, (char *)file_names->elems[i]);
-  }
-  ary_destroy(file_names);
-}
-
-
-Array *sr_file_names(IndexReader *ir);
-void iw_delete_segments(IndexWriter *iw, IndexReader **segment_readers, int del_cnt)
-{
-  /* The java version keeps a record of files that it couldn't delete. This
-   * shouldn't be a problem on linux I hope. */
-  IndexReader *ir;
-  int i;
-  for (i = 0; i < del_cnt; i++) {
-    ir = segment_readers[i];
-    delete_files(sr_file_names(ir), ir->store);
-  }
-}
-
-void make_compound_file(IndexWriter *iw, char *merged_name, SegmentMerger *merger)
-{
-  Array *files_to_delete;
-  Lock *commit_lock;
-  char merged_tmp[SEGMENT_NAME_MAX_LENGTH], merged_cfs[SEGMENT_NAME_MAX_LENGTH];
-
-  mutex_lock(&iw->store->mutex);
-  sprintf(merged_tmp, "%s.tmp", merged_name);
-  sprintf(merged_cfs, "%s.cfs", merged_name);
-
-  files_to_delete = sm_create_compound_file(merger, merged_tmp);
-  commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
-
-  if (!commit_lock->obtain(commit_lock)) {
-    RAISE(STATE_ERROR, COMMIT_LOCK_ERROR_MSG);
-  }
-
-  /* make compound file visible for SegmentReaders */
-  iw->store->rename(iw->store, merged_tmp, merged_cfs);
-
-  /* delete now unused files of segment */
-  delete_files(files_to_delete, iw->store);
-
-  commit_lock->release(commit_lock);
-  iw->store->close_lock(commit_lock);
-  mutex_unlock(&iw->store->mutex);
-}
-
-void iw_merge_segments_with_max(IndexWriter *iw, int min_segment, int max_segment)
-{
-  int i;
-  int merged_doc_count;
-  Lock *commit_lock;
-  IndexReader **segments_to_delete = ALLOC_N(IndexReader *, max_segment - min_segment);
-  int del_cnt = 0;
-
-  char *merged_name = new_segment_name(iw->sis->counter++);
-
-  SegmentMerger *merger = sm_create(iw->store, merged_name, iw->term_index_interval);
-  IndexReader *reader;
-
-
-  for (i = min_segment; i < max_segment; i++) {
-    reader = sr_open(iw->sis, i, false);
-    sm_add(merger, reader);
-    if ((reader->store == iw->store) ||         /* if we own the directory */
-        (reader->store == iw->ram_store)) {
-      segments_to_delete[del_cnt++] = reader;   /* queue segment for deletion */
-    }
-  }
-  
-  merged_doc_count = sm_merge(merger);
-
-  sis_del_from_to(iw->sis, min_segment, max_segment);
-
-  sis_add_si(iw->sis, si_create(merged_name, merged_doc_count, iw->store));
-
-  /* close readers before we attempt to delete now-obsolete segments */
-
-  mutex_lock(&iw->store->mutex);
-  commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
-  if (!commit_lock->obtain(commit_lock)) {
-    RAISE(STATE_ERROR, COMMIT_LOCK_ERROR_MSG);
-  }
-  /* commit the index */
-  sis_write(iw->sis, iw->store);
-  iw_delete_segments(iw, segments_to_delete, del_cnt);
-
-  commit_lock->release(commit_lock);
-  iw->store->close_lock(commit_lock);
-  mutex_unlock(&iw->store->mutex);
-
-  if (iw->use_compound_file) {
-    make_compound_file(iw, merged_name, merger);
-  }
-
-  free(segments_to_delete);
-  sm_destroy(merger);
-}
-
-void iw_merge_segments(IndexWriter *iw, int min_segment)
-{
-  iw_merge_segments_with_max(iw, min_segment, iw->sis->scnt);
-}
-
-void iw_maybe_merge_segments(IndexWriter *iw)
-{
-  int target_merge_docs = iw->min_merge_docs;
-  int min_segment, merge_docs;
-  SegmentInfo *si;
-
-  while (target_merge_docs <= iw->max_merge_docs) {
-    /* find segments smaller than current target size */
-    min_segment = iw->sis->scnt - 1;
-    merge_docs = 0;
-    while (min_segment >= 0) {
-      si = iw->sis->segs[min_segment];
-      if (si->doc_cnt >= target_merge_docs) {
-        break;
-      }
-      merge_docs += si->doc_cnt;
-      min_segment -= 1;
-    }
-
-    if (merge_docs >= target_merge_docs) { /* found a merge to do */
-      iw_merge_segments(iw, min_segment + 1);
-    } else {
-      break;
-    }
-
-    target_merge_docs *= iw->merge_factor; /* increase target size */
-  }
-}
-
-void iw_flush_ram_segments(IndexWriter *iw)
-{
-  int min_segment = iw->sis->scnt-1;
-  int doc_count = 0;
-  SegmentInfo **segs = iw->sis->segs;
-  while ((min_segment >= 0) &&
-          (segs[min_segment]->store == iw->ram_store)) {
-    doc_count += segs[min_segment]->doc_cnt;
-    min_segment--;
-  }
-  /* the following if statement is actually incrementing for different
-   * reasons. If min_segment < 0 then we must increment as we searched
-   * off the end. If the top segment is not ram_store there are no 
-   * ram segments to flush so we increment so the next check will return
-   * us from this function. Lastly, the min_segment stopped at a segment
-   * that wasn't the ram segment. But if it fit's in with the merge
-   * factor, why not merge it. Otherwise we leave it and increment min_seg
-   */
-  if ((min_segment < 0) ||                       /* add one FS segment? */
-      ((doc_count + segs[min_segment]->doc_cnt) > iw->merge_factor) ||
-      (segs[iw->sis->scnt - 1]->store != iw->ram_store)) {
-    min_segment++;
-  }
-  if (min_segment >= iw->sis->scnt) {
-    return;
-  }
-  iw_merge_segments(iw, min_segment);
-}
-
-void iw_add_doc(IndexWriter *iw, Document *doc)
-{
-  DocumentWriter *dw;
-  char *segment_name;
-
-  mutex_lock(&iw->mutex);
-  dw = dw_open(iw->ram_store,
-               iw->analyzer,
-               iw->similarity,
-               iw->max_field_length,
-               iw->term_index_interval);
-  segment_name = new_segment_name(iw->sis->counter++);
-  dw_add_doc(dw, segment_name, doc);
-  dw_close(dw);
-  sis_add_si(iw->sis, si_create(segment_name, 1, iw->ram_store));
-  iw_maybe_merge_segments(iw);
-  mutex_unlock(&iw->mutex);
-}
-
-static inline void iw_optimize_internal(IndexWriter *iw)
-{
-  int min_segment;
-  iw_flush_ram_segments(iw);
-  while (iw->sis->scnt > 1 ||
-          (iw->sis->scnt == 1 &&
-            (  si_has_deletions(iw->sis->segs[0]) ||
-              (iw->sis->segs[0]->store != iw->store) ||
-              (iw->use_compound_file &&
-                (!si_uses_compound_file(iw->sis->segs[0]) ||
-                  si_has_separate_norms(iw->sis->segs[0])))))) {
-    min_segment = iw->sis->scnt - iw->merge_factor;
-    iw_merge_segments(iw, min_segment < 0 ? 0 : min_segment);
-  }
-}
-void iw_optimize(IndexWriter *iw)
-{
-  mutex_lock(&iw->mutex);
-  iw_optimize_internal(iw);
-  mutex_unlock(&iw->mutex);
-}
-
-void iw_close(IndexWriter *iw)
-{
-  mutex_lock(&iw->mutex);
-  iw_flush_ram_segments(iw);
-  store_deref(iw->ram_store);
-  sis_destroy(iw->sis);
-
-  sim_destroy(iw->similarity);
-  a_deref(iw->analyzer);
-
-  iw->write_lock->release(iw->write_lock);
-  iw->store->close_lock(iw->write_lock);
-
-  store_deref(iw->store);
-  mutex_destroy(&iw->mutex);
-  free(iw);
-}
-
-void iw_add_indexes(IndexWriter *iw, Store **stores, int cnt)
-{
-  int i, j, end, start;
-
-  mutex_lock(&iw->mutex);
-  iw_optimize_internal(iw); /* start with zero or 1 seg */
-
-  start = iw->sis->scnt;
-
-  for (i = 0; i < cnt; i++) {
-    Store *store = stores[i];
-    SegmentInfos *sis = sis_create(); /* read infos from dir */
-    sis_read(sis, store);
-
-    for (j = 0; j < sis->scnt; j++) {
-      SegmentInfo *si = sis->segs[j];
-      sis_add_si(iw->sis, si);
-    }
-    sis_destroy_not_infos(sis);
-  }
-
-  /* merge newly added segments in log(n) passes */
-  while (iw->sis->scnt > start + iw->merge_factor) {
-    for (i = start + 1; i < iw->sis->scnt; i++) {
-      end = MIN(iw->sis->scnt, i + iw->merge_factor);
-      if (end - i > 1) {
-        iw_merge_segments_with_max(iw, i, end);
-      }
-    }
-  }
-
-  /* final cleanup */
-  iw_optimize_internal(iw);
-  mutex_unlock(&iw->mutex);
-}
-
-
-/**
- * This adds an array of readers to the index leaving the added readers open.
- */
-void iw_add_readers(IndexWriter *iw, IndexReader **irs, int cnt)
-{
-  IndexReader *ir = NULL;
-  int i, del_cnt = 0;
-  int doc_count;
-  char *merged_name;
-  SegmentMerger *merger;
-  Lock *commit_lock;
-  
-  mutex_lock(&iw->mutex);
-  iw_optimize_internal(iw); /* start with zero or 1 seg */
-
-  merged_name = new_segment_name(iw->sis->counter++);
-
-  merger = sm_create(iw->store, merged_name, iw->term_index_interval);
-  merger->readers->free_elem = NULL; /* don't close readers */
-
-  if (iw->sis->scnt == 1) { /* add existing index, if any */
-    ir = sr_open_si(iw->sis->segs[0]);
-    sm_add(merger, ir);
-    del_cnt = 1;
-  }
-
-  for (i = 0; i < cnt; i++) {
-    sm_add(merger, irs[i]);
-  }
-
-  doc_count = sm_merge(merger); /* merge 'em */
-
-  /* pop old infos and add new ones. */
-  sis_clear(iw->sis);
-  sis_add_si(iw->sis, si_create(merged_name, doc_count, iw->store));
-
-
-  commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
-  if (!commit_lock->obtain(commit_lock)) { /* obtain write lock */
-    RAISE(STATE_ERROR, COMMIT_LOCK_ERROR_MSG);
-  }
-
-  sis_write(iw->sis, iw->store); /* commit changes */
-  iw_delete_segments(iw, &ir, del_cnt);
-  if (ir) ir_close(ir);
-
-  commit_lock->release(commit_lock);
-  iw->store->close_lock(commit_lock);
-
-  if (iw->use_compound_file) {
-    make_compound_file(iw, merged_name, merger);
-  }
-
-  iw_optimize_internal(iw);
-  sm_destroy(merger);
-
-  mutex_unlock(&iw->mutex);
-}
-
-/****************************************************************************
- *
- * Norm
- *
- ****************************************************************************/
-
-Norm *norm_create(InStream *is, int field_num)
-{
-  Norm *norm = ALLOC(Norm);
-  norm->is = is;
-  norm->field_num = field_num;
-  norm->bytes = NULL;
-  norm->is_dirty = false;
-  return norm;
-}
-
-void norm_destroy(Norm *norm)
-{
-  is_close(norm->is);
-  if (norm->bytes != NULL) {
-    free(norm->bytes);
-  }
-  free(norm);
-}
-
-void norm_rewrite(Norm *norm, Store *store, char *segment,
-    int doc_count, Store *cfs_store)
-{
-  OutStream *os;
-  char tmp_fname[SEGMENT_NAME_MAX_LENGTH];
-  char norm_fname[SEGMENT_NAME_MAX_LENGTH];
-
-  if (norm->bytes == NULL) {
-    return; /* These norms do not need to be rewritten */
-  }
-
-  sprintf(tmp_fname, "%s.tmp", segment);
-  os = store->create_output(store, tmp_fname);
-  TRY
-    os_write_bytes(os, norm->bytes, doc_count);
-  XFINALLY
-    os_close(os);
-  XENDTRY
-  if (cfs_store) {
-    sprintf(norm_fname, "%s.s%d", segment, norm->field_num);
-  } else {
-    sprintf(norm_fname, "%s.f%d", segment, norm->field_num);
-  }
-  store->rename(store, tmp_fname, norm_fname);
-  norm->is_dirty = false;
-}
-
-/****************************************************************************
- *
- * SegmentReader
- *
- ****************************************************************************/
-
-#define GET_SR SegmentReader *sr = (SegmentReader *)ir->data
-
-int sr_max_doc(IndexReader *ir)
-{
-  return ((SegmentReader *)ir->data)->fr->len;
-}
-
-static inline void sr_close_norms(SegmentReader *sr)
-{
-  h_destroy(sr->norms);
-}
-
-static inline TermVectorsReader *sr_tvr(SegmentReader *sr)
-{
-  TermVectorsReader *tvr;
-  if ((tvr = thread_getspecific(sr->thread_tvr)) == NULL) {
-    tvr = tvr_clone(sr->orig_tvr);
-    if (tvr == NULL) printf("scuk\n");
-    ary_append(sr->tvr_bucket, tvr);
-    thread_setspecific(sr->thread_tvr, tvr);
-  }
-  return tvr;
-}
-
-void sr_close(IndexReader *ir)
-{
-  GET_SR;
-  fr_close(sr->fr);
-  tir_close(sr->tir);
-
-  if (sr->freq_in) is_close(sr->freq_in);
-  if (sr->prox_in) is_close(sr->prox_in);
-  
-  fis_destroy(sr->fis);
-  sr_close_norms(sr);
-  
-  if (sr->orig_tvr) {
-    tvr_close(sr->orig_tvr);
-    thread_key_delete(sr->thread_tvr);
-    ary_destroy(sr->tvr_bucket);
-  }
-  if (sr->deleted_docs) bv_destroy(sr->deleted_docs);
-  if (sr->cfs_store) store_deref(sr->cfs_store);
-  if (sr->fake_norms) free(sr->fake_norms);
-  free(sr->segment);
-  free(sr);
-}
-
-void sr_delete_doc(IndexReader *ir, int doc_num) 
-{
-  GET_SR;
-  if (sr->deleted_docs == NULL)
-    sr->deleted_docs = bv_create();
-
-  sr->deleted_docs_dirty = true;
-  sr->undelete_all = false;
-  bv_set(sr->deleted_docs, doc_num);
-}
-
-static inline bool sr_is_deleted_internal(IndexReader *ir, int doc_num)
-{
-  GET_SR;
-  return (sr->deleted_docs != NULL && bv_get(sr->deleted_docs, doc_num));
-}
-
-bool sr_is_deleted(IndexReader *ir, int doc_num)
-{
-  bool is_del;
-
-  mutex_lock(&ir->mutex);
-  is_del = sr_is_deleted_internal(ir, doc_num);
-  mutex_unlock(&ir->mutex);
-
-  return is_del;
-}
-
-bool sr_has_norms(IndexReader *ir, char *field)
-{
-  GET_SR;
-  bool has_norms;
-  mutex_lock(&ir->mutex);
-  has_norms = h_has_key(sr->norms, field);
-  mutex_unlock(&ir->mutex);
-
-  return has_norms;
-}
-
-bool sr_has_deletions(IndexReader *ir)
-{
-  GET_SR;
-  return (sr->deleted_docs != NULL);
-}
-
-void sr_undelete_all(IndexReader *ir)
-{
-  GET_SR;
-  sr->undelete_all = true;
-  sr->deleted_docs_dirty = false;
-  if (sr->deleted_docs != NULL) bv_destroy(sr->deleted_docs);
-  sr->deleted_docs = NULL;
-}
-
-TermEnum *sr_terms(IndexReader *ir)
-{
-  TermEnum *te = ((SegmentReader *)ir->data)->tir->orig_te;
-  return te->clone(te);
-}
-
-TermEnum *sr_terms_from(IndexReader *ir, Term *term)
-{
-  TermEnum *te = ((SegmentReader *)ir->data)->tir->orig_te;
-  TermEnum *ret_te = te->clone(te);
-  te_skip_to(ret_te, term);
-  return ret_te;
-}
-
-Document *sr_get_doc(IndexReader *ir, int doc_num)
-{
-  GET_SR;
-  Document *doc;
-  mutex_lock(&ir->mutex);
-  if (sr_is_deleted_internal(ir, doc_num)) {
-    mutex_unlock(&ir->mutex);
-    RAISE(STATE_ERROR, DELETED_DOC_ERROR_MSG);
-  }
-  doc = fr_get_doc(sr->fr, doc_num);
-  mutex_unlock(&ir->mutex);
-  return doc;
-}
-
-static inline void
-sr_get_norms_into_internal(IndexReader *ir, char *field, uchar *buf, int offset)
-{
-  GET_SR;
-  Norm *norm = h_get(sr->norms, field);
-  if (norm == NULL) {
-    memset(buf + offset*sizeof(uchar), 0, sr_max_doc(ir)*sizeof(uchar));
-  } else if (norm->bytes != NULL) { /* can copy from cache */
-    memcpy(buf + offset*sizeof(uchar), norm->bytes, sr_max_doc(ir)*sizeof(uchar));
-  } else {
-    InStream *norm_in = is_clone(norm->is);
-    /* read from disk */
-    is_seek(norm_in, 0);
-    is_read_bytes(norm_in, buf, offset, sr_max_doc(ir));
-    is_close(norm_in);
-  }
-}
-
-void sr_get_norms_into(IndexReader *ir, char *field, uchar *buf, int offset)
-{
-  mutex_lock(&ir->mutex);
-  sr_get_norms_into_internal(ir, field, buf, offset);
-  mutex_unlock(&ir->mutex);
-}
-
-static inline uchar *sr_get_norms_internal(IndexReader *ir, char *field)
-{
-  GET_SR;
-  Norm *norm = h_get(sr->norms, field);
-  if (norm == NULL) {                           /* not an indexed field */
-    return NULL;
-  }
-
-  if (norm->bytes == NULL) {                    /* value not yet read */
-    uchar *bytes = ALLOC_N(uchar, ir->max_doc(ir));
-    sr_get_norms_into_internal(ir, field, bytes, 0);
-    norm->bytes = bytes;                        /* cache it */
-  }
-  return norm->bytes;
-}
-
-uchar *sr_get_norms(IndexReader *ir, char *field)
-{
-  uchar *norms;
-  mutex_lock(&ir->mutex);
-  norms = sr_get_norms_internal(ir, field);
-  mutex_unlock(&ir->mutex);
-  return norms;
-}
-
-static inline uchar *sr_get_norms_always(IndexReader *ir, char *field)
-{
-  GET_SR;
-  uchar *bytes;
-  mutex_lock(&ir->mutex);
-
-  bytes = sr_get_norms_internal(ir, field);
-  if (bytes == NULL) {
-    if (sr->fake_norms) {
-      bytes = sr->fake_norms;
-    } else {
-      int len = ir->max_doc(ir);
-      sr->fake_norms = bytes = ALLOC_N(uchar, len);
-      memset(bytes, 0, len);
-    }
-  }
-  mutex_unlock(&ir->mutex);
-  return bytes;
-}
-
-void sr_set_norm(IndexReader *ir, int doc_num, char *field, uchar val)
-{
-  GET_SR;
-  Norm *norm;
-  
-  norm = h_get(sr->norms, field);
-  if (norm != NULL) { /* an indexed field */
-    norm->is_dirty = true;                           /* mark it dirty */
-    sr->norms_dirty = true;
-
-    sr_get_norms_internal(ir, field)[doc_num] = val;
-  }
-}
-
-int sr_doc_freq(IndexReader *ir, Term *t)
-{
-  GET_SR;
-  TermInfo *ti = tir_get_ti(sr->tir, t);
-  if (ti != NULL) {
-    int df = ti->doc_freq;
-    ti_destroy(ti);
-    return df;
-  } else {
-    return 0;
-  }
-}
-
-Array *sr_file_names(IndexReader *ir)
-{
-  GET_SR;
-  Array *file_names = ary_create(0, &free);
-  FieldInfo *fi;
-  int i;
-  char fname[SEGMENT_NAME_MAX_LENGTH];
-
-  for (i = 0; i < NELEMS(INDEX_EXTENSIONS); i++) {
-    sprintf(fname, "%s.%s", sr->segment, INDEX_EXTENSIONS[i]);
-    if (ir->store->exists(ir->store, fname))
-      ary_append(file_names, estrdup(fname));
-  }
-
-  for (i = 0; i < sr->fis->fcnt; i++) {
-    fi = sr->fis->by_number[i];
-    if (fi->is_indexed && !fi->omit_norms) {
-      if (sr->cfs_store) {
-        sprintf(fname, "%s.s%d", sr->segment, i);
-      } else {
-        sprintf(fname, "%s.f%d", sr->segment, i);
-      }
-      if (ir->store->exists(ir->store, fname))
-        ary_append(file_names, estrdup(fname));
-    }
-  }
-  return file_names;
-}
-
-HashSet *sr_get_field_names(IndexReader *ir, int field_type)
-{
-  GET_SR;
-  int i;
-  HashSet *field_set = hs_str_create(NULL);
-  FieldInfo *fi;
-  for (i = 0; i < sr->fis->fcnt; i++) {
-    fi = sr->fis->by_number[i];
-    switch(field_type) {
-      case IR_ALL:
-        hs_add(field_set, fi->name);
-        break;
-      case IR_UNINDEXED:
-        if (!fi->is_indexed) hs_add(field_set, fi->name);
-        break;
-      case IR_INDEXED:
-        if (fi->is_indexed) hs_add(field_set, fi->name);
-        break;
-      case IR_INDEXED_NO_TERM_VECTOR:
-        if (fi->is_indexed && !fi->store_tv) hs_add(field_set, fi->name);
-        break;
-      case IR_TERM_VECTOR:
-        if (fi->store_tv && !fi->store_pos && !fi->store_offset)
-          hs_add(field_set, fi->name);
-        break;
-      case IR_INDEXED_WITH_TERM_VECTOR:
-        if (fi->is_indexed && fi->store_tv) hs_add(field_set, fi->name);
-        break;
-      case IR_TERM_VECTOR_WITH_POSITION:
-        if (fi->store_pos && !fi->store_offset) hs_add(field_set, fi->name);
-        break;
-      case IR_TERM_VECTOR_WITH_OFFSET:
-        if (!fi->store_pos && fi->store_offset) hs_add(field_set, fi->name);
-      case IR_TERM_VECTOR_WITH_POSITION_OFFSET:
-        if (fi->store_pos && fi->store_offset) hs_add(field_set, fi->name);
-        break;
-      default:
-        RAISE(ARG_ERROR, INVALID_FIELD_TYPE_MSG);
-    }
-  }
-  return field_set;
-}
-
-int sr_num_docs(IndexReader *ir)
-{
-  GET_SR;
-  int num_docs;
-  
-  mutex_lock(&ir->mutex);
-  num_docs = sr_max_doc(ir);
-  if (sr->deleted_docs != NULL)
-    num_docs -= sr->deleted_docs->count;
-  mutex_unlock(&ir->mutex);
-  return num_docs;
-}
-
-TermDocEnum *sr_term_docs(IndexReader *ir)
-{
-  return stde_create(ir);
-}
-
-TermDocEnum *sr_term_positions(IndexReader *ir)
-{
-  return stpe_create(ir);
-}
-
-void sr_open_norms(IndexReader *ir, Store *cfs_store)
-{
-  GET_SR;
-  int i;
-  FieldInfo *fi;
-  Store *tmp_store;
-  char fname[SEGMENT_NAME_MAX_LENGTH];
-  for (i = 0; i < sr->fis->fcnt; i++) {
-    tmp_store = ir->store;
-    fi = sr->fis->by_number[i];
-    if (fi->is_indexed && !fi->omit_norms) {
-      sprintf(fname, "%s.s%d", sr->segment, fi->number);
-      if (! tmp_store->exists(tmp_store, fname)) {
-        sprintf(fname, "%s.f%d", sr->segment, fi->number);
-        tmp_store = cfs_store;
-      }
-      h_set(sr->norms, fi->name,
-          norm_create(tmp_store->open_input(tmp_store, fname), fi->number));
-    }
-  }
-  sr->norms_dirty = false;
-}
-
-TermVector *sr_get_term_vector(IndexReader *ir, int doc_num, char *field)
-{
-  GET_SR;
-  FieldInfo *fi = (FieldInfo *)ht_get(sr->fis->by_name, field);
-  TermVectorsReader *tvr;
-
-  if (fi == NULL || !fi->store_tv || !sr->orig_tvr || !(tvr = sr_tvr(sr))) {
-    return NULL;
-  }
-
-  return tvr_get_field_tv(tvr, doc_num, field);
-}
-
-Array *sr_get_term_vectors(IndexReader *ir, int doc_num)
-{
-  GET_SR;
-  TermVectorsReader *tvr;
-  if (sr->orig_tvr == NULL || (tvr = sr_tvr(sr)) == NULL) {
-    return NULL;
-  }
-
-  return tvr_get_tv(tvr, doc_num);
-}
-
-void sr_commit(IndexReader *ir)
-{
-  GET_SR;
-  char tmp_fname[SEGMENT_NAME_MAX_LENGTH];
-  char del_fname[SEGMENT_NAME_MAX_LENGTH];
-
-  sprintf(del_fname, "%s.del", sr->segment);
-
-  if (sr->deleted_docs_dirty) { /* re-write deleted */
-    sprintf(tmp_fname, "%s.tmp", sr->segment);
-    bv_write(sr->deleted_docs, ir->store, tmp_fname);
-    ir->store->rename(ir->store, tmp_fname, del_fname);
-  }
-  if (sr->undelete_all && ir->store->exists(ir->store, del_fname))
-    ir->store->remove(ir->store, del_fname);
-  if (sr->norms_dirty) {/* re-write norms */
-    int i;
-    FieldInfo *fi;
-    for (i = 0; i < sr->fis->fcnt; i++) {
-      fi = sr->fis->by_number[i];
-      if (fi->is_indexed) {
-        norm_rewrite((Norm *)h_get(sr->norms, fi->name), ir->store,
-            sr->segment, sr_max_doc(ir), sr->cfs_store);
-      }
-    }
-  }
-  sr->deleted_docs_dirty = false;
-  sr->norms_dirty = false;
-  sr->undelete_all = false;
-}
-
-IndexReader *sr_open_internal(IndexReader *ir, SegmentInfo *si)
-{
-  Store *store = si->store;
-  SegmentReader *sr = ALLOC(SegmentReader);
-  char fname[SEGMENT_NAME_MAX_LENGTH];
-  
-  ir->get_term_vector = &sr_get_term_vector;
-  ir->get_term_vectors = &sr_get_term_vectors;
-  ir->num_docs = &sr_num_docs;
-  ir->max_doc = &sr_max_doc;
-  ir->get_doc = &sr_get_doc;
-  ir->get_norms_into = &sr_get_norms_into;
-  ir->get_norms = &sr_get_norms;
-  ir->get_norms_always = &sr_get_norms_always;
-  ir->do_set_norm = &sr_set_norm;
-  ir->terms = &sr_terms;
-  ir->terms_from = &sr_terms_from;
-  ir->doc_freq = &sr_doc_freq;
-  ir->term_docs = &sr_term_docs;
-  ir->term_positions = &sr_term_positions;
-  ir->do_delete_doc = &sr_delete_doc;
-  ir->is_deleted = &sr_is_deleted;
-  ir->has_norms = &sr_has_norms;
-  ir->has_deletions = &sr_has_deletions;
-  ir->do_undelete_all = &sr_undelete_all;
-  ir->get_field_names = &sr_get_field_names;
-  ir->do_commit = &sr_commit;
-  ir->do_close = &sr_close;
-  ir->data = sr;
-  sr->segment = estrdup(si->name);
-  sr->cfs_store = NULL;
-  sr->fake_norms = NULL;
-  sprintf(fname, "%s.cfs", sr->segment);
-  if (store->exists(store, fname)) {
-    sr->cfs_store = open_cmpd_store(store, fname);
-    store = sr->cfs_store;
-  }
-
-  sprintf(fname, "%s.fnm", sr->segment);
-  
-  sr->fis = fis_open(store, fname);
-  sr->fr = fr_open(store, sr->segment, sr->fis);
-
-  sr->tir = tir_open(store, sr->segment, sr->fis);
-  sr->deleted_docs = NULL;
-  sr->deleted_docs_dirty = false;
-  sr->undelete_all = false;
-  if (si_has_deletions(si)) {
-    sprintf(fname, "%s.del", sr->segment);
-    sr->deleted_docs = bv_read(si->store, fname);
-  }
-
-  sprintf(fname, "%s.frq", sr->segment);
-  sr->freq_in = store->open_input(store, fname);
-  sprintf(fname, "%s.prx", sr->segment);
-  sr->prox_in = store->open_input(store, fname);
-  sr->norms = h_new_str((free_ft)NULL, (free_ft)&norm_destroy);
-  sr_open_norms(ir, store);
-
-  if (fis_has_vectors(sr->fis)) {
-    sr->orig_tvr = tvr_open(store, sr->segment, sr->fis);
-    thread_key_create(&sr->thread_tvr, NULL);
-    sr->tvr_bucket = ary_create(1, (free_ft)&tvr_close);
-  } else {
-    sr->orig_tvr = NULL;
-  }
-  return ir;
-}
-
-IndexReader *sr_open_si(SegmentInfo *si)
-{
-  IndexReader *ir = ir_create(si->store, NULL, false);
-  ref(si->store);
-  return sr_open_internal(ir, si);
-}
-
-IndexReader *sr_open(SegmentInfos *sis, int si_num, bool is_owner)
-{
-  SegmentInfo *si = sis->segs[si_num];
-  IndexReader *ir = ir_create(si->store, sis, is_owner);
-  ref(si->store);
-  return sr_open_internal(ir, si);
-}
-
-/****************************************************************************
- *
- * MultiReader
- *
- ****************************************************************************/
-
-#define GET_MR MultiReader *mr = (MultiReader *)ir->data
-#define GET_READER(doc_num) MultiReader *mr = (MultiReader *)ir->data;\
-  int i = mr_reader_index(mr, doc_num);\
-  IndexReader *reader = mr->sub_readers[i]
-
-
-
-int mr_reader_index(MultiReader *mr, int doc_num)
-{
-  int lo = 0;                       /* search @starts array */
-  int hi = mr->rcnt - 1;            /* for first element less */
-  int mid;
-  int mid_value;
-
-  while (hi >= lo) {
-    mid = (lo + hi) >> 1;
-    mid_value = mr->starts[mid];
-    if (doc_num < mid_value) {
-      hi = mid - 1;
-    } else if (doc_num > mid_value) {
-      lo = mid + 1;
-    } else { /* found a match */
-      while ((mid+1 < mr->rcnt) && (mr->starts[mid+1] == mid_value))
-        mid += 1; /* scan to last match in case we have empty segments */
-      return mid;
-    }
-  }
-  return hi;
-}
-
-TermVector *mr_get_term_vector(IndexReader *ir, int doc_num, char *field)
-{
-  GET_READER(doc_num);
-  return reader->get_term_vector(reader, doc_num - mr->starts[i], field);
-}
-
-Array *mr_get_term_vectors(IndexReader *ir, int doc_num)
-{
-  GET_READER(doc_num);
-  return reader->get_term_vectors(reader, doc_num - mr->starts[i]);
-}
-
-int mr_num_docs(IndexReader *ir)
-{
-  int i, num_docs;
-  GET_MR;
-  mutex_lock(&ir->mutex);
-  if (mr->num_docs_cache == -1) {
-    IndexReader *reader;
-    mr->num_docs_cache = 0;
-    for (i = 0; i < mr->rcnt; i++) {
-      reader = mr->sub_readers[i];
-      mr->num_docs_cache += reader->num_docs(reader);
-    }
-  }
-  num_docs = mr->num_docs_cache;
-  mutex_unlock(&ir->mutex);
-
-  return num_docs;
-}
-
-int mr_max_doc(IndexReader *ir)
-{
-  GET_MR;
-  return mr->max_doc;
-}
-
-Document *mr_get_doc(IndexReader *ir, int doc_num)
-{
-  GET_READER(doc_num);
-  return reader->get_doc(reader, doc_num - mr->starts[i]);
-}
-
-void mr_get_norms_into(IndexReader *ir, char *field, uchar *buf, int offset)
-{
-  int i;
-  uchar *bytes;
-  GET_MR;
-
-  mutex_lock(&ir->mutex);
-  bytes = h_get(mr->norms_cache, field);
-  if (bytes != NULL) {
-    memcpy(buf + offset, bytes, mr->max_doc);
-  } else {
-    IndexReader *reader;
-    for (i = 0; i < mr->rcnt; i++) {
-      reader = mr->sub_readers[i];
-      reader->get_norms_into(reader, field, buf, offset + mr->starts[i]);
-    }
-  }
-  mutex_unlock(&ir->mutex);
-}
-
-uchar *mr_get_norms(IndexReader *ir, char *field)
-{
-  int i;
-  GET_MR;
-  uchar *bytes;
-  IndexReader *reader;
-
-  mutex_lock(&ir->mutex);
-  bytes = h_get(mr->norms_cache, field);
-  if (bytes == NULL) {
-    bytes = ALLOC_N(uchar, mr->max_doc);
-
-    for (i = 0; i < mr->rcnt; i++) {
-      reader = mr->sub_readers[i];
-      reader->get_norms_into(reader, field, bytes, mr->starts[i]);
-    }
-    h_set(mr->norms_cache, field, bytes); /* update cache */
-  }
-  mutex_unlock(&ir->mutex);
-
-  return bytes;
-}
-
-void mr_set_norm(IndexReader *ir, int doc_num, char *field, uchar val)
-{
-  GET_READER(doc_num);
-  h_del(mr->norms_cache, field);               /* clear cache */
-  ir_set_norm(reader, doc_num - mr->starts[i], field, val);
-}
-
-TermEnum *mr_terms(IndexReader *ir)
-{
-  GET_MR;
-  return mte_create(mr->sub_readers, mr->starts, mr->rcnt, NULL);
-}
-
-TermEnum *mr_terms_from(IndexReader *ir, Term *term)
-{
-  GET_MR;
-  return mte_create(mr->sub_readers, mr->starts, mr->rcnt, term);
-}
-
-int mr_doc_freq(IndexReader *ir, Term *t)
-{
-  int total = 0, i;          /* sum freqs in segments */
-  GET_MR;
-
-  IndexReader *reader;
-  for (i = 0; i < mr->rcnt; i++) {
-    reader = mr->sub_readers[i];
-    total += reader->doc_freq(reader, t);
-  }
-  return total;
-}
-
-TermDocEnum *mr_term_docs(IndexReader *ir)
-{
-  GET_MR;
-  return mtde_create(mr->sub_readers, mr->starts, mr->rcnt);
-}
-
-TermDocEnum *mr_term_positions(IndexReader *ir)
-{
-  GET_MR;
-  return mtpe_create(mr->sub_readers, mr->starts, mr->rcnt);
-}
-
-void mr_delete_doc(IndexReader *ir, int doc_num)
-{
-  GET_READER(doc_num);
-  mr->num_docs_cache = -1; /* invalidate cache */
-
-  /* dispatch to segment reader */
-  reader->do_delete_doc(reader, doc_num - mr->starts[i]);
-  mr->has_deletions = true;
-}
-
-bool mr_is_deleted(IndexReader *ir, int doc_num)
-{
-  GET_READER(doc_num);
-  return reader->is_deleted(reader, doc_num - mr->starts[i]);
-}
-
-bool mr_has_norms(IndexReader *ir, char *field)
-{
-  bool has_norms = false;
-  int i;
-  GET_MR;
-
-  IndexReader *reader;
-  for (i = 0; i < mr->rcnt; i++) {
-    reader = mr->sub_readers[i];
-    if (reader->has_norms(reader, field)) {
-      has_norms = true;
-      break;
-    }
-  }
-
-  return has_norms;
-}
-
-bool mr_has_deletions(IndexReader *ir)
-{
-  GET_MR;
-  return mr->has_deletions;
-}
-
-void mr_undelete_all(IndexReader *ir)
-{
-  int i;
-  GET_MR;
-  IndexReader *reader;
-
-  mr->num_docs_cache = -1;                     /* invalidate cache */
-  for (i = 0; i < mr->rcnt; i++) {
-    reader = mr->sub_readers[i];
-    reader->do_undelete_all(reader);
-  }
-  mr->has_deletions = false;
-}
-
-HashSet *mr_get_field_names(IndexReader *ir, int field_type)
-{
-  int i;
-  GET_MR;
-  HashSet *field_set = hs_str_create(NULL);
-  IndexReader *reader;
-  for (i = 0; i < mr->rcnt; i++) {
-    reader = mr->sub_readers[i];
-    hs_merge(field_set, reader->get_field_names(reader, field_type));
-  }
-  return field_set;
-}
-
-void mr_commit(IndexReader *ir)
-{
-  GET_MR;
-  int i;
-  IndexReader *reader;
-  for (i = 0; i < mr->rcnt; i++) {
-    reader = mr->sub_readers[i];
-    reader->do_commit(reader);
-  }
-}
-
-void mr_close(IndexReader *ir)
-{
-  GET_MR;
-  int i;
-  IndexReader *reader;
-  for (i = 0; i < mr->rcnt; i++) {
-    reader = mr->sub_readers[i];
-    ir_close(reader);
-  }
-  free(mr->sub_readers);
-  h_destroy(mr->norms_cache);
-  free(mr->starts);
-  free(mr);
-}
-
-IndexReader *mr_open(Store *store,
-    SegmentInfos *sis,
-    IndexReader **sub_readers,
-    int rcnt)
-{
-  int i;
-  MultiReader *mr = ALLOC(MultiReader);
-  IndexReader *sub_reader;
-  IndexReader *ir;
-  mr->sub_readers = sub_readers;
-  mr->rcnt = rcnt;
-
-  mr->max_doc = 0;
-  mr->num_docs_cache = -1;
-  mr->has_deletions = false;
-
-  mr->starts = ALLOC_N(int, (rcnt+1));
-  for (i = 0; i < rcnt; i++) {
-    sub_reader = sub_readers[i];
-    mr->starts[i] = mr->max_doc;
-    mr->max_doc += sub_reader->max_doc(sub_reader); /* compute max_docs */
-
-    if (sub_reader->has_deletions(sub_reader)) {
-      mr->has_deletions = true;
-    }
-  }
-  mr->starts[rcnt] = mr->max_doc;
-  mr->norms_cache = h_new_str(NULL, &free);
-
-  ir = ir_create(store, sis, true);
-  ir->get_term_vector = &mr_get_term_vector;
-  ir->get_term_vectors = &mr_get_term_vectors;
-  ir->num_docs = &mr_num_docs;
-  ir->max_doc = &mr_max_doc;
-  ir->get_doc = &mr_get_doc;
-  ir->get_norms_into = &mr_get_norms_into;
-  ir->get_norms = &mr_get_norms;
-  ir->get_norms_always = &mr_get_norms;
-  ir->do_set_norm = &mr_set_norm;
-  ir->terms = &mr_terms;
-  ir->terms_from = &mr_terms_from;
-  ir->doc_freq = &mr_doc_freq;
-  ir->term_docs = &mr_term_docs;
-  ir->term_positions = &mr_term_positions;
-  ir->do_delete_doc = &mr_delete_doc;
-  ir->is_deleted = &mr_is_deleted;
-  ir->has_norms = &mr_has_norms;
-  ir->has_deletions = &mr_has_deletions;
-  ir->do_undelete_all = &mr_undelete_all;
-  ir->get_field_names = &mr_get_field_names;
-  ir->do_commit = &mr_commit;
-  ir->do_close = &mr_close;
-  ir->data = mr;
-
-  return ir;
-}
-
-/****************************************************************************
- *
- * SegmentMergeInfo
- *
- ****************************************************************************/
-
-bool smi_lt(SegmentMergeInfo *smi1, SegmentMergeInfo *smi2)
-{
-  int cmpres = tb_cmp(smi1->tb, smi2->tb);
-  if (cmpres == 0) {
-    return smi1->base < smi2->base;
-  } else {
-    return cmpres < 0;
-  }
-}
-
-int *smi_load_doc_map(SegmentMergeInfo *smi)
-{
-  IndexReader *ir = smi->ir;
-  if (ir->has_deletions(ir) && (smi->doc_map == NULL)) {
-    int max_doc = ir->max_doc(ir);
-    int j = 0, i;
-
-    smi->doc_map = ALLOC_N(int, max_doc);
-    for (i = 0; i < max_doc; i++) {
-      if (ir->is_deleted(ir, i)) {
-        smi->doc_map[i] = -1;
-      } else {
-        smi->doc_map[i] = j++;
-      }
-    }
-  }
-  return smi->doc_map;
-}
-
-SegmentMergeInfo *smi_create(int base, TermEnum *te, IndexReader *ir)
-{
-  SegmentMergeInfo *smi = ALLOC(SegmentMergeInfo);
-  smi->base = base;
-  smi->ir = ir;
-  smi->te = te;
-  smi->tb = te->tb_curr;
-  smi->postings = ir->term_positions(ir);
-  smi->doc_map = NULL;
-  return smi;
-}
-
-void smi_destroy(SegmentMergeInfo *smi)
-{
-  smi->postings->close(smi->postings);
-  smi->te->close(smi->te);
-  if (smi->doc_map != NULL)
-    free(smi->doc_map);
-  free(smi);
-}
-
-TermBuffer *smi_next(SegmentMergeInfo *smi)
-{
-  return (smi->tb = smi->te->next(smi->te));
-}
-
-/****************************************************************************
- *
- * SegmentMerger
- *
- ****************************************************************************/
-
-SegmentMerger *sm_create(Store *store, char *name, int term_index_interval)
-{
-  SegmentMerger *sm = ALLOC(SegmentMerger);
-  sm->store = store;
-  sm->name = estrdup(name);
-  sm->readers = ary_create(config.merge_factor, (free_ft)&ir_close);
-  sm->fis = NULL;
-  sm->freq_out = NULL;
-  sm->prox_out = NULL;
-  sm->tiw = NULL;
-  sm->queue = NULL;
-  sm->ti = ti_create(0, 0, 0, 0);
-  sm->term_index_interval = term_index_interval;
-  sm->skip_buffer = ram_create_buffer();
-  sm->skip_interval = -1;
-  return sm;
-}
-
-void sm_close(SegmentMerger *sm)
-{
-  int i;
-  if (sm->freq_out != NULL) os_close(sm->freq_out);
-  if (sm->prox_out != NULL) os_close(sm->prox_out);
-  if (sm->tiw != NULL) {
-    for (i = 0; i < sm->terms_buf_size; i++) {
-      free(sm->terms_buf[i].text);
-    }
-    free(sm->terms_buf);
-    tiw_close(sm->tiw);
-  }
-  if (sm->queue != NULL) pq_destroy(sm->queue);
-  sm->freq_out = NULL;
-  sm->prox_out = NULL;
-  sm->tiw = NULL;
-  sm->queue = NULL;
-}
-
-void sm_destroy(SegmentMerger *sm)
-{
-  if (sm->fis != NULL) fis_destroy(sm->fis);
-  ary_destroy(sm->readers);
-  sm_close(sm);
-  free(sm->name);
-  ti_destroy(sm->ti);
-  ram_destroy_buffer(sm->skip_buffer);
-  free(sm);
-}
-
-void sm_add(SegmentMerger *sm, IndexReader *ir)
-{
-  ary_append(sm->readers, ir);
-}
-
-static inline void sm_add_indexed(IndexReader *ir,
-    FieldInfos *fis,
-    HashSet *fields,
-    bool store_tv,
-    bool store_pos,
-    bool store_offset)
-{
-  int i;
-  char *field;
-  for (i = 0; i < fields->size; i++) {
-    field = (char *)fields->elems[i];
-    fis_add(fis, field, true, store_tv, store_pos, store_offset,
-        !ir->has_norms(ir, field));
-  }
-  hs_destroy(fields);
-}
-
-int sm_merge_fields(SegmentMerger *sm)
-{
-  int i, j, maxdoc;
-  FieldInfos *fis = sm->fis = fis_create();
-  int doc_count = 0;
-  Document *doc;
-  FieldsWriter *fw;
-  
-  for (i = 0; i < sm->readers->size; i++) {
-    IndexReader *ir = sm->readers->elems[i];
-    
-    sm_add_indexed(ir, fis,
-        ir->get_field_names(ir, IR_TERM_VECTOR_WITH_POSITION_OFFSET),
-        true, true, true);
-    sm_add_indexed(ir, fis,
-        ir->get_field_names(ir, IR_TERM_VECTOR_WITH_POSITION),
-        true, true, false);
-    sm_add_indexed(ir, fis,
-        ir->get_field_names(ir, IR_TERM_VECTOR_WITH_OFFSET),
-        true, false, true);
-    sm_add_indexed(ir, fis, ir->get_field_names(ir, IR_TERM_VECTOR),
-        true, false, false);
-    sm_add_indexed(ir, fis, ir->get_field_names(ir, IR_INDEXED),
-        false, false, false);
-    fis_add_fields(fis, ir->get_field_names(ir, IR_UNINDEXED),
-        false, false, false, false, false);
-  }
-  fis_write(fis, sm->store, sm->name, ".fnm");
-
-  /* merge field values */
-  fw = fw_open(sm->store, sm->name, fis);
-
-  TRY
-    for (i = 0; i < sm->readers->size; i++) {
-      IndexReader *ir = sm->readers->elems[i];
-      maxdoc = ir->max_doc(ir);
-      for (j = 0; j < maxdoc; j++) {
-        if (!ir->is_deleted(ir, j)) { /* skip deleted docs */
-          doc = ir->get_doc(ir, j);
-          fw_add_doc(fw, doc);
-          doc_destroy(doc);
-          doc_count++;
-        }
-      }
-    }
-  XFINALLY
-    fw_close(fw);
-  XENDTRY
-  return doc_count;
-}
-
-void sm_reset_skip(SegmentMerger *sm)
-{
-  ramo_reset(sm->skip_buffer);
-  sm->last_skip_doc = 0;
-  sm->last_skip_freq_pointer = os_pos(sm->freq_out);
-  sm->last_skip_prox_pointer = os_pos(sm->prox_out);
-}
-
-inline void sm_buffer_skip(SegmentMerger *sm, int doc)
-{
-  int freq_pointer = os_pos(sm->freq_out);
-  int prox_pointer = os_pos(sm->prox_out);
-
-  os_write_vint(sm->skip_buffer, doc - sm->last_skip_doc);
-  os_write_vint(sm->skip_buffer, freq_pointer - sm->last_skip_freq_pointer);
-  os_write_vint(sm->skip_buffer, prox_pointer - sm->last_skip_prox_pointer);
-
-  sm->last_skip_doc = doc;
-  sm->last_skip_freq_pointer = freq_pointer;
-  sm->last_skip_prox_pointer = prox_pointer;
-}
-
-int sm_append_postings(SegmentMerger *sm, SegmentMergeInfo **smis, int cnt)
-{
-  int i, j;
-  int last_doc = 0, base, doc, doc_code, freq, last_position, position;
-  int *doc_map = NULL;
-  int df = 0;            /* number of docs w/ term */
-  TermDocEnum *postings;
-  SegmentMergeInfo *smi;
-  sm_reset_skip(sm);
-  for (i = 0; i < cnt; i++) {
-    smi = smis[i];
-    postings = smi->postings;
-    base = smi->base;
-    doc_map = smi_load_doc_map(smi);
-
-    stde_seek_ti(postings, smi->te->ti_curr);
-    while (postings->next(postings)) {
-      doc = postings->doc_num(postings);
-      if (doc_map != NULL) {
-        doc = doc_map[doc]; /* work around deletions */
-      }
-      doc += base;          /* convert to merged space */
-
-      if (doc < last_doc) {
-        RAISE(STATE_ERROR, DOC_ORDER_ERROR_MSG);
-      }
-
-      df++;
-
-      if ((df % sm->skip_interval) == 0) {
-        sm_buffer_skip(sm, last_doc);
-      }
-
-      doc_code = (doc - last_doc) << 1;    /* use low bit to flag freq=1 */
-      last_doc = doc;
-
-      freq = postings->freq(postings);
-      if (freq == 1) {
-        os_write_vint(sm->freq_out, doc_code | 1); /* write doc & freq=1 */
-      } else {
-        os_write_vint(sm->freq_out, doc_code); /* write doc */
-        os_write_vint(sm->freq_out, freq);     /* write freqency in doc */
-      }
-        
-
-      last_position = 0;        /* write position deltas */
-      for (j = 0; j < freq; j++) {
-        position = postings->next_position(postings);
-        os_write_vint(sm->prox_out, position - last_position);
-        last_position = position;
-      }
-    }
-  }
-  return df;
-}
-
-int sm_write_skip(SegmentMerger *sm)
-{
-  int skip_pointer = os_pos(sm->freq_out);
-  ramo_write_to(sm->skip_buffer, sm->freq_out);
-  return skip_pointer;
-}
-
-Term *sm_tb_to_term(SegmentMerger *sm, TermBuffer *tb)
-{
-  int index = sm->terms_buf_pointer % sm->terms_buf_size;
-  sm->terms_buf_pointer++;
-  sm->terms_buf[index].field = tb->field;
-  strcpy(sm->terms_buf[index].text, tb->text);
-  return &(sm->terms_buf[index]);
-}
-
-void sm_merge_term_info(SegmentMerger *sm, SegmentMergeInfo **smis, int cnt)
-{
-  int freq_pointer = os_pos(sm->freq_out);
-  int prox_pointer = os_pos(sm->prox_out);
-
-  int df = sm_append_postings(sm, smis, cnt);      /* append posting data */
-
-  int skip_pointer = sm_write_skip(sm);
-
-  if (df > 0) {
-    /* add an entry to the dictionary with pointers to prox and freq files */
-    ti_set(sm->ti, df, freq_pointer, prox_pointer, (skip_pointer - freq_pointer));
-    tiw_add(sm->tiw, sm_tb_to_term(sm, smis[0]->tb), sm->ti);
-  }
-}
-
-void sm_merge_term_infos(SegmentMerger *sm)
-{
-  int base = 0;
-  int i, match_size;
-  IndexReader *ir;
-  TermEnum *te;
-  SegmentMergeInfo *smi, *top, **match;
-  TermBuffer *tb;
-
-  for (i = 0; i < sm->readers->size; i++) {
-    ir = sm->readers->elems[i];
-    te = ir->terms(ir);
-    smi = smi_create(base, te, ir);
-    base += ir->num_docs(ir);
-    if (smi_next(smi) != NULL) {
-      pq_push(sm->queue, smi); /* initialize @queue */
-    } else {
-      smi_destroy(smi);
-    }
-  }
-
-  match = ALLOC_N(SegmentMergeInfo *, sm->readers->size);
-
-  while (sm->queue->count > 0) {
-    /*
-    for (i = 1; i <= sm->queue->count; i++) {
-      printf("<{%s:%s}>", ((SegmentMergeInfo *)sm->queue->heap[i])->tb->field,
-                          ((SegmentMergeInfo *)sm->queue->heap[i])->tb->text);
-    }printf("\n\n");
-    */
-    match_size = 0;     /* pop matching terms */
-    match[match_size] = pq_pop(sm->queue);
-    match_size++;
-    tb = match[0]->tb;
-    top = pq_top(sm->queue);
-    while ((top != NULL) && (tb_cmp(tb, top->tb) == 0)) {
-      match[match_size] = pq_pop(sm->queue);
-      match_size++;
-      top = pq_top(sm->queue);
-    }
-
-    /* printf(">%s:%s<\n", match[0]->tb->field, match[0]->tb->text); */
-    sm_merge_term_info(sm, match, match_size);      /* add new TermInfo */
-
-    while (match_size > 0) {
-      match_size--;
-      smi = match[match_size];
-      if (smi_next(smi) != NULL) {
-        pq_push(sm->queue, smi); /* restore queue */
-      } else {
-        smi_destroy(smi);        /* done with a segment */
-      }
-    }
-  }
-  free(match);
-}
-
-void sm_merge_terms(SegmentMerger *sm)
-{
-  int i;
-  char fname[SEGMENT_NAME_MAX_LENGTH];
-
-  TRY
-    sprintf(fname, "%s.frq", sm->name);
-    sm->freq_out = sm->store->create_output(sm->store, fname);
-    sprintf(fname, "%s.prx", sm->name);
-    sm->prox_out = sm->store->create_output(sm->store, fname);
-    sm->tiw = tiw_open(sm->store, sm->name, sm->fis, sm->term_index_interval);
-    /* terms_buf_pointer holds a buffer of terms since the TermInfosWriter needs
-     * to keep the last index_interval terms so that it can compare the last term
-     * put in the index with the next one. So the size of the buffer must by 
-     * index_interval + 2. */
-    sm->terms_buf_pointer = 0;
-    sm->terms_buf_size = sm->tiw->index_interval + 2;
-    sm->terms_buf = ALLOC_N(Term, sm->terms_buf_size);
-    for (i = 0; i < sm->terms_buf_size; i++) {
-      sm->terms_buf[i].field = NULL;
-      sm->terms_buf[i].text = ALLOC_N(char, MAX_WORD_SIZE);
-    }
-    sm->skip_interval = sm->tiw->skip_interval;
-    sm->queue = pq_create(sm->readers->size, (lt_ft)&smi_lt);
-
-    sm_merge_term_infos(sm);
-
-  XFINALLY
-    sm_close(sm);
-  XENDTRY
-}
-
-void sm_merge_norms(SegmentMerger *sm)
-{
-  int i, j, k, max_doc;
-  uchar *norm_buf;
-  FieldInfo *fi;
-  OutStream *os;
-  char fname[SEGMENT_NAME_MAX_LENGTH];
-  IndexReader *ir;
-  for (i = 0; i < sm->fis->fcnt; i++) {
-    fi = sm->fis->by_number[i];
-    if (fi->is_indexed && !fi->omit_norms)  {
-      sprintf(fname, "%s.f%d", sm->name, i);
-      os = sm->store->create_output(sm->store, fname);
-      TRY
-        for (j = 0; j < sm->readers->size; j++) {
-          ir = sm->readers->elems[j];
-          max_doc = ir->max_doc(ir);
-          norm_buf = ALLOC_N(uchar, max_doc);
-          memset(norm_buf, 0, sizeof(uchar) * max_doc);
-          ir->get_norms_into(ir, fi->name, norm_buf, 0);
-          for (k = 0; k < max_doc; k++) {
-            if (!ir->is_deleted(ir, k)) {
-              os_write_byte(os, norm_buf[k]);
-            }
-          }
-          free(norm_buf);
-        }
-      XFINALLY
-        os_close(os);
-      XENDTRY
-    }
-  }
-}
-
-void sm_merge_vectors(SegmentMerger *sm)
-{
-  int i, j, max_doc;
-  TermVectorsWriter *tvw = tvw_open(sm->store, sm->name, sm->fis);
-  IndexReader *ir;
-  Array *tvs;
-  TRY
-    for (i = 0; i < sm->readers->size; i++) {
-      ir = sm->readers->elems[i];
-      max_doc = ir->max_doc(ir);
-      for (j = 0; j < max_doc; j++) {
-        /* skip deleted docs */
-        if (! ir->is_deleted(ir, j)) {
-          tvs = ir->get_term_vectors(ir, j);
-          if (tvs) {
-            tvw_add_all_doc_vectors(tvw, tvs);
-            ary_destroy(tvs);
-          }
-        }
-      }
-    }
-  XFINALLY
-    tvw_close(tvw);
-  XENDTRY
-}
-
-int sm_merge(SegmentMerger *sm)
-{
-  int doc_count = sm_merge_fields(sm);
-  sm_merge_terms(sm);
-  sm_merge_norms(sm);
-  if (fis_has_vectors(sm->fis))
-    sm_merge_vectors(sm);
-  return doc_count;
-}
-
-Array *sm_create_compound_file(SegmentMerger *sm, char *file_name)
-{
-  Array *files = ary_create(0, &free);
-  CompoundWriter *cw = open_cw(sm->store, file_name);
-  FieldInfo *fi;
-  char fname[SEGMENT_NAME_MAX_LENGTH];
-
-  int i;
-  for (i = 0; i < NELEMS(COMPOUND_EXTENSIONS); i++) {
-    sprintf(fname, "%s.%s", sm->name, COMPOUND_EXTENSIONS[i]);
-    ary_append(files, estrdup(fname));
-  }
-
-  /* Field norm files */
-  for (i = 0; i < sm->fis->fcnt; i++) {
-    fi = sm->fis->by_number[i];
-    if (fi->is_indexed && !fi->omit_norms) {
-      sprintf(fname, "%s.f%d", sm->name, i);
-      ary_append(files, estrdup(fname));
-    }
-  }
-
-  /* Vector files */
-  if (fis_has_vectors(sm->fis)) {
-    for (i = 0; i < NELEMS(VECTOR_EXTENSIONS); i++) {
-      sprintf(fname, "%s.%s", sm->name, VECTOR_EXTENSIONS[i]);
-      ary_append(files, estrdup(fname));
-    }
-  }
-
-  /* Now merge all added files */
-  for (i = 0; i < files->size; i++) {
-    cw_add_file(cw, (char *)files->elems[i]);
-  }
-  
-  /* Perform the merge */
-  cw_close(cw);
- 
-  return files;
-}
-
-/****************************************************************************
- *
- * IndexReader
- *
- ****************************************************************************/
-
-void ir_acquire_not_necessary(IndexReader *ir) {}
-void ir_acquire_write_lock(IndexReader *ir)
-{
-  if (ir->is_stale)
-    RAISE(STATE_ERROR, STALE_READER_ERROR_MSG);
-
-  if (ir->write_lock == NULL) {
-    ir->write_lock = ir->store->open_lock(ir->store, WRITE_LOCK_NAME);
-    if (!ir->write_lock->obtain(ir->write_lock)) /* obtain write lock */
-      RAISE(STATE_ERROR, WRITE_LOCK_ERROR_MSG);
-
-    /* we have to check whether index has changed since this reader was opened.
-     * if so, this reader is no longer valid for deletion */
-    if (sis_read_current_version(ir->store) > ir->sis->version) {
-      ir->is_stale = true;
-      ir->write_lock->release(ir->write_lock);
-      ir->store->close_lock(ir->write_lock);
-      ir->write_lock = NULL;
-      RAISE(STATE_ERROR, STALE_READER_ERROR_MSG);
-    }
-  }
-}
-
-IndexReader *ir_create(Store *store, SegmentInfos *sis, int is_owner)
-{
-  IndexReader *ir = ALLOC(IndexReader);
-
-  mutex_init(&ir->mutex, NULL);
-  ir->is_owner = is_owner;
-  if (is_owner) {
-    ir->acquire_write_lock = &ir_acquire_write_lock;
-  } else {
-    ir->acquire_write_lock = &ir_acquire_not_necessary;
-  }
-
-  ir->store = store;
-  ir->sis = sis;
-  ir->has_changes = false;
-  ir->is_stale = false;
-  ir->write_lock = NULL;
-  ir->cache = NULL;
-  ir->sort_cache = NULL;
-  return ir;
-}
-
-/**
- * Will keep a reference to the store. To let this method delete the store
- * make sure you deref the store that you pass to it
- */
-IndexReader *ir_open(Store *store)
-{
-  int i;
-  IndexReader *ir;
-  SegmentInfos *sis;
-
-  mutex_lock(&store->mutex);
-  sis = sis_create();
-  sis_read(sis, store);
-  if (sis->scnt == 1) {
-    ir = sr_open(sis, 0, true);
-  } else {
-    IndexReader **readers = ALLOC_N(IndexReader *, sis->scnt);
-    for (i = 0; i < sis->scnt; i++) {
-      readers[i] = sr_open(sis, i, false);
-    }
-    ref(store);
-    ir = mr_open(store, sis, readers, sis->scnt);
-  }
-  mutex_unlock(&store->mutex);
-  return ir;
-}
-
-bool ir_index_exists(Store *store)
-{
-  return store->exists(store, "segments");
-}
-
-void ir_set_norm(IndexReader *ir, int doc_num, char *field, uchar val)
-{
-  mutex_lock(&ir->mutex);
-  ir->acquire_write_lock(ir);
-  ir->do_set_norm(ir, doc_num, field, val);
-  ir->has_changes = true;
-  mutex_unlock(&ir->mutex);
-}
-
-void ir_undelete_all(IndexReader *ir)
-{
-  mutex_lock(&ir->mutex);
-  ir->acquire_write_lock(ir);
-  ir->do_undelete_all(ir);
-  ir->has_changes = true;
-  mutex_unlock(&ir->mutex);
-}
-
-void ir_delete_doc(IndexReader *ir, int doc_num)
-{
-  mutex_lock(&ir->mutex);
-  ir->acquire_write_lock(ir);
-  ir->do_delete_doc(ir, doc_num);
-  ir->has_changes = true;
-  mutex_unlock(&ir->mutex);
-}
-
-Document *ir_get_doc_with_term(IndexReader *ir, Term *term)
-{
-  TermDocEnum *tde = ir_term_docs_for(ir, term);
-  Document *doc = NULL;
-
-  if (!tde) return NULL;
-
-  if (tde->next(tde)) {
-    doc = ir->get_doc(ir, tde->doc_num(tde));
-  }
-  tde->close(tde);
-  return doc;
-}
-
-TermDocEnum *ir_term_docs_for(IndexReader *ir, Term *term)
-{
-  TermDocEnum *tde = ir->term_docs(ir);
-  tde->seek(tde, term);
-  return tde;
-}
-
-TermDocEnum *ir_term_positions_for(IndexReader *ir, Term *term)
-{
-  TermDocEnum *tde = ir->term_positions(ir);
-  tde->seek(tde, term);
-  return tde;
-}
-
-void ir_commit_internal(IndexReader *ir)
-{
-  if (ir->has_changes) {
-    if (ir->is_owner) {
-      Lock *commit_lock;
-
-      mutex_lock(&ir->store->mutex);
-      commit_lock = ir->store->open_lock(ir->store, COMMIT_LOCK_NAME);
-      if (!commit_lock->obtain(commit_lock)) { /* obtain write lock */
-        RAISE(STATE_ERROR, COMMIT_LOCK_ERROR_MSG);
-      }
-
-      ir->do_commit(ir);
-      sis_write(ir->sis, ir->store);
-
-      commit_lock->release(commit_lock);
-      ir->store->close_lock(commit_lock);
-      mutex_unlock(&ir->store->mutex);
-
-      if (ir->write_lock != NULL) {
-        ir->write_lock->release(ir->write_lock);  /* release write lock */
-        ir->store->close_lock(ir->write_lock);
-        ir->write_lock = NULL;
-      }
-    } else {
-      ir->do_commit(ir);
-    }
-    ir->has_changes = false;
-  }
-}
-
-void ir_commit(IndexReader *ir)
-{
-  mutex_lock(&ir->mutex);
-  ir_commit_internal(ir);
-  mutex_unlock(&ir->mutex);
-}
-
-void ir_close(IndexReader *ir)
-{
-  mutex_lock(&ir->mutex);
-  ir_commit_internal(ir);
-  ir->do_close(ir);
-  store_deref(ir->store);
-  if (ir->is_owner) {
-    sis_destroy(ir->sis);
-  }
-  if (ir->cache) {
-    h_destroy(ir->cache);
-  }
-  if (ir->sort_cache) {
-    h_destroy(ir->sort_cache);
-  }
-
-  mutex_destroy(&ir->mutex);
-  free(ir);
-}
-
-/**
- * Don't call this method if the cache already exists
- **/
-void ir_add_cache(IndexReader *ir)
-{
-  ir->cache = co_hsh_create();
-}
-
-bool ir_is_latest(IndexReader *ir)
-{
-  bool is_latest = false;
-  Lock *commit_lock = ir->store->open_lock(ir->store, COMMIT_LOCK_NAME);
-  if (!commit_lock->obtain(commit_lock)) {
-    ir->store->close_lock(commit_lock);
-    RAISE(STATE_ERROR, COMMIT_LOCK_ERROR_MSG);
-  }
-  TRY
-    is_latest = (sis_read_current_version(ir->store) == ir->sis->version);
-  XFINALLY
-    commit_lock->release(commit_lock);
-    ir->store->close_lock(commit_lock);
-  XENDTRY
-  return is_latest;
-}
-