ferret 0.3.2 → 0.9.0

data/ext/index_rw.c

@@ -0,0 +1,2543 @@
+#include <index.h>
+#include <stdlib.h>
+#include <string.h>
+#include <array.h>
+
+const char *INDEX_EXTENSIONS[] = {
+    "cfs", "fnm", "fdx", "fdt", "tii", "tis", "frq", "prx", "del",
+    "tvx", "tvd", "tvf", "tvp"
+};
+
+const char *COMPOUND_EXTENSIONS[] = {
+        "fnm", "frq", "prx", "fdx", "fdt", "tii", "tis"
+};
+      
+const char *VECTOR_EXTENSIONS[] = {
+        "tvx", "tvd", "tvf"
+};
+      
+FerretConfig config = {
+  10,       // default merge_factor
+  10,       // default min_merge_docs
+  INT_MAX,  // default max_merge_docs
+  10000,    // default max_field_length
+  128       // default term_index_interval
+};
+
+/***************************************************************************
+ *
+ * CacheObject
+ *
+ ***************************************************************************/
+
+unsigned int co_hash(const void *key)
+{
+  return (unsigned int)key;
+}
+
+int co_eq(const void *key1, const void *key2)
+{
+  return (key1 == key2);
+}
+
+void co_destroy(void *p)
+{
+  CacheObject *co = (CacheObject *)p;
+  h_rem(co->ref_tab1, co->ref2, false);
+  h_rem(co->ref_tab2, co->ref1, false);
+  co->destroy(co->obj);
+  free(co);
+}
+
+CacheObject *co_create(HshTable *ref_tab1, HshTable *ref_tab2,
+    void *ref1, void *ref2, void (*destroy)(void *p), void *obj)
+{
+  CacheObject *co = ALLOC(CacheObject);
+  h_set(ref_tab1, ref2, co);
+  h_set(ref_tab2, ref1, co);
+  co->ref_tab1 = ref_tab1;
+  co->ref_tab2 = ref_tab2;
+  co->ref1 = ref1;
+  co->ref2 = ref2;
+  co->destroy = destroy;
+  co->obj = obj;
+  return co;
+}
+
+HshTable *co_hsh_create()
+{
+  return h_new(&co_hash, &co_eq, NULL, &co_destroy);
+}
+
+/***************************************************************************
+ *
+ * Posting
+ *
+ ***************************************************************************/
+
+Posting *p_create(Term *term, int position, TVOffsetInfo *offset)
+{
+  Posting *p = ALLOC(Posting);
+  p->freq = 1;
+  p->size = 1;
+  p->term = term;
+  p->positions = ALLOC(int);
+  p->positions[0] = position;
+  p->offsets = ALLOC(TVOffsetInfo *);
+  p->offsets[0] = offset;
+  return p;
+}
+
+void p_destroy(void *p)
+{
+  // the positions and offsets will be put in a TVTerm so no need to free
+  int i;
+  Posting *post = (Posting *)p;
+  free(post->positions);
+  for (i = 0; i < post->freq; i++)
+    tvoi_destroy(post->offsets[i]);
+  free(post->offsets);
+  free(p);
+}
+
+void p_add_occurance(Posting *p, int position, TVOffsetInfo *offset)
+{
+  if (p->freq >= p->size) {
+    p->size *= 2;
+    REALLOC_N(p->positions, int, p->size);
+    REALLOC_N(p->offsets, TVOffsetInfo *, p->size);
+  }
+  p->positions[p->freq] = position;
+  p->offsets[p->freq] = offset;
+  p->freq++;
+}
+
+inline int p_cmp(const void *const p1, const void *const p2)
+{
+  Term *t1 = (*(Posting **)p1)->term;
+  Term *t2 = (*(Posting **)p2)->term;
+  int res = strcmp(t1->field, t2->field);
+  if (res != 0) {
+    return res;
+  } else {
+    return strcmp(t1->text, t2->text);
+  }
+}
+
+DocumentWriter *dw_open(Store *store,
+                        Analyzer *analyzer,
+                        Similarity *similarity,
+                        int max_field_length,
+                        int term_index_interval)
+{
+  DocumentWriter *dw = ALLOC(DocumentWriter);
+  dw->store = store;
+  dw->analyzer = analyzer;
+  dw->similarity = similarity;
+  dw->fis = NULL;
+  dw->postingtable = h_new(&term_hash, &term_eq, &term_destroy, &p_destroy);
+  dw->max_field_length = max_field_length;
+  dw->term_index_interval = term_index_interval;
+  return dw;
+}
+
+void dw_close(DocumentWriter *dw)
+{
+  if (dw->fis) fis_destroy(dw->fis);
+  h_destroy(dw->postingtable);
+  free(dw);
+}
+
+void dw_add_position(DocumentWriter *dw, char *field, char *text,
+    int position, TVOffsetInfo *offset) 
+{
+  Term termbuf = {field, text}, *term;
+  Posting *p = (Posting *)h_get(dw->postingtable, &termbuf);
+
+  if (p) { // word seen before
+    // double the size of posting to make room for more posts.
+    if (p->freq >= p->size) {
+      p->size <<= 1;
+      REALLOC_N(p->positions, int, p->size);
+      p->offsets = REALLOC_N(p->offsets, TVOffsetInfo *, p->size);
+    }
+    p->positions[p->freq] = position;    // add new position
+    p->offsets[p->freq] = offset;        // add new position
+    p->freq++;                        // update frequency
+  } else { // word not seen before
+    term = term_create(field, text);
+    h_set(dw->postingtable, term, p_create(term, position, offset));
+  }
+}
+
+void dw_invert_doc(DocumentWriter *dw, Document *doc)
+{
+  int i;
+  int dfcnt = doc->dfcnt;
+  char *field_name, *text;
+  int field_number, length, position, offset, slen;
+  TokenStream *stream;
+  Token *token;
+  FieldInfo *fi;
+
+  DocField **fields = doc->df_arr, *field;
+  for (i = 0; i < dfcnt; i++) {
+    field = fields[i];
+    field_name = field->name;
+    fi = ((FieldInfo *)ht_get(dw->fis->by_name, field_name));
+    field_number = fi->number;
+
+    length = dw->field_lengths[field_number];
+    offset = dw->field_offsets[field_number];
+    position = dw->field_positions[field_number];
+
+    if (fi->is_indexed) {
+      if (!field->is_tokenized) {// un-tokenized field
+        text = field->data;
+        slen = strlen(text);
+        if (fi->store_offset) {
+          dw_add_position(dw, field_name, text, position,
+              tvoi_create(offset, offset+slen));
+        } else {
+          dw_add_position(dw, field_name, text, position, NULL);
+        }
+        offset += slen;
+        length++;
+      } else {
+
+        // Tokenize field and add to posting_table
+        stream = a_get_ts(dw->analyzer, field_name, field->data);
+
+        while ((token = ts_next(stream)) != NULL) {
+          position += (token->pos_inc - 1);
+
+          if (fi->store_offset) {
+            dw_add_position(dw, 
+                field_name,
+                token->text,
+                position,
+                tvoi_create(offset + token->start, offset + token->end));
+            position++;
+          } else {
+            dw_add_position(dw, field_name, token->text, position, NULL);
+            position++;
+          }
+
+          length++;
+          // stop if we reach the max field length
+          if (length > dw->max_field_length)
+            break;
+        }
+
+        if (token)
+          offset += token->end + 1;
+      }
+      dw->field_lengths[field_number] = length;
+      dw->field_offsets[field_number] = offset;
+      dw->field_positions[field_number] = position;
+      dw->field_boosts[field_number] *= field->boost;
+    }
+  }
+}
+
+Posting **dw_sort_posting_table(DocumentWriter *dw)
+{
+  HshTable *ht = dw->postingtable;
+  int i;
+  dw->pcnt = i = ht->used;
+  Posting **postings = ALLOC_N(Posting *, i);
+  HshEntry *he = ht->table;
+  while (i > 0) {
+    if (he->value != NULL) {
+      i--;
+      postings[i] = (Posting *)he->value;
+    }
+    he++;
+  }
+  qsort(postings, dw->pcnt, sizeof(Posting *), &p_cmp);
+  return postings;
+}
+
+void dw_write_postings(DocumentWriter *dw, Posting **postings, char *segment)
+{
+  OutStream *freq_out, *prox_out;
+  TermInfosWriter *tiw;
+  TermVectorsWriter *tvw = NULL;
+  FieldInfo *fi;
+  Store *store = dw->store;
+  TermInfo *ti;
+  Posting *posting;
+  int i, j, posting_freq, position, last_position;
+  char fname[SEGMENT_NAME_MAX_LENGTH], *curr_field = NULL, *term_field;
+  strcpy(fname, segment);
+
+  //open files for inverse index storage
+  sprintf(fname, "%s.frq", segment);
+  freq_out = store->create_output(store, fname);
+  sprintf(fname, "%s.prx", segment);
+  prox_out = store->create_output(store, fname);
+  tiw = tiw_open(store, segment, dw->fis, dw->term_index_interval);
+  ti = ti_create(0, 0, 0, 0);
+
+  for (i = 0; i < dw->pcnt; i++) {
+    posting = postings[i];
+
+    // add an entry to the dictionary with pointers to prox and freq_out files
+    ti_set(ti, 1, os_pos(freq_out), os_pos(prox_out), -1);
+    tiw_add(tiw, posting->term, ti);
+
+    // add an entry to the freq_out file
+    posting_freq = posting->freq;
+    if (posting_freq == 1) {         // optimize freq=1
+      os_write_vint(freq_out, 1);            // set low bit of doc num.
+    } else {
+      os_write_vint(freq_out, 0);            // the doc number
+      os_write_vint(freq_out, posting_freq); // frequency in doc
+    }
+
+    last_position = 0;               // write positions
+
+    for (j = 0; j < posting_freq; j++) {
+      position = posting->positions[j];
+      os_write_vint(prox_out, position - last_position);
+      last_position = position;
+    }
+
+    // check to see if we switched to a new field
+    term_field = posting->term->field;
+    if (curr_field != term_field)  {
+      // changing field - see if there is something to save
+      curr_field = term_field;
+      fi = (FieldInfo *)ht_get(dw->fis->by_name, curr_field);
+      if (fi->store_tv) {
+        if (tvw == NULL) {
+          tvw = tvw_open(store, segment, dw->fis);
+          tvw_open_doc(tvw);
+        }
+        tvw_open_field(tvw, curr_field);
+
+      } else if (tvw != NULL) {
+        tvw_close_field(tvw);
+      }
+    }
+    // tvw->curr_field != NULL implies field is still open
+    if (tvw != NULL && tvw->curr_field != NULL) {
+      tvw_add_term(tvw, posting->term->text, posting_freq, posting->positions, posting->offsets);
+    }
+  }
+  if (tvw != NULL) {
+    tvw_close_doc(tvw);
+    tvw_close(tvw);
+  }
+  // make an effort to close all streams we can but remember and re-raise
+  // the last exception encountered in this process
+  os_close(freq_out);
+  os_close(prox_out);
+  tiw_close(tiw);
+  ti_destroy(ti);
+}
+
+void dw_write_norms(DocumentWriter *dw, char *segment)
+{
+  int i;
+  float norm;
+  OutStream *norms_out;
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+  FieldInfos *fis = dw->fis;
+  FieldInfo *fi;
+
+  for (i = 0; i < fis->fcnt; i++) {
+    fi = fis->by_number[i];
+    
+    if (fi->is_indexed && !fi->omit_norms) {
+      norm = dw->field_boosts[i] * sim_length_norm(dw->similarity, fi->name, dw->field_lengths[i]);
+      sprintf(fname, "%s.f%d", segment, i);
+      norms_out = dw->store->create_output(dw->store, fname);
+      os_write_byte(norms_out, sim_encode_norm(dw->similarity, norm));
+      os_close(norms_out);
+    }
+  }
+}
+
+void dw_add_doc(DocumentWriter *dw, char *segment, Document *doc)
+{
+  int i;
+  // write field names
+  dw->fis = fis_create();
+  fis_add_doc(dw->fis, doc);
+  fis_write(dw->fis, dw->store, segment, ".fnm");
+
+  // write field values
+  FieldsWriter *fw = fw_open(dw->store, segment, dw->fis);
+  fw_add_doc(fw, doc);
+  fw_close(fw);
+
+  // invert doc into posting_table
+  h_clear(dw->postingtable);        // clear posting_table
+
+  dw->field_boosts = ALLOC_N(float, dw->fis->fcnt);
+  dw->field_lengths = ALLOC_AND_ZERO_N(int, dw->fis->fcnt);
+  dw->field_offsets = ALLOC_AND_ZERO_N(int, dw->fis->fcnt);
+  dw->field_positions = ALLOC_AND_ZERO_N(int, dw->fis->fcnt);
+
+  for (i = 0; i < dw->fis->fcnt; i++)
+    dw->field_boosts[i] = doc->boost;
+
+  dw_invert_doc(dw, doc);
+
+  // sort posting_table into an array
+  Posting **postings = dw_sort_posting_table(dw);
+
+  // write postings
+  dw_write_postings(dw, postings, segment);
+  free(postings);
+
+  // write norms of indexed fields
+  dw_write_norms(dw, segment);
+
+  free(dw->field_boosts);
+  free(dw->field_lengths);
+  free(dw->field_offsets);
+  free(dw->field_positions);
+}
+
+/****************************************************************************
+ *
+ * SegmentInfo
+ *
+ ****************************************************************************/
+
+SegmentInfo *si_create(char *name, int doc_cnt, Store *store)
+{
+  SegmentInfo *si = ALLOC(SegmentInfo);
+  si->name = name;
+  si->doc_cnt = doc_cnt;
+  si->store = store;
+  return si;
+}
+
+void si_destroy(void *p)
+{
+  SegmentInfo *si = (SegmentInfo *)p;
+  free(si->name);
+  free(si);
+}
+
+bool si_has_deletions(SegmentInfo *si)
+{
+  char del_file_name[SEGMENT_NAME_MAX_LENGTH];
+  sprintf(del_file_name, "%s.del", si->name);
+  return si->store->exists(si->store, del_file_name);
+}
+
+bool si_uses_compound_file(SegmentInfo *si)
+{
+  char compound_file_name[SEGMENT_NAME_MAX_LENGTH];
+  sprintf(compound_file_name, "%s.cfs", si->name);
+  return si->store->exists(si->store, compound_file_name);
+}
+
+struct NormTester {
+  bool has_norm_file;
+  char *segment_name;
+};
+void is_norm_file(char *fname, void *arg)
+{
+  struct NormTester *nt = (struct NormTester *)arg;
+  char norm_file_pattern[SEGMENT_NAME_MAX_LENGTH];
+  sprintf(norm_file_pattern, "%s.s", nt->segment_name);
+  if (strncmp(fname, norm_file_pattern, strlen(norm_file_pattern)) == 0) {
+    nt->has_norm_file = true;
+  }
+}
+
+bool si_has_separate_norms(SegmentInfo *si)
+{
+  struct NormTester nt;
+  nt.segment_name = si->name;
+  nt.has_norm_file = false;
+  si->store->each(si->store, &is_norm_file, &nt);
+
+  return nt.has_norm_file;
+}
+
+
+/****************************************************************************
+ *
+ * SegmentInfos
+ *
+ ****************************************************************************/
+
+#include <time.h>
+#define FORMAT -1
+#define SEGMENT_FILENAME "segments"
+#define TEMPORARY_SEGMENT_FILENAME "segments.new"
+
+SegmentInfos *sis_create()
+{
+  SegmentInfos *sis = ALLOC(SegmentInfos);
+  sis->format = FORMAT;
+  sis->version = (unsigned int)time(NULL);
+  sis->scnt = 0;
+  sis->counter = 0;
+  sis->size = 4;
+  sis->segs = ALLOC_N(SegmentInfo *, sis->size);
+  return sis;
+}
+
+void sis_destroy_not_infos(void *p)
+{
+  SegmentInfos *sis = (SegmentInfos *)p;
+  free(sis->segs);
+  free(p);
+}
+
+void sis_destroy(void *p)
+{
+  int i;
+  SegmentInfos *sis = (SegmentInfos *)p;
+  for (i = 0; i < sis->scnt; i++)
+    si_destroy(sis->segs[i]);
+  free(sis->segs);
+  free(p);
+}
+
+void sis_add_si(SegmentInfos *sis, SegmentInfo *si)
+{
+  if (sis->scnt >= sis->size) {
+    sis->size = sis->scnt * 2;
+    REALLOC_N(sis->segs, SegmentInfo *, sis->size);
+  }
+  sis->segs[sis->scnt] = si;
+  sis->scnt++;
+}
+
+void sis_del_at(SegmentInfos *sis, int at)
+{
+  int i;
+  si_destroy(sis->segs[at]);
+  sis->scnt--;
+  for (i = at; i < sis->scnt; i++)
+    sis->segs[i] = sis->segs[i+1];
+}
+
+void sis_del_from_to(SegmentInfos *sis, int from, int to)
+{
+  int i, num_to_del = to - from;
+  sis->scnt -= num_to_del;
+  for (i = from; i < to; i++) {
+    si_destroy(sis->segs[i]);
+  }
+  for (i = from; i < sis->scnt; i++) {
+    sis->segs[i] = sis->segs[i+num_to_del];
+  }
+}
+
+void sis_clear(SegmentInfos *sis)
+{
+  int i;
+  for (i = 0; i < sis->scnt; i++) {
+    si_destroy(sis->segs[i]);
+  }
+  sis->scnt = 0;
+}
+
+void sis_read(SegmentInfos *sis, Store *store)
+{
+  int doc_cnt;
+  char *name;
+  InStream *is = store->open_input(store, SEGMENT_FILENAME);
+  sis->format = is_read_int(is);
+
+  if (sis->format < 0) { // file contains explicit format info
+    // check that it is a format we can understand
+    if (sis->format < FORMAT)
+      eprintf(ERROR, "Unknown format version: %ld", sis->format);
+    sis->version = is_read_long(is);
+    sis->counter = is_read_int(is);
+  } else { // file is in old format without explicit format info
+    sis->counter = sis->format;
+  }
+    
+  int seg_count = is_read_int(is);
+  int i;
+  for (i = 0; i < seg_count; i++) {
+    name = is_read_string(is);
+    doc_cnt = is_read_int(is);
+    sis_add_si(sis, si_create(name, doc_cnt, store));
+  }
+ 
+  if (sis->format >= 0) {
+    // in old format the version number may be at the end of the file
+    if (is_pos(is) >= is_length(is))
+      sis->version = 0; // old file format without version number
+    else
+      sis->version = is_read_long(is); // read version
+  }
+  is_close(is);
+}
+
+void sis_write(SegmentInfos *sis, Store *store)
+{
+  int i;
+  SegmentInfo *si;
+  OutStream *os = store->create_output(store, TEMPORARY_SEGMENT_FILENAME);
+  os_write_int(os, FORMAT);
+  os_write_long(os, ++(sis->version)); // every write changes the index
+  os_write_int(os, sis->counter);
+  os_write_int(os, sis->scnt); 
+  for (i = 0; i < sis->scnt; i++) {
+    si = sis->segs[i];
+    os_write_string(os, si->name);
+    os_write_int(os, si->doc_cnt);
+  }
+  
+  os_close(os);
+
+  //install new segment info
+  store->rename(store, TEMPORARY_SEGMENT_FILENAME, SEGMENT_FILENAME);
+}
+
+int sis_read_current_version(Store *store)
+{
+  if (!store->exists(store, SEGMENT_FILENAME))
+    return 0;
+  InStream *is = store->open_input(store, SEGMENT_FILENAME);
+  int format = 0;
+  int version = 0;
+  format = is_read_int(is);
+  if (format < 0) {
+    if (format < FORMAT) 
+      eprintf(ERROR, "Unknown format version: %ld", format);
+    version = is_read_long(is);
+  }
+  is_close(is);
+   
+  if (format < 0)
+    return version;
+
+  // We cannot be sure about the format of the file.
+  // Therefore we have to read the whole file and cannot simply
+  // seek to the version entry.
+
+  SegmentInfos *sis = sis_create();
+  sis_read(sis, store);
+  version = sis->version;
+  sis_destroy(sis);
+  return version;
+}
+
+/****************************************************************************
+ *
+ * IndexWriter
+ *
+ ****************************************************************************/
+
+IndexWriter *iw_open(Store *store, Analyzer *analyzer,
+    bool create, bool close_store, bool close_analyzer)
+{
+  IndexWriter *iw = ALLOC(IndexWriter);
+  if (create)
+    store->clear_all(store);
+  mutex_init(&iw->mutex, NULL);
+  iw->merge_factor = config.merge_factor;
+  iw->min_merge_docs = config.min_merge_docs;
+  iw->max_merge_docs = config.max_merge_docs;
+  iw->max_field_length = config.max_field_length;
+  iw->term_index_interval = config.term_index_interval;
+  iw->use_compound_file = true;
+  iw->store = store;
+  iw->close_store = close_store;
+  iw->close_analyzer = close_analyzer;
+  iw->analyzer = analyzer;
+  iw->sis = sis_create();
+  iw->similarity = sim_create_default();
+  iw->ram_store = open_ram_store();
+
+  mutex_lock(&store->mutex);
+  // keep the write_lock obtained until the IndexWriter is closed.
+  iw->write_lock = store->open_lock(store, WRITE_LOCK_NAME);
+  if (!iw->write_lock->obtain(iw->write_lock)) {
+    eprintf(STATE_ERROR,
+        "Could not obtain write lock when trying to write index");
+  }
+
+  if (create) {
+    Lock *commit_lock = store->open_lock(store, COMMIT_LOCK_NAME);
+    if (!commit_lock->obtain(commit_lock)) {
+      eprintf(STATE_ERROR,
+          "Could not obtain commit lock when trying to commit index");
+    }
+    // commit the index
+    store->clear(store);
+    sis_write(iw->sis, store);
+    //
+    commit_lock->release(commit_lock);
+    store->close_lock(commit_lock);
+  } else {
+    sis_read(iw->sis, store);
+  }
+  mutex_unlock(&store->mutex);
+  return iw;
+}
+
+const char base36_digitmap[] = "0123456789abcdefghijklmnopqrstuvwxyz";
+
+char *new_segment_name(int counter) 
+{
+  char buf[SEGMENT_NAME_MAX_LENGTH];
+  buf[SEGMENT_NAME_MAX_LENGTH - 1] = '\0';
+  int i;
+  for (i = SEGMENT_NAME_MAX_LENGTH - 2; ; i--) {
+    buf[i] = base36_digitmap[counter%36];
+    counter /= 36;
+    if (counter == 0) break;
+  }
+  i--;
+  buf[i] = '_';
+  return estrdup(&buf[i]);
+}
+
+int iw_doc_count(IndexWriter *iw)
+{
+  int i, doc_cnt = 0;
+  mutex_lock(&iw->mutex);
+  for (i = 0; i < iw->sis->scnt; i++)
+    doc_cnt += iw->sis->segs[i]->doc_cnt;
+  mutex_unlock(&iw->mutex);
+  return doc_cnt;
+}
+
+void delete_files(Array *file_names, Store *store)
+{
+  int i;
+  for (i = 0; i < file_names->size; i++) {
+    store->remove(store, (char *)file_names->elems[i]);
+  }
+  ary_destroy(file_names);
+}
+
+
+Array *sr_file_names(IndexReader *ir);
+void iw_delete_segments(IndexWriter *iw, IndexReader **segment_readers, int del_cnt)
+{
+  // The java version keeps a record of files that it couldn't delete. This
+  // shouldn't be a problem on linux I hope.
+  IndexReader *ir;
+  int i;
+  for (i = 0; i < del_cnt; i++) {
+    ir = segment_readers[i];
+    delete_files(sr_file_names(ir), ir->store);
+  }
+}
+
+void make_compound_file(IndexWriter *iw, char *merged_name, SegmentMerger *merger)
+{
+  char merged_tmp[SEGMENT_NAME_MAX_LENGTH], merged_cfs[SEGMENT_NAME_MAX_LENGTH];
+
+  mutex_lock(&iw->store->mutex);
+  sprintf(merged_tmp, "%s.tmp", merged_name);
+  sprintf(merged_cfs, "%s.cfs", merged_name);
+
+  Array *files_to_delete = sm_create_compound_file(merger, merged_tmp);
+  Lock *commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
+
+  if (!commit_lock->obtain(commit_lock)) {
+    eprintf(STATE_ERROR,
+        "Could not obtain commit lock when  trying to commit index");
+  }
+
+  // make compound file visible for SegmentReaders
+  iw->store->rename(iw->store, merged_tmp, merged_cfs);
+  // delete now unused files of segment
+  delete_files(files_to_delete, iw->store);
+
+  commit_lock->release(commit_lock);
+  iw->store->close_lock(commit_lock);
+  mutex_unlock(&iw->store->mutex);
+}
+
+void iw_merge_segments_with_max(IndexWriter *iw, int min_segment, int max_segment)
+{
+  int i;
+  IndexReader *segments_to_delete[max_segment - min_segment];
+  int del_cnt = 0;
+
+  char *merged_name = new_segment_name(iw->sis->counter++);
+
+  SegmentMerger *merger = sm_create(iw->store, merged_name, iw->term_index_interval);
+  IndexReader *reader;
+
+
+  for (i = min_segment; i < max_segment; i++) {
+    reader = sr_open(iw->sis, i, false, false);
+    sm_add(merger, reader);
+    if ((reader->store == iw->store) || // if we own the directory
+        (reader->store == iw->ram_store)) {
+      segments_to_delete[del_cnt++] = reader;   // queue segment for deletion
+    }
+  }
+  
+  int merged_doc_count = sm_merge(merger);
+
+  sis_del_from_to(iw->sis, min_segment, max_segment);
+
+  sis_add_si(iw->sis, si_create(merged_name, merged_doc_count, iw->store));
+
+  // close readers before we attempt to delete now-obsolete segments
+
+  mutex_lock(&iw->store->mutex);
+  Lock *commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
+  if (!commit_lock->obtain(commit_lock)) {
+    eprintf(STATE_ERROR,
+        "Could not obtain commit lock when trying to commit index");
+  }
+  // commit the index
+  sis_write(iw->sis, iw->store);
+  iw_delete_segments(iw, segments_to_delete, del_cnt);
+  //
+  commit_lock->release(commit_lock);
+  iw->store->close_lock(commit_lock);
+  mutex_unlock(&iw->store->mutex);
+
+  if (iw->use_compound_file) {
+    make_compound_file(iw, merged_name, merger);
+  }
+
+  sm_destroy(merger);
+}
+
+void iw_merge_segments(IndexWriter *iw, int min_segment)
+{
+  iw_merge_segments_with_max(iw, min_segment, iw->sis->scnt);
+}
+
+void iw_maybe_merge_segments(IndexWriter *iw)
+{
+  int target_merge_docs = iw->min_merge_docs;
+  int min_segment, merge_docs;
+  SegmentInfo *si;
+
+  while (target_merge_docs <= iw->max_merge_docs) {
+    // find segments smaller than current target size
+    min_segment = iw->sis->scnt - 1;
+    merge_docs = 0;
+    while (min_segment >= 0) {
+      si = iw->sis->segs[min_segment];
+      if (si->doc_cnt >= target_merge_docs)
+        break;
+      merge_docs += si->doc_cnt;
+      min_segment -= 1;
+    }
+
+    if (merge_docs >= target_merge_docs)      // found a merge to do
+      iw_merge_segments(iw, min_segment + 1);
+    else
+      break;
+
+    target_merge_docs *= iw->merge_factor; // increase target size
+  }
+}
+
+void iw_flush_ram_segments(IndexWriter *iw)
+{
+  int min_segment = iw->sis->scnt-1;
+  int doc_count = 0;
+  SegmentInfo **segs = iw->sis->segs;
+  while ((min_segment >= 0) &&
+          (segs[min_segment]->store == iw->ram_store)) {
+    doc_count += segs[min_segment]->doc_cnt;
+    min_segment--;
+  }
+  /* the following if statement is actually incrementing for different
+   * reasons. If min_segment < 0 then we must increment as we searched
+   * off the end. If the top segment is not ram_store there are no 
+   * ram segments to flush so we increment so the next check will return
+   * us from this function. Lastly, the min_segment stopped at a segment
+   * that wasn't the ram segment. But if it fit's in with the merge
+   * factor, why not merge it. Otherwise we leave it and increment min_seg
+   */
+  if (min_segment < 0 ||                          // add one FS segment?
+      (doc_count + segs[min_segment]->doc_cnt) > iw->merge_factor ||
+      (segs[iw->sis->scnt-1]->store != iw->ram_store))
+    min_segment++;
+  if (min_segment >= iw->sis->scnt)
+    return;
+  iw_merge_segments(iw, min_segment);
+}
+
+void iw_add_doc(IndexWriter *iw, Document *doc)
+{
+  DocumentWriter *dw;
+  char *segment_name;
+
+  mutex_lock(&iw->mutex);
+  dw = dw_open(iw->ram_store,
+               iw->analyzer,
+               iw->similarity,
+               iw->max_field_length,
+               iw->term_index_interval);
+  segment_name = new_segment_name(iw->sis->counter++);
+  dw_add_doc(dw, segment_name, doc);
+  dw_close(dw);
+  sis_add_si(iw->sis, si_create(segment_name, 1, iw->ram_store));
+  iw_maybe_merge_segments(iw);
+  mutex_unlock(&iw->mutex);
+}
+
+static inline void iw_optimize_internal(IndexWriter *iw)
+{
+  int min_segment;
+  iw_flush_ram_segments(iw);
+  while (iw->sis->scnt > 1 ||
+          (iw->sis->scnt == 1 &&
+            (  si_has_deletions(iw->sis->segs[0]) ||
+              (iw->sis->segs[0]->store != iw->store) ||
+              (iw->use_compound_file &&
+                (!si_uses_compound_file(iw->sis->segs[0]) ||
+                  si_has_separate_norms(iw->sis->segs[0])))))) {
+    min_segment = iw->sis->scnt - iw->merge_factor;
+    iw_merge_segments(iw, min_segment < 0 ? 0 : min_segment);
+  }
+}
+void iw_optimize(IndexWriter *iw)
+{
+  mutex_lock(&iw->mutex);
+  iw_optimize_internal(iw);
+  mutex_unlock(&iw->mutex);
+}
+
+void iw_close(IndexWriter *iw)
+{
+  mutex_lock(&iw->mutex);
+  iw_flush_ram_segments(iw);
+  ram_close(iw->ram_store);
+  sis_destroy(iw->sis);
+
+  sim_destroy(iw->similarity);
+  if (iw->close_analyzer) a_destroy(iw->analyzer);
+
+  iw->write_lock->release(iw->write_lock);
+  iw->store->close_lock(iw->write_lock);
+
+  if (iw->close_store)
+    store_close(iw->store);
+  mutex_destroy(&iw->mutex);
+  free(iw);
+}
+
+void iw_add_indexes(IndexWriter *iw, Store **stores, int cnt)
+{
+  int i, j, end, start;
+
+  mutex_lock(&iw->mutex);
+  iw_optimize_internal(iw); // start with zero or 1 seg
+
+  start = iw->sis->scnt;
+
+  for (i = 0; i < cnt; i++) {
+    Store *store = stores[i];
+    SegmentInfos *sis = sis_create(); // read infos from dir
+    sis_read(sis, store);
+
+    for (j = 0; j < sis->scnt; j++) {
+      SegmentInfo *si = sis->segs[j];
+      sis_add_si(iw->sis, si);
+    }
+    sis_destroy_not_infos(sis);
+  }
+
+  // merge newly added segments in log(n) passes
+  while (iw->sis->scnt > start + iw->merge_factor) {
+    for (i = start + 1; i < iw->sis->scnt; i++) {
+      end = MIN(iw->sis->scnt, i + iw->merge_factor);
+      if (end - i > 1) {
+        iw_merge_segments_with_max(iw, i, end);
+      }
+    }
+  }
+
+  // final cleanup
+  iw_optimize_internal(iw);
+  mutex_unlock(&iw->mutex);
+}
+
+
+/**
+ * This adds an array of readers to the index leaving the added readers open.
+ */
+void iw_add_readers(IndexWriter *iw, IndexReader **irs, int cnt)
+{
+  IndexReader *ir = NULL;
+  int i, del_cnt = 0;
+
+  mutex_lock(&iw->mutex);
+  iw_optimize_internal(iw); // start with zero or 1 seg
+
+  char *merged_name = new_segment_name(iw->sis->counter++);
+
+  SegmentMerger *merger = sm_create(iw->store, merged_name, iw->term_index_interval);
+  merger->readers->free_elem = NULL; // don't close readers
+
+  if (iw->sis->scnt == 1) {// add existing index, if any
+    ir = sr_open_si(iw->sis->segs[0]);
+    sm_add(merger, ir);
+    del_cnt = 1;
+  }
+
+  for (i = 0; i < cnt; i++) {
+    sm_add(merger, irs[i]);
+  }
+
+  int doc_count = sm_merge(merger); // merge 'em
+
+  // pop old infos and add new ones.
+  sis_clear(iw->sis);
+  sis_add_si(iw->sis, si_create(merged_name, doc_count, iw->store));
+
+
+  Lock *commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
+  if (!commit_lock->obtain(commit_lock)) // obtain write lock
+    eprintf(STATE_ERROR, "Index locked for commit: %s", COMMIT_LOCK_NAME);
+
+  sis_write(iw->sis, iw->store); // commit changes
+  iw_delete_segments(iw, &ir, del_cnt);
+  if (ir) ir_close(ir);
+
+  commit_lock->release(commit_lock);
+  iw->store->close_lock(commit_lock);
+
+  if (iw->use_compound_file) {
+    make_compound_file(iw, merged_name, merger);
+  }
+
+  iw_optimize_internal(iw);
+  sm_destroy(merger);
+
+  mutex_unlock(&iw->mutex);
+}
+
+/****************************************************************************
+ *
+ * Norm
+ *
+ ****************************************************************************/
+
+Norm *norm_create(InStream *is, int field_num)
+{
+  Norm *norm = ALLOC(Norm);
+  norm->is = is;
+  norm->field_num = field_num;
+  norm->bytes = NULL;
+  norm->is_dirty = false;
+  return norm;
+}
+
+void norm_destroy(void *p)
+{
+  Norm *norm = (Norm *)p;
+  is_close(norm->is);
+  if (norm->bytes != NULL) free(norm->bytes);
+  free(norm);
+}
+
+void norm_rewrite(Norm *norm, Store *store, char *segment,
+    int doc_count, Store *cfs_store)
+{
+  if (norm->bytes == NULL)
+    return; // These norms do not need to be rewritten
+
+  char tmp_fname[SEGMENT_NAME_MAX_LENGTH];
+  char norm_fname[SEGMENT_NAME_MAX_LENGTH];
+  sprintf(tmp_fname, "%s.tmp", segment);
+  OutStream *os = store->create_output(store, tmp_fname);
+  os_write_bytes(os, norm->bytes, doc_count);
+  os_close(os);
+  if (cfs_store) {
+    sprintf(norm_fname, "%s.s%d", segment, norm->field_num);
+  } else {
+    sprintf(norm_fname, "%s.f%d", segment, norm->field_num);
+  }
+  store->rename(store, tmp_fname, norm_fname);
+  norm->is_dirty = false;
+}
+
+/****************************************************************************
+ *
+ * SegmentReader
+ *
+ ****************************************************************************/
+
+#define GET_SR SegmentReader *sr = (SegmentReader *)ir->data;
+
+int sr_max_doc(IndexReader *ir)
+{
+  return ((SegmentReader *)ir->data)->fr->len;
+}
+
+static inline void sr_close_norms(SegmentReader *sr)
+{
+  h_destroy(sr->norms);
+}
+
+static inline TermVectorsReader *sr_tvr(SegmentReader *sr)
+{
+  TermVectorsReader *tvr;
+  if ((tvr = thread_getspecific(sr->thread_tvr)) == NULL) {
+    tvr = tvr_clone(sr->orig_tvr);
+    if (tvr == NULL) printf("scuk\n");
+    ary_append(sr->tvr_bucket, tvr);
+    thread_setspecific(sr->thread_tvr, tvr);
+  }
+  return tvr;
+}
+
+void sr_close(IndexReader *ir)
+{
+  GET_SR;
+  fr_close(sr->fr);
+  tir_close(sr->tir);
+
+  if (sr->freq_in) is_close(sr->freq_in);
+  if (sr->prox_in) is_close(sr->prox_in);
+  fis_destroy(sr->fis);
+
+  sr_close_norms(sr);
+  
+  if (sr->orig_tvr) {
+    tvr_close(sr->orig_tvr);
+    thread_key_delete(sr->thread_tvr);
+    ary_destroy(sr->tvr_bucket);
+  }
+  if (sr->deleted_docs) bv_destroy(sr->deleted_docs);
+  if (sr->cfs_store) sr->cfs_store->close(sr->cfs_store);
+  if (sr->fake_norms) free(sr->fake_norms);
+  free(sr->segment);
+  free(sr);
+}
+
+void sr_delete_doc(IndexReader *ir, int doc_num) 
+{
+  GET_SR;
+  if (sr->deleted_docs == NULL)
+    sr->deleted_docs = bv_create();
+
+  sr->deleted_docs_dirty = true;
+  sr->undelete_all = false;
+  bv_set(sr->deleted_docs, doc_num);
+}
+
+static inline bool sr_is_deleted_internal(IndexReader *ir, int doc_num)
+{
+  GET_SR;
+  return (sr->deleted_docs != NULL && bv_get(sr->deleted_docs, doc_num));
+}
+
+bool sr_is_deleted(IndexReader *ir, int doc_num)
+{
+  bool is_del;
+
+  mutex_lock(&ir->mutex);
+  is_del = sr_is_deleted_internal(ir, doc_num);
+  mutex_unlock(&ir->mutex);
+
+  return is_del;
+}
+
+bool sr_has_norms(IndexReader *ir, char *field)
+{
+  bool has_norms;
+  GET_SR;
+  mutex_lock(&ir->mutex);
+  has_norms = h_has_key(sr->norms, field);
+  mutex_unlock(&ir->mutex);
+
+  return has_norms;
+}
+
+bool sr_has_deletions(IndexReader *ir)
+{
+  GET_SR;
+  return (sr->deleted_docs != NULL);
+}
+
+void sr_undelete_all(IndexReader *ir)
+{
+  GET_SR;
+  sr->undelete_all = true;
+  sr->deleted_docs_dirty = false;
+  if (sr->deleted_docs != NULL) bv_destroy(sr->deleted_docs);
+  sr->deleted_docs = NULL;
+}
+
+TermEnum *sr_terms(IndexReader *ir)
+{
+  TermEnum *te = ((SegmentReader *)ir->data)->tir->orig_te;
+  return te->clone(te);
+}
+
+TermEnum *sr_terms_from(IndexReader *ir, Term *term)
+{
+  TermEnum *te = ((SegmentReader *)ir->data)->tir->orig_te;
+  TermEnum *ret_te = te->clone(te);
+  te_skip_to(ret_te, term);
+  return ret_te;
+}
+
+Document *sr_get_doc(IndexReader *ir, int doc_num)
+{
+  Document *doc;
+  mutex_lock(&ir->mutex);
+  if (sr_is_deleted_internal(ir, doc_num)) {
+    mutex_unlock(&ir->mutex);
+    eprintf(STATE_ERROR,
+        "Tried to get doc <%ld> that has already been deleted", doc_num);
+  }
+  GET_SR;
+  doc = fr_get_doc(sr->fr, doc_num);
+  mutex_unlock(&ir->mutex);
+  return doc;
+}
+
+static inline void
+sr_get_norms_into_internal(IndexReader *ir, char *field, uchar *buf, int offset)
+{
+  GET_SR;
+  Norm *norm = h_get(sr->norms, field);
+  if (norm == NULL) {
+    memset(buf + offset*sizeof(uchar), 0, sr_max_doc(ir)*sizeof(uchar));
+  } else if (norm->bytes != NULL) { // can copy from cache
+    memcpy(buf + offset*sizeof(uchar), norm->bytes, sr_max_doc(ir)*sizeof(uchar));
+  } else {
+    InStream *norm_in = is_clone(norm->is);
+    // read from disk
+    is_seek(norm_in, 0);
+    is_read_bytes(norm_in, buf, offset, sr_max_doc(ir));
+    is_close(norm_in);
+  }
+}
+
+void sr_get_norms_into(IndexReader *ir, char *field, uchar *buf, int offset)
+{
+  mutex_lock(&ir->mutex);
+  sr_get_norms_into_internal(ir, field, buf, offset);
+  mutex_unlock(&ir->mutex);
+}
+
+static inline uchar *sr_get_norms_internal(IndexReader *ir, char *field)
+{
+  GET_SR;
+  Norm *norm = h_get(sr->norms, field);
+  if (norm == NULL)                             // not an indexed field
+    return NULL;
+
+  if (norm->bytes == NULL) {                    // value not yet read
+    uchar *bytes = ALLOC_N(uchar, ir->max_doc(ir));
+    sr_get_norms_into_internal(ir, field, bytes, 0);
+    norm->bytes = bytes;                        // cache it
+  }
+  return norm->bytes;
+}
+
+uchar *sr_get_norms(IndexReader *ir, char *field)
+{
+  uchar *norms;
+  mutex_lock(&ir->mutex);
+  norms = sr_get_norms_internal(ir, field);
+  mutex_unlock(&ir->mutex);
+  return norms;
+}
+
+static inline uchar *sr_get_norms_always(IndexReader *ir, char *field)
+{
+  uchar *bytes;
+  GET_SR;
+  mutex_lock(&ir->mutex);
+
+  bytes = sr_get_norms_internal(ir, field);
+  if (bytes == NULL) {
+    if (sr->fake_norms) {
+      bytes = sr->fake_norms;
+    } else {
+      int len = ir->max_doc(ir);
+      sr->fake_norms = bytes = ALLOC_N(uchar, len);
+      memset(bytes, 0, len);
+    }
+  }
+  mutex_unlock(&ir->mutex);
+  return bytes;
+}
+
+void sr_set_norm(IndexReader *ir, int doc_num, char *field, uchar val)
+{
+  GET_SR;
+  Norm *norm;
+  
+  norm = h_get(sr->norms, field);
+  if (norm != NULL) { /* an indexed field */
+    norm->is_dirty = true;                           // mark it dirty
+    sr->norms_dirty = true;
+
+    sr_get_norms_internal(ir, field)[doc_num] = val;
+  }
+}
+
+int sr_doc_freq(IndexReader *ir, Term *t)
+{
+  GET_SR;
+  TermInfo *ti = tir_get_ti(sr->tir, t);
+  if (ti != NULL) {
+    int df = ti->doc_freq;
+    ti_destroy(ti);
+    return df;
+  } else return 0;
+}
+
+Array *sr_file_names(IndexReader *ir)
+{
+  GET_SR;
+  Array *file_names = ary_create(0, &efree);
+  FieldInfo *fi;
+  int i;
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+
+  for (i = 0; i < NELEMS(INDEX_EXTENSIONS); i++) {
+    sprintf(fname, "%s.%s", sr->segment, INDEX_EXTENSIONS[i]);
+    if (ir->store->exists(ir->store, fname))
+      ary_append(file_names, estrdup(fname));
+  }
+
+  for (i = 0; i < sr->fis->fcnt; i++) {
+    fi = sr->fis->by_number[i];
+    if (fi->is_indexed && !fi->omit_norms) {
+      if (sr->cfs_store) {
+        sprintf(fname, "%s.s%d", sr->segment, i);
+      } else {
+        sprintf(fname, "%s.f%d", sr->segment, i);
+      }
+      if (ir->store->exists(ir->store, fname))
+        ary_append(file_names, estrdup(fname));
+    }
+  }
+  return file_names;
+}
+
+HashSet *sr_get_field_names(IndexReader *ir, int field_type)
+{
+  int i;
+  GET_SR;
+  HashSet *field_set = hs_str_create(NULL);
+  FieldInfo *fi;
+  for (i = 0; i < sr->fis->fcnt; i++) {
+    fi = sr->fis->by_number[i];
+    switch(field_type) {
+      case IR_ALL:
+        hs_add(field_set, fi->name);
+        break;
+      case IR_UNINDEXED:
+        if (!fi->is_indexed) hs_add(field_set, fi->name);
+        break;
+      case IR_INDEXED:
+        if (fi->is_indexed) hs_add(field_set, fi->name);
+        break;
+      case IR_INDEXED_NO_TERM_VECTOR:
+        if (fi->is_indexed && !fi->store_tv) hs_add(field_set, fi->name);
+        break;
+      case IR_TERM_VECTOR:
+        if (fi->store_tv && !fi->store_pos && !fi->store_offset)
+          hs_add(field_set, fi->name);
+        break;
+      case IR_INDEXED_WITH_TERM_VECTOR:
+        if (fi->is_indexed && fi->store_tv) hs_add(field_set, fi->name);
+        break;
+      case IR_TERM_VECTOR_WITH_POSITION:
+        if (fi->store_pos && !fi->store_offset) hs_add(field_set, fi->name);
+        break;
+      case IR_TERM_VECTOR_WITH_OFFSET:
+        if (!fi->store_pos && fi->store_offset) hs_add(field_set, fi->name);
+      case IR_TERM_VECTOR_WITH_POSITION_OFFSET:
+        if (fi->store_pos && fi->store_offset) hs_add(field_set, fi->name);
+        break;
+      default:
+        eprintf(ARG_ERROR, "Invalid field_type <%ld>.", field_type);
+    }
+  }
+  return field_set;
+}
+
+int sr_num_docs(IndexReader *ir)
+{
+  GET_SR;
+
+  mutex_lock(&ir->mutex);
+  int num_docs = sr_max_doc(ir);
+  if (sr->deleted_docs != NULL)
+    num_docs -= sr->deleted_docs->count;
+  mutex_unlock(&ir->mutex);
+  return num_docs;
+}
+
+TermDocEnum *sr_term_docs(IndexReader *ir)
+{
+  return stde_create(ir);
+}
+
+TermDocEnum *sr_term_positions(IndexReader *ir)
+{
+  return stpe_create(ir);
+}
+
+void sr_open_norms(IndexReader *ir, Store *cfs_store)
+{
+  GET_SR;
+  int i;
+  FieldInfo *fi;
+  Store *tmp_store;
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+  for (i = 0; i < sr->fis->fcnt; i++) {
+    tmp_store = ir->store;
+    fi = sr->fis->by_number[i];
+    if (fi->is_indexed && !fi->omit_norms) {
+      sprintf(fname, "%s.s%d", sr->segment, fi->number);
+      if (! tmp_store->exists(tmp_store, fname)) {
+        sprintf(fname, "%s.f%d", sr->segment, fi->number);
+        tmp_store = cfs_store;
+      }
+      h_set(sr->norms, fi->name,
+          norm_create(tmp_store->open_input(tmp_store, fname), fi->number));
+    }
+  }
+  sr->norms_dirty = false;
+}
+
+TermVector *sr_get_term_vector(IndexReader *ir, int doc_num, char *field)
+{
+  GET_SR;
+  FieldInfo *fi = (FieldInfo *)ht_get(sr->fis->by_name, field);
+  TermVectorsReader *tvr;
+
+  if (fi == NULL || !fi->store_tv || !sr->orig_tvr || !(tvr = sr_tvr(sr)))
+    return NULL;
+
+  return tvr_get_field_tv(tvr, doc_num, field);
+}
+
+Array *sr_get_term_vectors(IndexReader *ir, int doc_num)
+{
+  GET_SR;
+  TermVectorsReader *tvr;
+  if (sr->orig_tvr == NULL || (tvr = sr_tvr(sr)) == NULL)
+    return NULL;
+
+  return tvr_get_tv(tvr, doc_num);
+}
+
+void sr_commit(IndexReader *ir)
+{
+  GET_SR;
+  char tmp_fname[SEGMENT_NAME_MAX_LENGTH];
+  char del_fname[SEGMENT_NAME_MAX_LENGTH];
+  sprintf(del_fname, "%s.del", sr->segment);
+
+  if (sr->deleted_docs_dirty) { // re-write deleted 
+    sprintf(tmp_fname, "%s.tmp", sr->segment);
+    bv_write(sr->deleted_docs, ir->store, tmp_fname);
+    ir->store->rename(ir->store, tmp_fname, del_fname);
+  }
+  if (sr->undelete_all && ir->store->exists(ir->store, del_fname))
+    ir->store->remove(ir->store, del_fname);
+  if (sr->norms_dirty) {// re-write norms 
+    int i;
+    FieldInfo *fi;
+    for (i = 0; i < sr->fis->fcnt; i++) {
+      fi = sr->fis->by_number[i];
+      if (fi->is_indexed) {
+        norm_rewrite((Norm *)h_get(sr->norms, fi->name), ir->store,
+            sr->segment, sr_max_doc(ir), sr->cfs_store);
+      }
+    }
+  }
+  sr->deleted_docs_dirty = false;
+  sr->norms_dirty = false;
+  sr->undelete_all = false;
+}
+
+IndexReader *sr_open_internal(IndexReader *ir, SegmentInfo *si)
+{
+  Store *store = si->store;
+  SegmentReader *sr = ALLOC(SegmentReader);
+  ir->get_term_vector = &sr_get_term_vector;
+  ir->get_term_vectors = &sr_get_term_vectors;
+  ir->num_docs = &sr_num_docs;
+  ir->max_doc = &sr_max_doc;
+  ir->get_doc = &sr_get_doc;
+  ir->get_norms_into = &sr_get_norms_into;
+  ir->get_norms = &sr_get_norms;
+  ir->get_norms_always = &sr_get_norms_always;
+  ir->do_set_norm = &sr_set_norm;
+  ir->terms = &sr_terms;
+  ir->terms_from = &sr_terms_from;
+  ir->doc_freq = &sr_doc_freq;
+  ir->term_docs = &sr_term_docs;
+  ir->term_positions = &sr_term_positions;
+  ir->do_delete_doc = &sr_delete_doc;
+  ir->is_deleted = &sr_is_deleted;
+  ir->has_norms = &sr_has_norms;
+  ir->has_deletions = &sr_has_deletions;
+  ir->do_undelete_all = &sr_undelete_all;
+  ir->get_field_names = &sr_get_field_names;
+  ir->do_commit = &sr_commit;
+  ir->do_close = &sr_close;
+  ir->data = sr;
+  sr->segment = estrdup(si->name);
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+  sr->cfs_store = NULL;
+  sr->fake_norms = NULL;
+  sprintf(fname, "%s.cfs", sr->segment);
+  if (store->exists(store, fname)) {
+    sr->cfs_store = open_cmpd_store(store, fname);
+    store = sr->cfs_store;
+  }
+
+  sprintf(fname, "%s.fnm", sr->segment);
+  sr->fis = fis_open(store, fname);
+  sr->fr = fr_open(store, sr->segment, sr->fis);
+
+  sr->tir = tir_open(store, sr->segment, sr->fis);
+  sr->deleted_docs = NULL;
+  sr->deleted_docs_dirty = false;
+  sr->undelete_all = false;
+  if (si_has_deletions(si)) {
+    sprintf(fname, "%s.del", sr->segment);
+    sr->deleted_docs = bv_read(si->store, fname);
+  }
+
+  sprintf(fname, "%s.frq", sr->segment);
+  sr->freq_in = store->open_input(store, fname);
+  sprintf(fname, "%s.prx", sr->segment);
+  sr->prox_in = store->open_input(store, fname);
+  sr->norms = h_new_str(NULL, &norm_destroy);
+  sr_open_norms(ir, store);
+
+  if (fis_has_vectors(sr->fis)) {
+    sr->orig_tvr = tvr_open(store, sr->segment, sr->fis);
+    thread_key_create(&sr->thread_tvr, NULL);
+    sr->tvr_bucket = ary_create(1, (destroy_func_t)&tvr_close);
+  } else {
+    sr->orig_tvr = NULL;
+  }
+  return ir;
+}
+
+IndexReader *sr_open_si(SegmentInfo *si)
+{
+  IndexReader *ir = ir_create(si->store, NULL, false, false);
+  return sr_open_internal(ir, si);
+}
+
+IndexReader *sr_open(SegmentInfos *sis, int si_num, int is_owner, int close_store)
+{
+  SegmentInfo *si = sis->segs[si_num];
+  IndexReader *ir = ir_create(si->store, sis, is_owner, close_store);
+  return sr_open_internal(ir, si);
+}
+/****************************************************************************
+ *
+ * MultiReader
+ *
+ ****************************************************************************/
+
+#define GET_MR MultiReader *mr = (MultiReader *)ir->data
+#define GET_READER(doc_num) MultiReader *mr = (MultiReader *)ir->data;\
+  int i = mr_reader_index(mr, doc_num);\
+  IndexReader *reader = mr->sub_readers[i];
+
+
+
+int mr_reader_index(MultiReader *mr, int doc_num)
+{
+  int lo = 0;                       // search @starts array
+  int hi = mr->rcnt - 1;            // for first element less
+  int mid;
+  int mid_value;
+
+  while (hi >= lo) {
+    mid = (lo + hi) >> 1;
+    mid_value = mr->starts[mid];
+    if (doc_num < mid_value) {
+      hi = mid - 1;
+    } else if (doc_num > mid_value) {
+      lo = mid + 1;
+    } else { // found a match
+      while ((mid+1 < mr->rcnt) && (mr->starts[mid+1] == mid_value))
+        mid += 1; // scan to last match in case we have empty segments
+      return mid;
+    }
+  }
+  return hi;
+}
+
+TermVector *mr_get_term_vector(IndexReader *ir, int doc_num, char *field)
+{
+  GET_READER(doc_num);
+  return reader->get_term_vector(reader, doc_num - mr->starts[i], field);
+}
+
+Array *mr_get_term_vectors(IndexReader *ir, int doc_num)
+{
+  GET_READER(doc_num);
+  return reader->get_term_vectors(reader, doc_num - mr->starts[i]);
+}
+
+int mr_num_docs(IndexReader *ir)
+{
+  int i, num_docs;
+  GET_MR;
+  mutex_lock(&ir->mutex);
+  if (mr->num_docs_cache == -1) {
+    IndexReader *reader;
+    mr->num_docs_cache = 0;
+    for (i = 0; i < mr->rcnt; i++) {
+      reader = mr->sub_readers[i];
+      mr->num_docs_cache += reader->num_docs(reader);
+    }
+  }
+  num_docs = mr->num_docs_cache;
+  mutex_unlock(&ir->mutex);
+
+  return num_docs;
+}
+
+int mr_max_doc(IndexReader *ir)
+{
+  GET_MR;
+  return mr->max_doc;
+}
+
+Document *mr_get_doc(IndexReader *ir, int doc_num)
+{
+  GET_READER(doc_num);
+  return reader->get_doc(reader, doc_num - mr->starts[i]);
+}
+
+void mr_get_norms_into(IndexReader *ir, char *field, uchar *buf, int offset)
+{
+  int i;
+  GET_MR;
+
+  mutex_lock(&ir->mutex);
+  uchar *bytes = h_get(mr->norms_cache, field);
+  if (bytes != NULL) {
+    memcpy(buf + offset, bytes, mr->max_doc);
+  } else {
+    IndexReader *reader;
+    for (i = 0; i < mr->rcnt; i++) {
+      reader = mr->sub_readers[i];
+      reader->get_norms_into(reader, field, buf, offset + mr->starts[i]);
+    }
+  }
+  mutex_unlock(&ir->mutex);
+}
+
+uchar *mr_get_norms(IndexReader *ir, char *field)
+{
+  int i;
+  GET_MR;
+  uchar *bytes;
+  IndexReader *reader;
+
+  mutex_lock(&ir->mutex);
+  bytes = h_get(mr->norms_cache, field);
+  if (bytes == NULL) {
+    bytes = ALLOC_N(uchar, mr->max_doc);
+
+    for (i = 0; i < mr->rcnt; i++) {
+      reader = mr->sub_readers[i];
+      reader->get_norms_into(reader, field, bytes, mr->starts[i]);
+    }
+    h_set(mr->norms_cache, field, bytes); // update cache
+  }
+  mutex_unlock(&ir->mutex);
+
+  return bytes;
+}
+
+void mr_set_norm(IndexReader *ir, int doc_num, char *field, uchar val)
+{
+  GET_READER(doc_num);
+  h_del(mr->norms_cache, field);               // clear cache
+  ir_set_norm(reader, doc_num - mr->starts[i], field, val);
+}
+
+TermEnum *mr_terms(IndexReader *ir)
+{
+  GET_MR;
+  return mte_create(mr->sub_readers, mr->starts, mr->rcnt, NULL);
+}
+
+TermEnum *mr_terms_from(IndexReader *ir, Term *term)
+{
+  GET_MR;
+  return mte_create(mr->sub_readers, mr->starts, mr->rcnt, term);
+}
+
+int mr_doc_freq(IndexReader *ir, Term *t)
+{
+  int total = 0, i;          // sum freqs in segments
+  GET_MR;
+
+  IndexReader *reader;
+  for (i = 0; i < mr->rcnt; i++) {
+    reader = mr->sub_readers[i];
+    total += reader->doc_freq(reader, t);
+  }
+  return total;
+}
+
+TermDocEnum *mr_term_docs(IndexReader *ir)
+{
+  GET_MR;
+  return mtde_create(mr->sub_readers, mr->starts, mr->rcnt);
+}
+
+TermDocEnum *mr_term_positions(IndexReader *ir)
+{
+  GET_MR;
+  return mtpe_create(mr->sub_readers, mr->starts, mr->rcnt);
+}
+
+void mr_delete_doc(IndexReader *ir, int doc_num)
+{
+  GET_READER(doc_num);
+  mr->num_docs_cache = -1;                     // invalidate cache
+
+  reader->do_delete_doc(reader, doc_num - mr->starts[i]); // dispatch to segment reader
+  mr->has_deletions = true;
+}
+
+bool mr_is_deleted(IndexReader *ir, int doc_num)
+{
+  GET_READER(doc_num);
+  return reader->is_deleted(reader, doc_num - mr->starts[i]);
+}
+
+bool mr_has_norms(IndexReader *ir, char *field)
+{
+  bool has_norms = false;
+  int i;
+  GET_MR;
+
+  IndexReader *reader;
+  for (i = 0; i < mr->rcnt; i++) {
+    reader = mr->sub_readers[i];
+    if (reader->has_norms(reader, field)) {
+      has_norms = true;
+      break;
+    }
+  }
+
+  return has_norms;
+}
+
+bool mr_has_deletions(IndexReader *ir)
+{
+  GET_MR;
+  return mr->has_deletions;
+}
+
+void mr_undelete_all(IndexReader *ir)
+{
+  int i;
+  GET_MR;
+  mr->num_docs_cache = -1;                     // invalidate cache
+  IndexReader *reader;
+  for (i = 0; i < mr->rcnt; i++) {
+    reader = mr->sub_readers[i];
+    reader->do_undelete_all(reader);
+  }
+  mr->has_deletions = false;
+}
+
+HashSet *mr_get_field_names(IndexReader *ir, int field_type)
+{
+  int i;
+  GET_MR;
+  HashSet *field_set = hs_str_create(NULL);
+  IndexReader *reader;
+  for (i = 0; i < mr->rcnt; i++) {
+    reader = mr->sub_readers[i];
+    hs_merge(field_set, reader->get_field_names(reader, field_type));
+  }
+  return field_set;
+}
+
+void mr_commit(IndexReader *ir)
+{
+  GET_MR;
+  int i;
+  IndexReader *reader;
+  for (i = 0; i < mr->rcnt; i++) {
+    reader = mr->sub_readers[i];
+    reader->do_commit(reader);
+  }
+}
+
+void mr_close(IndexReader *ir)
+{
+  GET_MR;
+  int i;
+  IndexReader *reader;
+  for (i = 0; i < mr->rcnt; i++) {
+    reader = mr->sub_readers[i];
+    ir_close(reader);
+  }
+  free(mr->sub_readers);
+  h_destroy(mr->norms_cache);
+  free(mr->starts);
+  free(mr);
+}
+
+IndexReader *mr_open(Store *store,
+    SegmentInfos *sis,
+    IndexReader **sub_readers,
+    int rcnt,
+    int close_store)
+{
+  int i;
+  MultiReader *mr = ALLOC(MultiReader);
+  IndexReader *sub_reader;
+  mr->sub_readers = sub_readers;
+  mr->rcnt = rcnt;
+
+  mr->max_doc = 0;
+  mr->num_docs_cache = -1;
+  mr->has_deletions = false;
+
+  mr->starts = ALLOC_N(int, (rcnt+1));
+  for (i = 0; i < rcnt; i++) {
+    sub_reader = sub_readers[i];
+    mr->starts[i] = mr->max_doc;
+    mr->max_doc += sub_reader->max_doc(sub_reader); // compute max_docs
+
+    if (sub_reader->has_deletions(sub_reader))
+      mr->has_deletions = true;
+  }
+  mr->starts[rcnt] = mr->max_doc;
+  mr->norms_cache = h_new_str(NULL, &efree);
+
+  IndexReader *ir = ir_create(store, sis, true, close_store);
+  ir->get_term_vector = &mr_get_term_vector;
+  ir->get_term_vectors = &mr_get_term_vectors;
+  ir->num_docs = &mr_num_docs;
+  ir->max_doc = &mr_max_doc;
+  ir->get_doc = &mr_get_doc;
+  ir->get_norms_into = &mr_get_norms_into;
+  ir->get_norms = &mr_get_norms;
+  ir->get_norms_always = &mr_get_norms;
+  ir->do_set_norm = &mr_set_norm;
+  ir->terms = &mr_terms;
+  ir->terms_from = &mr_terms_from;
+  ir->doc_freq = &mr_doc_freq;
+  ir->term_docs = &mr_term_docs;
+  ir->term_positions = &mr_term_positions;
+  ir->do_delete_doc = &mr_delete_doc;
+  ir->is_deleted = &mr_is_deleted;
+  ir->has_norms = &mr_has_norms;
+  ir->has_deletions = &mr_has_deletions;
+  ir->do_undelete_all = &mr_undelete_all;
+  ir->get_field_names = &mr_get_field_names;
+  ir->do_commit = &mr_commit;
+  ir->do_close = &mr_close;
+  ir->data = mr;
+
+  return ir;
+}
+
+/****************************************************************************
+ *
+ * SegmentMergeInfo
+ *
+ ****************************************************************************/
+
+bool smi_lt(void *p1, void *p2)
+{
+  SegmentMergeInfo *smi1 = (SegmentMergeInfo *)p1;
+  SegmentMergeInfo *smi2 = (SegmentMergeInfo *)p2;
+
+  int cmpres = tb_cmp(smi1->tb, smi2->tb);
+  if (cmpres == 0) {
+    return smi1->base < smi2->base;
+  } else {
+    return cmpres < 0;
+  }
+}
+
+int *smi_load_doc_map(SegmentMergeInfo *smi)
+{
+  IndexReader *ir = smi->ir;
+  if (ir->has_deletions(ir) && (smi->doc_map == NULL)) {
+    int max_doc = ir->max_doc(ir);
+    smi->doc_map = ALLOC_N(int, max_doc);
+    int j = 0, i;
+    for (i = 0; i < max_doc; i++) {
+      if (ir->is_deleted(ir, i)) {
+        smi->doc_map[i] = -1;
+      } else {
+        smi->doc_map[i] = j++;
+      }
+    }
+  }
+  return smi->doc_map;
+}
+
+SegmentMergeInfo *smi_create(int base, TermEnum *te, IndexReader *ir)
+{
+  SegmentMergeInfo *smi = ALLOC(SegmentMergeInfo);
+  smi->base = base;
+  smi->ir = ir;
+  smi->te = te;
+  smi->tb = te->tb_curr;
+  smi->postings = ir->term_positions(ir);
+  smi->doc_map = NULL;
+  return smi;
+}
+
+void smi_destroy(void *p)
+{
+  SegmentMergeInfo *smi = (SegmentMergeInfo *)p;
+  smi->postings->close(smi->postings);
+  smi->te->close(smi->te);
+  if (smi->doc_map != NULL)
+    free(smi->doc_map);
+  free(smi);
+}
+
+TermBuffer *smi_next(SegmentMergeInfo *smi)
+{
+  return (smi->tb = smi->te->next(smi->te));
+}
+
+/****************************************************************************
+ *
+ * SegmentMerger
+ *
+ ****************************************************************************/
+
+SegmentMerger *sm_create(Store *store, char *name, int term_index_interval)
+{
+  SegmentMerger *sm = ALLOC(SegmentMerger);
+  sm->store = store;
+  sm->name = estrdup(name);
+  sm->readers = ary_create(config.merge_factor, &ir_destroy);
+  sm->fis = NULL;
+  sm->freq_out = NULL;
+  sm->prox_out = NULL;
+  sm->tiw = NULL;
+  sm->queue = NULL;
+  sm->ti = ti_create(0, 0, 0, 0);
+  sm->term_index_interval = term_index_interval;
+  sm->skip_buffer = ram_create_buffer();
+  sm->skip_interval = -1;
+  return sm;
+}
+
+void sm_close(SegmentMerger *sm)
+{
+  int i;
+  if (sm->freq_out != NULL) os_close(sm->freq_out);
+  if (sm->prox_out != NULL) os_close(sm->prox_out);
+  if (sm->tiw != NULL) {
+    for (i = 0; i < sm->terms_buf_size; i++)
+      free(sm->terms_buf[i].text);
+    free(sm->terms_buf);
+    tiw_close(sm->tiw);
+  }
+  if (sm->queue != NULL) pq_destroy(sm->queue);
+  sm->freq_out = NULL;
+  sm->prox_out = NULL;
+  sm->tiw = NULL;
+  sm->queue = NULL;
+}
+
+void sm_destroy(void *p)
+{
+  SegmentMerger *sm = (SegmentMerger *)p;
+  if (sm->fis != NULL) fis_destroy(sm->fis);
+  ary_destroy(sm->readers);
+  sm_close(sm);
+  free(sm->name);
+  ti_destroy(sm->ti);
+  ram_destroy_buffer(sm->skip_buffer);
+  free(sm);
+}
+
+void sm_add(SegmentMerger *sm, IndexReader *ir)
+{
+  ary_append(sm->readers, ir);
+}
+
+static inline void sm_add_indexed(IndexReader *ir,
+    FieldInfos *fis,
+    HashSet *fields,
+    bool store_tv,
+    bool store_pos,
+    bool store_offset)
+{
+  int i;
+  char *field;
+  for (i = 0; i < fields->size; i++) {
+    field = (char *)fields->elems[i];
+    fis_add(fis, field, true, store_tv, store_pos, store_offset,
+        !ir->has_norms(ir, field));
+  }
+  hs_destroy(fields);
+}
+
+int sm_merge_fields(SegmentMerger *sm)
+{
+  int i, j, maxdoc;
+  FieldInfos *fis = sm->fis = fis_create();
+  int doc_count = 0;
+  Document *doc;
+  for (i = 0; i < sm->readers->size; i++) {
+    IndexReader *ir = sm->readers->elems[i];
+    
+    sm_add_indexed(ir, fis,
+        ir->get_field_names(ir, IR_TERM_VECTOR_WITH_POSITION_OFFSET),
+        true, true, true);
+    sm_add_indexed(ir, fis,
+        ir->get_field_names(ir, IR_TERM_VECTOR_WITH_POSITION),
+        true, true, false);
+    sm_add_indexed(ir, fis,
+        ir->get_field_names(ir, IR_TERM_VECTOR_WITH_OFFSET),
+        true, false, true);
+    sm_add_indexed(ir, fis, ir->get_field_names(ir, IR_TERM_VECTOR),
+        true, false, false);
+    sm_add_indexed(ir, fis, ir->get_field_names(ir, IR_INDEXED),
+        false, false, false);
+    fis_add_fields(fis, ir->get_field_names(ir, IR_UNINDEXED),
+        false, false, false, false, false);
+  }
+  fis_write(fis, sm->store, sm->name, ".fnm");
+
+  // merge field values
+  FieldsWriter *fw = fw_open(sm->store, sm->name, fis);
+
+  for (i = 0; i < sm->readers->size; i++) {
+    IndexReader *ir = sm->readers->elems[i];
+    maxdoc = ir->max_doc(ir);
+    for (j = 0; j < maxdoc; j++) {
+      if (!ir->is_deleted(ir, j)) { // skip deleted docs
+        doc = ir->get_doc(ir, j);
+        fw_add_doc(fw, doc);
+        doc_destroy(doc);
+        doc_count++;
+      }
+    }
+  }
+  fw_close(fw);
+  return doc_count;
+}
+
+void sm_reset_skip(SegmentMerger *sm)
+{
+  ramo_reset(sm->skip_buffer);
+  sm->last_skip_doc = 0;
+  sm->last_skip_freq_pointer = os_pos(sm->freq_out);
+  sm->last_skip_prox_pointer = os_pos(sm->prox_out);
+}
+
+inline void sm_buffer_skip(SegmentMerger *sm, int doc)
+{
+  int freq_pointer = os_pos(sm->freq_out);
+  int prox_pointer = os_pos(sm->prox_out);
+
+  os_write_vint(sm->skip_buffer, doc - sm->last_skip_doc);
+  os_write_vint(sm->skip_buffer, freq_pointer - sm->last_skip_freq_pointer);
+  os_write_vint(sm->skip_buffer, prox_pointer - sm->last_skip_prox_pointer);
+
+  sm->last_skip_doc = doc;
+  sm->last_skip_freq_pointer = freq_pointer;
+  sm->last_skip_prox_pointer = prox_pointer;
+}
+
+int sm_append_postings(SegmentMerger *sm, SegmentMergeInfo **smis, int cnt)
+{
+  int i, j;
+  int last_doc = 0, base, doc, doc_code, freq, last_position, position;
+  int *doc_map = NULL;
+  int df = 0;            // number of docs w/ term
+  TermDocEnum *postings;
+  SegmentMergeInfo *smi;
+  sm_reset_skip(sm);
+  for (i = 0; i < cnt; i++) {
+    smi = smis[i];
+    postings = smi->postings;
+    base = smi->base;
+    doc_map = smi_load_doc_map(smi);
+
+    stde_seek_ti(postings, smi->te->ti_curr);
+    while (postings->next(postings)) {
+      doc = postings->doc_num(postings);
+      if (doc_map != NULL) 
+        doc = doc_map[doc]; // work around deletions
+      doc += base;          // convert to merged space
+
+      if (doc < last_doc)
+        eprintf(STATE_ERROR,
+            "docs out of order curent doc = %ld and previous doc = %ld",
+            doc, last_doc);
+
+      df++;
+
+      if ((df % sm->skip_interval) == 0) 
+        sm_buffer_skip(sm, last_doc);
+
+      doc_code = (doc - last_doc) << 1;    // use low bit to flag freq=1
+      last_doc = doc;
+
+      freq = postings->freq(postings);
+      if (freq == 1) {
+        os_write_vint(sm->freq_out, doc_code | 1); // write doc & freq=1
+      } else {
+        os_write_vint(sm->freq_out, doc_code); // write doc
+        os_write_vint(sm->freq_out, freq);     // write freqency in doc
+      }
+        
+
+      last_position = 0;        // write position deltas
+      for (j = 0; j < freq; j++) {
+        position = postings->next_position(postings);
+        os_write_vint(sm->prox_out, position - last_position);
+        last_position = position;
+      }
+    }
+  }
+  return df;
+}
+
+int sm_write_skip(SegmentMerger *sm)
+{
+  int skip_pointer = os_pos(sm->freq_out);
+  ramo_write_to(sm->skip_buffer, sm->freq_out);
+  return skip_pointer;
+}
+
+Term *sm_tb_to_term(SegmentMerger *sm, TermBuffer *tb)
+{
+  int index = sm->terms_buf_pointer % sm->terms_buf_size;
+  sm->terms_buf_pointer++;
+  sm->terms_buf[index].field = tb->field;
+  strcpy(sm->terms_buf[index].text, tb->text);
+  return &(sm->terms_buf[index]);
+}
+
+void sm_merge_term_info(SegmentMerger *sm, SegmentMergeInfo **smis, int cnt)
+{
+  int freq_pointer = os_pos(sm->freq_out);
+  int prox_pointer = os_pos(sm->prox_out);
+
+  int df = sm_append_postings(sm, smis, cnt);      // append posting data
+
+  int skip_pointer = sm_write_skip(sm);
+
+  if (df > 0) {
+    // add an entry to the dictionary with pointers to prox and freq files
+    ti_set(sm->ti, df, freq_pointer, prox_pointer, (skip_pointer - freq_pointer));
+    tiw_add(sm->tiw, sm_tb_to_term(sm, smis[0]->tb), sm->ti);
+  }
+}
+
+void sm_merge_term_infos(SegmentMerger *sm)
+{
+  int base = 0;
+  int i, match_size;
+  IndexReader *ir;
+  TermEnum *te;
+  SegmentMergeInfo *smi, *top;
+  TermBuffer *tb;
+
+  for (i = 0; i < sm->readers->size; i++) {
+    ir = sm->readers->elems[i];
+    te = ir->terms(ir);
+    smi = smi_create(base, te, ir);
+    base += ir->num_docs(ir);
+    if (smi_next(smi) != NULL)
+      pq_push(sm->queue, smi); // initialize @queue
+    else
+      smi_destroy(smi);
+  }
+
+  SegmentMergeInfo **match = ALLOC_N(SegmentMergeInfo *, sm->readers->size);
+
+  while (sm->queue->count > 0) {
+    //  for (i = 1; i <= sm->queue->count; i++) {
+    //    printf("<{%s:%s}>", ((SegmentMergeInfo *)sm->queue->heap[i])->tb->field,
+    //                        ((SegmentMergeInfo *)sm->queue->heap[i])->tb->text);
+    //  }printf("\n\n");
+    match_size = 0;     // pop matching terms
+    match[match_size] = pq_pop(sm->queue);
+    match_size++;
+    tb = match[0]->tb;
+    top = pq_top(sm->queue);
+    while ((top != NULL) && (tb_cmp(tb, top->tb) == 0)) {
+      match[match_size] = pq_pop(sm->queue);
+      match_size++;
+      top = pq_top(sm->queue);
+    }
+
+    //printf(">%s:%s<\n", match[0]->tb->field, match[0]->tb->text);
+    sm_merge_term_info(sm, match, match_size);      // add new TermInfo
+
+    while (match_size > 0) {
+      match_size--;
+      smi = match[match_size];
+      if (smi_next(smi) != NULL)
+        pq_push(sm->queue, smi); // restore queue
+      else
+        smi_destroy(smi);      // done with a segment
+    }
+  }
+  free(match);
+}
+
+void sm_merge_terms(SegmentMerger *sm)
+{
+  int i;
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+  sprintf(fname, "%s.frq", sm->name);
+  sm->freq_out = sm->store->create_output(sm->store, fname);
+  sprintf(fname, "%s.prx", sm->name);
+  sm->prox_out = sm->store->create_output(sm->store, fname);
+  sm->tiw = tiw_open(sm->store, sm->name, sm->fis, sm->term_index_interval);
+  // terms_buf_pointer holds a buffer of terms since the TermInfosWriter needs
+  // to keep the last index_interval terms so that it can compare the last term
+  // put in the index with the next one. So the size of the buffer must by 
+  // index_interval + 2.
+  sm->terms_buf_pointer = 0;
+  sm->terms_buf_size = sm->tiw->index_interval + 2;
+  sm->terms_buf = ALLOC_N(Term, sm->terms_buf_size);
+  for (i = 0; i < sm->terms_buf_size; i++) {
+    sm->terms_buf[i].field = NULL;
+    sm->terms_buf[i].text = ALLOC_N(char, MAX_WORD_SIZE);
+  }
+  sm->skip_interval = sm->tiw->skip_interval;
+  sm->queue = pq_create(sm->readers->size, &smi_lt);
+
+  sm_merge_term_infos(sm);
+
+  sm_close(sm);
+}
+
+void sm_merge_norms(SegmentMerger *sm)
+{
+  int i, j, k, max_doc;
+  uchar *norm_buf;
+  FieldInfo *fi;
+  OutStream *os;
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+  IndexReader *ir;
+  for (i = 0; i < sm->fis->fcnt; i++) {
+    fi = sm->fis->by_number[i];
+    if (fi->is_indexed && !fi->omit_norms)  {
+      sprintf(fname, "%s.f%d", sm->name, i);
+      os = sm->store->create_output(sm->store, fname);
+      for (j = 0; j < sm->readers->size; j++) {
+        ir = sm->readers->elems[j];
+        max_doc = ir->max_doc(ir);
+        norm_buf = ALLOC_N(uchar, max_doc);
+        memset(norm_buf, 0, sizeof(uchar) * max_doc);
+        ir->get_norms_into(ir, fi->name, norm_buf, 0);
+        for (k = 0; k < max_doc; k++) {
+          if (!ir->is_deleted(ir, k)) {
+            os_write_byte(os, norm_buf[k]);
+          }
+        }
+        free(norm_buf);
+      }
+      os_close(os);
+    }
+  }
+}
+
+void sm_merge_vectors(SegmentMerger *sm)
+{
+  int i, j, max_doc;
+  TermVectorsWriter *tvw = tvw_open(sm->store, sm->name, sm->fis);
+  IndexReader *ir;
+  Array *tvs;
+  for (i = 0; i < sm->readers->size; i++) {
+    ir = sm->readers->elems[i];
+    max_doc = ir->max_doc(ir);
+    for (j = 0; j < max_doc; j++) {
+      // skip deleted docs
+      if (! ir->is_deleted(ir, j)) {
+        tvs = ir->get_term_vectors(ir, j);
+        tvw_add_all_doc_vectors(tvw, tvs);
+        ary_destroy(tvs);
+      }
+    }
+  }
+  tvw_close(tvw);
+}
+
+int sm_merge(SegmentMerger *sm)
+{
+  int doc_count = sm_merge_fields(sm);
+  sm_merge_terms(sm);
+  sm_merge_norms(sm);
+  if (fis_has_vectors(sm->fis))
+    sm_merge_vectors(sm);
+  return doc_count;
+}
+
+Array *sm_create_compound_file(SegmentMerger *sm, char *file_name)
+{
+  Array *files = ary_create(0, &efree);
+  CompoundWriter *cw = open_cw(sm->store, file_name);
+  FieldInfo *fi;
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+
+  int i;
+  for (i = 0; i < NELEMS(COMPOUND_EXTENSIONS); i++) {
+    sprintf(fname, "%s.%s", sm->name, COMPOUND_EXTENSIONS[i]);
+    ary_append(files, estrdup(fname));
+  }
+
+  // Field norm files
+  for (i = 0; i < sm->fis->fcnt; i++) {
+    fi = sm->fis->by_number[i];
+    if (fi->is_indexed && !fi->omit_norms) {
+      sprintf(fname, "%s.f%d", sm->name, i);
+      ary_append(files, estrdup(fname));
+    }
+  }
+
+  // Vector files
+  if (fis_has_vectors(sm->fis)) {
+    for (i = 0; i < NELEMS(VECTOR_EXTENSIONS); i++) {
+      sprintf(fname, "%s.%s", sm->name, VECTOR_EXTENSIONS[i]);
+      ary_append(files, estrdup(fname));
+    }
+  }
+
+  // Now merge all added files
+  for (i = 0; i < files->size; i++) {
+    cw_add_file(cw, (char *)files->elems[i]);
+  }
+  
+  // Perform the merge
+  cw_close(cw);
+ 
+  return files;
+}
+
+/****************************************************************************
+ *
+ * IndexReader
+ *
+ ****************************************************************************/
+
+void ir_acquire_not_necessary(IndexReader *ir) {}
+void ir_acquire_write_lock(IndexReader *ir)
+{
+  if (ir->is_stale)
+    eprintf(STATE_ERROR, "IndexReader out of date and no longer valid for delete, undelete, or set_norm operations");
+
+  if (ir->write_lock == NULL) {
+    ir->write_lock = ir->store->open_lock(ir->store, WRITE_LOCK_NAME);
+    if (!ir->write_lock->obtain(ir->write_lock)) // obtain write lock
+      eprintf(STATE_ERROR, "Index locked for write: %s", WRITE_LOCK_NAME);
+
+    // we have to check whether index has changed since this reader was opened.
+    // if so, this reader is no longer valid for deletion
+    if (sis_read_current_version(ir->store) > ir->sis->version) {
+      ir->is_stale = true;
+      ir->write_lock->release(ir->write_lock);
+      ir->store->close_lock(ir->write_lock);
+      ir->write_lock = NULL;
+      eprintf(STATE_ERROR, "IndexReader out of date and no longer valid for delete, undelete, or set_norm operations");
+    }
+  }
+}
+
+IndexReader *ir_create(Store *store, SegmentInfos *sis, int is_owner, int close_store)
+{
+  IndexReader *ir = ALLOC(IndexReader);
+
+  mutex_init(&ir->mutex, NULL);
+  ir->is_owner = is_owner;
+  if (is_owner) {
+    ir->acquire_write_lock = &ir_acquire_write_lock;
+  } else {
+    ir->acquire_write_lock = &ir_acquire_not_necessary;
+  }
+
+  ir->store = store;
+  ir->close_store = close_store;
+  ir->sis = sis;
+  ir->has_changes = false;
+  ir->is_stale = false;
+  ir->write_lock = NULL;
+  ir->cache = NULL;
+  ir->sort_cache = NULL;
+  return ir;
+}
+
+IndexReader *ir_open(Store *store, int close_store)
+{
+  int i;
+  IndexReader *ir;
+  SegmentInfos *sis;
+
+  mutex_lock(&store->mutex);
+  sis = sis_create();
+  sis_read(sis, store);
+  if (sis->scnt == 1) {
+    ir = sr_open(sis, 0, true, close_store);
+  } else {
+    IndexReader **readers = ALLOC_N(IndexReader *, sis->scnt);
+    for (i = 0; i < sis->scnt; i++) {
+      readers[i] = sr_open(sis, i, false, false);
+    }
+    ir = mr_open(store, sis, readers, sis->scnt, close_store);
+  }
+  mutex_unlock(&store->mutex);
+  return ir;
+}
+
+bool ir_index_exists(Store *store)
+{
+  return store->exists(store, "segments");
+}
+
+void ir_set_norm(IndexReader *ir, int doc_num, char *field, uchar val)
+{
+  mutex_lock(&ir->mutex);
+  ir->acquire_write_lock(ir);
+  ir->do_set_norm(ir, doc_num, field, val);
+  ir->has_changes = true;
+  mutex_unlock(&ir->mutex);
+}
+
+void ir_undelete_all(IndexReader *ir)
+{
+  mutex_lock(&ir->mutex);
+  ir->acquire_write_lock(ir);
+  ir->do_undelete_all(ir);
+  ir->has_changes = true;
+  mutex_unlock(&ir->mutex);
+}
+
+void ir_delete_doc(IndexReader *ir, int doc_num)
+{
+  mutex_lock(&ir->mutex);
+  ir->acquire_write_lock(ir);
+  ir->do_delete_doc(ir, doc_num);
+  ir->has_changes = true;
+  mutex_unlock(&ir->mutex);
+}
+
+Document *ir_get_doc_with_term(IndexReader *ir, Term *term)
+{
+  TermDocEnum *tde = ir_term_docs_for(ir, term);
+  if (!tde) return NULL;
+
+  Document *doc = NULL;
+  if (tde->next(tde))
+    doc = ir->get_doc(ir, tde->doc_num(tde));
+  tde->close(tde);
+  return doc;
+}
+
+TermDocEnum *ir_term_docs_for(IndexReader *ir, Term *term)
+{
+  TermDocEnum *tde = ir->term_docs(ir);
+  tde->seek(tde, term);
+  return tde;
+}
+
+TermDocEnum *ir_term_positions_for(IndexReader *ir, Term *term)
+{
+  TermDocEnum *tde = ir->term_positions(ir);
+  tde->seek(tde, term);
+  return tde;
+}
+
+void ir_commit_internal(IndexReader *ir)
+{
+  if (ir->has_changes) {
+    if (ir->is_owner) {
+
+      mutex_lock(&ir->store->mutex);
+      Lock *commit_lock = ir->store->open_lock(ir->store, COMMIT_LOCK_NAME);
+      if (!commit_lock->obtain(commit_lock)) // obtain write lock
+        eprintf(STATE_ERROR, "Index locked for commit: %s", COMMIT_LOCK_NAME);
+
+      ir->do_commit(ir);
+      sis_write(ir->sis, ir->store);
+
+      commit_lock->release(commit_lock);
+      ir->store->close_lock(commit_lock);
+      mutex_unlock(&ir->store->mutex);
+
+      if (ir->write_lock != NULL) {
+        ir->write_lock->release(ir->write_lock);  // release write lock
+        ir->store->close_lock(ir->write_lock);
+        ir->write_lock = NULL;
+      }
+    } else {
+      ir->do_commit(ir);
+    }
+    ir->has_changes = false;
+  }
+}
+
+void ir_commit(IndexReader *ir)
+{
+  mutex_lock(&ir->mutex);
+  ir_commit_internal(ir);
+  mutex_unlock(&ir->mutex);
+}
+
+void ir_close(IndexReader *ir)
+{
+  mutex_lock(&ir->mutex);
+  ir_commit_internal(ir);
+  ir->do_close(ir);
+  if (ir->close_store) {
+    ir->store->close(ir->store);
+  }
+  if (ir->is_owner) {
+    sis_destroy(ir->sis);
+  }
+  if (ir->cache) {
+    h_destroy(ir->cache);
+  }
+  if (ir->sort_cache) {
+    h_destroy(ir->sort_cache);
+  }
+
+  mutex_destroy(&ir->mutex);
+  free(ir);
+}
+
+void ir_destroy(void *p)
+{
+  IndexReader *ir = (IndexReader *)p;
+  ir_close(ir);
+}
+
+/**
+ * Don't call this method if the cache already exists
+ **/
+void ir_add_cache(IndexReader *ir)
+{
+  ir->cache = co_hsh_create();
+}
+
+bool ir_is_latest(IndexReader *ir)
+{
+  return sis_read_current_version(ir->store) == ir->sis->version;
+}
+