ferret 0.10.14 → 0.11.0

data/ext/hash.c

@@ -18,13 +18,6 @@ static char *dummy_key = "";
 static HashTable *free_hts[MAX_FREE_HASH_TABLES];
 static int num_free_hts = 0;
 
-unsigned long *imalloc(unsigned long value)
-{
-  unsigned long *p = ALLOC(unsigned long);
-  *p = value;
-  return p;
-}
-
 unsigned long str_hash(const char *const str)
 {
     register unsigned long h = 0;

data/ext/hash.h

@@ -81,14 +81,6 @@ typedef unsigned long (*hash_ft)(const void *key);
  */
 typedef int (*eq_ft)(const void *key1, const void *key2);
 
-
-/**
- * Create a pointer to an allocated U32 integer. This function is a utility
- * function used to add integers to a HashTable, either as the key or the
- * value.
- */
-extern unsigned long *imalloc(unsigned long value);
-
 /**
  * Determine a hash value for a string. The string must be null terminated
  *

data/ext/index.c

@@ -5,6 +5,7 @@
 #include "priorityqueue.h"
 #include <string.h>
 #include <limits.h>
+#include <ctype.h>
 
 #define GET_LOCK(lock, name, store, err_msg) do {\
     lock = store->open_lock(store, name);\
@@ -18,14 +19,6 @@
     store->close_lock(lock);\
 } while (0)
 
-const char *INDEX_EXTENSIONS[] = {
-    "fdx", "fdt", "tfx", "tix", "tis", "frq", "prx", "del"
-};
-
-const char *COMPOUND_EXTENSIONS[] = {
-    "frq", "prx", "fdx", "fdt", "tfx", "tix", "tis"
-};
-
 const Config default_config = {
     0x100000,       /* chunk size is 1Mb */
     0x1000000,      /* Max memory used for buffer is 16 Mb */
@@ -41,6 +34,128 @@ const Config default_config = {
 static void ste_reset(TermEnum *te);
 static char *ste_next(TermEnum *te);
 
+#define FORMAT 0
+#define SEGMENTS_GEN_FILE_NAME "segments.gen"
+#define MAX_EXT_LEN 10
+
+/* *** Must be three characters *** */
+const char *INDEX_EXTENSIONS[] = {
+    "frq", "prx", "fdx", "fdt", "tfx", "tix", "tis", "del", "gen", "cfs"
+};
+
+/* *** Must be three characters *** */
+const char *COMPOUND_EXTENSIONS[] = {
+    "frq", "prx", "fdx", "fdt", "tfx", "tix", "tis"
+};
+
+
+static const char BASE36_DIGITMAP[] = "0123456789abcdefghijklmnopqrstuvwxyz";
+
+static char *u64_to_str36(char *buf, int buf_size, f_u64 u)
+{
+    int i = buf_size--;
+    buf[i] = '\0';
+    for (i--; i >= 0; i--) {
+        buf[i] = BASE36_DIGITMAP[u % 36];
+        u /= 36;
+        if (0 == u) {
+            break;
+        }
+    }
+    if (0 < u) {
+        RAISE(EXCEPTION, "Max length of segment filename has been reached. "
+              "Perhaps it's time to re-index.\n");
+    }
+    return buf + i;
+}
+
+static f_u64 str36_to_u64(char *p)
+{
+    f_u64 u = 0;
+    while (true) {
+        if ('0' <= *p && '9' >= *p) {
+            u = u * 36 + *p - '0';
+        }
+        else if ('a' <= *p && 'z' >= *p) {
+            u = u * 36 + *p - 'a' + 10;
+        }
+        else {
+            break;
+        }
+        p++;
+    }
+    return u;
+}
+
+/*
+ * Computes the full file name from base, extension and generation.  If the
+ * generation is -1, the file name is NULL.  If it's 0, the file name is
+ * <base><extension>.  If it's > 0, the file name is
+ * <base>_<generation><extension>.
+ *
+ * @param buf buffer to write filename to
+ * @param base main part of the file name
+ * @param ext extension of the filename (including .)
+ * @param gen generation
+ */
+char *fn_for_generation(char *buf, char *base, char *ext, f_i64 gen)
+{
+    if (-1 == gen) {
+        return NULL;
+    }
+    else {
+        char b[SEGMENT_NAME_MAX_LENGTH];
+        char *u = u64_to_str36(b, SEGMENT_NAME_MAX_LENGTH, (f_u64)gen);
+        if (ext == NULL) {
+            sprintf(buf, "%s_%s", base, u);
+        }
+        else {
+            sprintf(buf, "%s_%s.%s", base, u, ext);
+        }
+        return buf;
+    }
+}
+
+char *segfn_for_generation(char *buf, int generation)
+{
+    char b[SEGMENT_NAME_MAX_LENGTH];
+    char *u = u64_to_str36(b, SEGMENT_NAME_MAX_LENGTH, (f_u64)generation);
+    sprintf(buf, SEGMENTS_FILE_NAME"_%s", u);
+    return buf;
+}
+
+/*
+ * Computes the field specific file name from base, extension, generation and
+ * field number.  If the generation is -1, the file name is NULL.  If it's 0,
+ * the file name is <base><extension>.  If it's > 0, the file name is
+ * <base>_<generation><extension>.
+ *
+ * @param buf buffer to write filename to
+ * @param base main part of the file name
+ * @param ext extension of the filename (including .)
+ * @param gen generation
+ * @param field_num field number
+ */
+static char *fn_for_gen_field(char *buf,
+                              char *base,
+                              char *ext,
+                              f_i64 gen,
+                              int field_num)
+{
+    if (-1 == gen) {
+        return NULL;
+    }
+    else {
+        char b[SEGMENT_NAME_MAX_LENGTH];
+        sprintf(buf, "%s_%s.%s%d",
+                base,
+                u64_to_str36(b, SEGMENT_NAME_MAX_LENGTH, (f_u64)gen),
+                ext,
+                field_num);
+        return buf;
+    }
+}
+
 /***************************************************************************
  *
  * CacheObject
@@ -175,7 +290,7 @@ FieldInfo *fi_new(const char *name,
 
 void fi_deref(FieldInfo *fi)
 {
-    if (--(fi->ref_cnt) == 0) {
+    if (0 == --(fi->ref_cnt)) {
         free(fi->name);
         free(fi);
     }
@@ -208,9 +323,6 @@ char *fi_to_s(FieldInfo *fi)
  *
  ****************************************************************************/
 
-#define FIELDS_FILENAME "fields"
-#define TEMPORARY_FIELDS_FILENAME "fields.new"
-
 FieldInfos *fis_new(int store, int index, int term_vector)
 {
     FieldInfos *fis = ALLOC(FieldInfos);
@@ -278,14 +390,13 @@ FieldInfo *fis_by_number(FieldInfos *fis, int num)
     }
 }
 
-FieldInfos *fis_read(Store *store)
+FieldInfos *fis_read(InStream *is)
 {
     int store_val, index_val, term_vector_val;
     int i;
     union { f_u32 i; float f; } tmp;
     FieldInfo *fi;
     FieldInfos *fis;
-    InStream *is = store->open_input(store, FIELDS_FILENAME);
 
     store_val = is_read_vint(is);
     index_val = is_read_vint(is);
@@ -300,17 +411,15 @@ FieldInfos *fis_read(Store *store)
         fis_add_field(fis, fi);
         fi->ref_cnt = 1;
     }
-    is_close(is);
 
     return fis; 
 }
 
-void fis_write(FieldInfos *fis, Store *store)
+void fis_write(FieldInfos *fis, OutStream *os)
 {
     int i;
     union { f_u32 i; float f; } tmp;
     FieldInfo *fi;
-    OutStream *os = store->new_output(store, TEMPORARY_FIELDS_FILENAME);
     const int fis_size = fis->size;
 
     os_write_vint(os, fis->store);
@@ -324,9 +433,6 @@ void fis_write(FieldInfos *fis, Store *store)
         os_write_u32(os, tmp.i);
         os_write_vint(os, fi->bits);
     }
-    os_close(os);
-
-    store->rename(store, TEMPORARY_FIELDS_FILENAME, FIELDS_FILENAME);
 }
 
 static const char *store_str[] = {
@@ -408,7 +514,7 @@ char *fis_to_s(FieldInfos *fis)
 
 void fis_deref(FieldInfos *fis)
 {
-    if (--(fis->ref_cnt) == 0) {
+    if (0 == --(fis->ref_cnt)) {
         h_destroy(fis->field_dict);
         free(fis->fields);
         free(fis);
@@ -440,54 +546,144 @@ SegmentInfo *si_new(char *name, int doc_cnt, Store *store)
     si->name = name;
     si->doc_cnt = doc_cnt;
     si->store = store;
+    si->del_gen = -1;
+    si->norm_gens = NULL;
+    si->norm_gens_size = 0;
+    si->ref_cnt = 1;
+    si->use_compound_file = false;
     return si;
 }
 
-void si_destroy(SegmentInfo *si)
+SegmentInfo *si_read(Store *store, InStream *is)
 {
-    free(si->name);
-    free(si);
+    SegmentInfo *si = ALLOC_AND_ZERO(SegmentInfo);
+    si->store = store;
+    si->name = is_read_string(is);
+    si->doc_cnt = is_read_vint(is);
+    si->del_gen = is_read_vint(is);
+    si->norm_gens_size = is_read_vint(is);
+    si->ref_cnt = 1;
+    if (0 < si->norm_gens_size) {
+        int i;
+        si->norm_gens = ALLOC_N(int, si->norm_gens_size);
+        for (i = si->norm_gens_size - 1; i >= 0; i--) {
+            si->norm_gens[i] = is_read_vint(is);
+        }
+    }
+    si->use_compound_file = (bool)is_read_byte(is);
+    return si;
 }
 
-bool si_has_deletions(SegmentInfo *si)
+void si_write(SegmentInfo *si, OutStream *os)
 {
-    char del_file_name[SEGMENT_NAME_MAX_LENGTH];
-    sprintf(del_file_name, "%s.del", si->name);
-    return si->store->exists(si->store, del_file_name);
+    os_write_string(os, si->name);
+    os_write_vint(os, si->doc_cnt);
+    os_write_vint(os, si->del_gen);
+    os_write_vint(os, si->norm_gens_size);
+    if (0 < si->norm_gens_size) {
+        int i;
+        for (i = si->norm_gens_size - 1; i >= 0; i--) {
+            os_write_vint(os, si->norm_gens[i]);
+        }
+    }
+    os_write_byte(os, (uchar)si->use_compound_file);
 }
 
-bool si_uses_compound_file(SegmentInfo *si)
+void si_deref(SegmentInfo *si)
 {
-    char compound_file_name[SEGMENT_NAME_MAX_LENGTH];
-    sprintf(compound_file_name, "%s.cfs", si->name);
-    return si->store->exists(si->store, compound_file_name);
+    if (--si->ref_cnt <= 0) {
+        free(si->name);
+        free(si->norm_gens);
+        free(si);
+    }
 }
 
-struct NormTester {
-    bool has_norm_file;
-    int norm_file_pattern_len;
-    char norm_file_pattern[SEGMENT_NAME_MAX_LENGTH];
-};
+bool si_has_deletions(SegmentInfo *si)
+{
+    return si->del_gen >= 0;
+}
 
-static void is_norm_file(char *file_name, struct NormTester *nt)
+char *si_del_file_name(SegmentInfo *si, char *buf)
 {
-    if (strncmp(file_name, nt->norm_file_pattern,
-                nt->norm_file_pattern_len) == 0) {
-        nt->has_norm_file = true;
+    if (si->del_gen < 0) {
+        return NULL;
+    }
+    else {
+        return fn_for_generation(buf, si->name, ".del", si->del_gen);
     }
 }
 
 bool si_has_separate_norms(SegmentInfo *si)
 {
-    struct NormTester nt;
-    sprintf(nt.norm_file_pattern, "%s.s", si->name);
-    nt.norm_file_pattern_len = strlen(nt.norm_file_pattern);
-    nt.has_norm_file = false;
-    si->store->each(si->store, (void (*)(char *file_name, void *arg))&is_norm_file, &nt);
+    if (si->use_compound_file && si->norm_gens) {
+        int i;
+        for (i = si->norm_gens_size - 1; i >= 0; i--) {
+            if (si->norm_gens[i] > 0) return true;
+        }
+    }
+    return false;
+}
 
-    return nt.has_norm_file;
+void si_advance_norm_gen(SegmentInfo *si, int field_num)
+{
+    if (field_num >= si->norm_gens_size) {
+        int i;
+        REALLOC_N(si->norm_gens, int, field_num + 1);
+        for (i = si->norm_gens_size; i <= field_num; i++) {
+            si->norm_gens[i] = -1;
+        }
+        si->norm_gens_size = field_num + 1;
+    }
+    si->norm_gens[field_num]++;
 }
 
+char *si_norm_file_name(SegmentInfo *si, char *buf, int field_num)
+{
+    int norm_gen;
+    if (field_num >= si->norm_gens_size
+        || 0 > (norm_gen = si->norm_gens[field_num])) {
+        return NULL;
+    }
+    else {
+        char *ext = (si->use_compound_file && norm_gen > 0) ? "s" : "f";
+        return fn_for_gen_field(buf, si->name, ext, norm_gen, field_num);
+    }
+}
+
+void deleter_queue_file(Deleter *dlr, char *file_name);
+#define DEL(file_name) deleter_queue_file(dlr, file_name)
+
+static void si_delete_files(SegmentInfo *si, FieldInfos *fis, Deleter *dlr)
+{
+    int i;
+    char file_name[SEGMENT_NAME_MAX_LENGTH];
+    size_t seg_len = strlen(si->name);
+    char *ext;
+
+    for (i = si->norm_gens_size - 1; i >= 0; i--) {
+        if (0 <= si->norm_gens[i]) {
+            DEL(si_norm_file_name(si, file_name, fis->fields[i]->number));
+        }
+    }
+
+    memcpy(file_name, si->name, seg_len);
+    file_name[seg_len] = '.';
+    ext = file_name + seg_len + 1;
+
+    if (si->use_compound_file) {
+        memcpy(ext, "cfs", 4);
+        DEL(file_name);
+        if (0 <= si->del_gen) {
+            DEL(fn_for_generation(file_name, si->name, "del", si->del_gen));
+        }
+    }
+    else {
+        for (i = NELEMS(INDEX_EXTENSIONS) - 1; i >= 0; i--) {
+            memcpy(ext, INDEX_EXTENSIONS[i], 4);
+            DEL(file_name);
+        }
+    }
+}
 
 /****************************************************************************
  *
@@ -496,42 +692,266 @@ bool si_has_separate_norms(SegmentInfo *si)
  ****************************************************************************/
 
 #include <time.h>
-#define FORMAT 0
-#define SEGMENTS_FILENAME "segments"
-#define TEMPORARY_SEGMENTS_FILENAME "segments.new"
-#define MAX_EXT_LEN 10
+static char *new_segment(f_i64 generation) 
+{
+    char buf[SEGMENT_NAME_MAX_LENGTH];
+    char *fn_p = u64_to_str36(buf, SEGMENT_NAME_MAX_LENGTH - 1,
+                              (f_u64)generation);
+    *(--fn_p) = '_';
+    return estrdup(fn_p);
+}
+
+/****************************************************************************
+ * FindSegmentsFile
+ ****************************************************************************/
 
-static const char base36_digitmap[] = "0123456789abcdefghijklmnopqrstuvwxyz";
+typedef struct FindSegmentsFile {
+    f_i64  generation;
+    f_u64  u64_return;
+    void  *p_return;
+} FindSegmentsFile;
 
-static char *new_segment(f_u64 counter) 
+static void which_gen_i(char *file_name, void *arg)
+{
+    f_i64 *max_generation = (f_i64 *)arg;
+    if (0 == strncmp(SEGMENTS_FILE_NAME"_", file_name,
+                     sizeof(SEGMENTS_FILE_NAME))) {
+        char *p = strrchr(file_name, '_') + 1;
+        f_i64 generation = (f_i64)str36_to_u64(p);
+        if (generation > *max_generation) *max_generation = generation;
+    }
+}
+
+static void si_put(SegmentInfo *si, FILE *stream)
 {
-    char file_name[SEGMENT_NAME_MAX_LENGTH];
     int i;
+    fprintf(stream, "\tSegmentInfo {\n");
+    fprintf(stream, "\t\tname = %s\n", si->name);
+    fprintf(stream, "\t\tdoc_cnt = %d\n", si->doc_cnt);
+    fprintf(stream, "\t\tdel_gen = %d\n", si->del_gen);
+    fprintf(stream, "\t\tnorm_gens_size = %d\n", si->norm_gens_size);
+    fprintf(stream, "\t\tnorm_gens {\n");
+    for (i = 0; i < si->norm_gens_size; i++) {
+        fprintf(stream, "\t\t\t%d\n", si->norm_gens[i]);
+    }
+    fprintf(stream, "\t\t}\n");
+    fprintf(stream, "\t\tref_cnt = %d\n", si->ref_cnt);
+    fprintf(stream, "\t}\n");
+}
 
-    file_name[SEGMENT_NAME_MAX_LENGTH - 1] = '\0';
-    for (i = SEGMENT_NAME_MAX_LENGTH - 2; i > MAX_EXT_LEN; i--) {
-        file_name[i] = base36_digitmap[counter%36];
-        counter /= 36;
-        if (counter == 0) {
-            break;
-        }
+void sis_put(SegmentInfos *sis, FILE *stream)
+{
+    int i;
+    fprintf(stream, "SegmentInfos {\n");
+    fprintf(stream, "\tcounter = %"POSH_I64_PRINTF_PREFIX"d\n", sis->counter);
+    fprintf(stream, "\tversion = %"POSH_I64_PRINTF_PREFIX"d\n", sis->version);
+    fprintf(stream, "\tgeneration = %"POSH_I64_PRINTF_PREFIX"d\n", sis->generation);
+    fprintf(stream, "\tformat = %d\n", sis->format);
+    fprintf(stream, "\tsize = %d\n", sis->size);
+    fprintf(stream, "\tcapa = %d\n", sis->capa);
+    for (i = 0; i < sis->size; i++) {
+        si_put(sis->segs[i], stream);
     }
-    if (i == MAX_EXT_LEN) {
-        RAISE(EXCEPTION, "Max length of segment filename has been reached. "
-              "Time to re-index.\n");
+    fprintf(stream, "}\n");
+}
+
+/*
+ * Get the generation (N) of the current segments_N file from a list of files.
+ *
+ * @param store - the Store to look in
+ */
+f_i64 sis_current_segment_generation(Store *store)
+{
+    f_i64 current_generation = -1;
+    store->each(store, &which_gen_i, &current_generation);
+    return current_generation;
+}
+
+/*
+ * Get the current generation filename.
+ *
+ * @param buf - buffer to write filename to
+ * @param store - the Store to look in
+ * @return segments_N where N is the current generation
+ */
+char *sis_curr_seg_file_name(char *buf, Store *store)
+{
+    return segfn_for_generation(buf, sis_current_segment_generation(store));
+}
+
+/*
+ * Get the next generation filename.
+ *
+ * @param buf - buffer to write filename to
+ * @param store - the Store to look in
+ * @return segments_N where N is the +next+ generation
+ */
+char *sis_next_seg_file_name(char *buf, Store *store)
+{
+    return segfn_for_generation(buf, sis_current_segment_generation(store) + 1);
+}
+
+#define GEN_FILE_RETRY_COUNT 10
+#define GEN_LOOK_AHEAD_COUNT 10
+void sis_find_segments_file(Store *store, FindSegmentsFile *fsf,
+                            void (*run)(Store *store, FindSegmentsFile *fsf))
+{
+    int i;
+    int gen_look_ahead_count = 0;
+    bool retry = false;
+    int method = 0;
+    f_i64 last_gen = -1;
+    f_i64 gen = 0;
+
+    /* Loop until we succeed in calling doBody() without hitting an
+     * IOException.  An IOException most likely means a commit was in process
+     * and has finished, in the time it took us to load the now-old infos
+     * files (and segments files).  It's also possible it's a true error
+     * (corrupt index).  To distinguish these, on each retry we must see
+     * "forward progress" on which generation we are trying to load.  If we
+     * don't, then the original error is real and we throw it.
+     *
+     * We have three methods for determining the current generation.  We try
+     * each in sequence. */
+    while (true) {
+        /* Method 1: list the directory and use the highest segments_N file.
+         * This method works well as long as there is no stale caching on the
+         * directory contents: */
+        if (0 == method) {
+            gen = sis_current_segment_generation(store);
+            if (gen == -1) {
+                /*fprintf(stderr, ">>\n%s\n>>\n", store_to_s(store));*/
+                RAISE(FILE_NOT_FOUND_ERROR, "couldn't find segments file");
+            }
+        }
+
+        /* Method 2 (fallback if Method 1 isn't reliable): if the directory
+         * listing seems to be stale, try loading the "segments.gen" file. */
+        if (1 == method || (0 == method && last_gen == gen && retry)) {
+            method = 1;
+            for (i = 0; i < GEN_FILE_RETRY_COUNT; i++) {
+                InStream *gen_is = NULL;
+                TRY
+                    gen_is = store->open_input(store, SEGMENTS_GEN_FILE_NAME);
+                XCATCHALL
+                    HANDLED();
+                    /* TODO:LOG "segments.gen open: IO_ERROR"*/
+                XENDTRY
+
+                if (NULL != gen_is) {
+                    f_i64 gen0 = -1, gen1 = -1;
+
+                    TRY
+                        gen0 = is_read_u64(gen_is);
+                        gen1 = is_read_u64(gen_is);
+                    XFINALLY
+                        /* if there is an error well simply try again */
+                        HANDLED();
+                        is_close(gen_is);
+                    XENDTRY
+                    /* TODO:LOG "fallback check: " + gen0 + "; " + gen1 */
+                    if (gen0 == gen1) {
+                        /* The file is consistent. */
+                        if (gen0 > gen) {
+                            /* TODO:LOG "fallback to '" +
+                             * IndexFileNames.SEGMENTS_GEN + "' check: now
+                             * try generation " + gen0 + " > " + gen */
+                            gen = gen0;
+                        }
+                        goto method_two_loop_end;
+                    }
+                    break;
+                }
+                /* sleep for 50 milliseconds */
+                micro_sleep(50000);
+            }
+        }
+method_two_loop_end:
+
+        /* Method 3 (fallback if Methods 2 & 3 are not reliable): since both
+         * directory cache and file contents cache seem to be stale, just
+         * advance the generation. */
+        if (2 == method || (1 == method && last_gen == gen && retry)) {
+            method = 2;
+            if (gen_look_ahead_count < GEN_LOOK_AHEAD_COUNT) {
+                gen++;
+                gen_look_ahead_count++;
+                /* TODO:LOG "look ahead increment gen to " + gen */
+            }
+        }
+
+        if (last_gen == gen) {
+            /* This means we're about to try the same segments_N last tried.
+             * This is allowed, exactly once, because writer could have been
+             * in the process of writing segments_N last time. */
+            if (retry) {
+                /* OK, we've tried the same segments_N file twice in a row, so
+                 * this must be a real error.  We throw the original exception
+                 * we got. */
+                RAISE(IO_ERROR, "Error reading the segment infos");
+            }
+            else {
+                retry = true;
+            }
+        }
+        else {
+            /* Segment file has advanced since our last loop, so reset retry: */
+            retry = false;
+        }
+        last_gen = gen;
+
+        TRY
+            fsf->generation = gen;
+            run(store, fsf);
+            RETURN_EARLY();
+            return;
+        case IO_ERROR: case FILE_NOT_FOUND_ERROR:
+            HANDLED();
+            /* Save the original root cause: */
+            /* TODO:LOG "primary Exception on '" + segmentFileName + "': " +
+             * err + "'; will retry: retry=" + retry + "; gen = " + gen */
+
+            if (!retry && gen > 1) {
+                /* This is our first time trying this segments file (because
+                 * retry is false), and, there is possibly a segments_(N-1)
+                 * (because gen > 1).  So, check if the segments_(N-1) exists
+                 * and try it if so: */
+                char prev_seg_file_name[SEGMENT_NAME_MAX_LENGTH];
+                segfn_for_generation(prev_seg_file_name, gen - 1);
+                if (store->exists(store, prev_seg_file_name)) {
+                    /* TODO:LOG "fallback to prior segment file '" +
+                     * prevSegmentFileName + "'" */
+                    TRY
+                        fsf->generation = gen - 1;
+                        run(store, fsf);
+                        /* TODO:LOG "success on fallback " +
+                         * prev_seg_file_name */
+
+                        /* pop two contexts as we are in nested try blocks */
+                        RETURN_EARLY();
+                        RETURN_EARLY();
+                        return;
+                    case IO_ERROR: case FILE_NOT_FOUND_ERROR:
+                        HANDLED();
+                        /* TODO:LOG "secondary Exception on '" +
+                         * prev_seg_file_name + "': " + err2 + "'; will retry"*/
+                    XENDTRY
+                }
+            }
+        XENDTRY
     }
-    i--;
-    file_name[i] = '_';
-    return estrdup(&file_name[i]);
 }
 
-SegmentInfos *sis_new()
+SegmentInfos *sis_new(FieldInfos *fis)
 {
-    SegmentInfos *sis = ALLOC(SegmentInfos);
+    SegmentInfos *sis = ALLOC_AND_ZERO(SegmentInfos);
+    REF(fis);
+    sis->fis = fis;
     sis->format = FORMAT;
     sis->version = (f_u64)time(NULL);
     sis->size = 0;
     sis->counter = 0;
+    sis->generation = -1;
     sis->capa = 4;
     sis->segs = ALLOC_N(SegmentInfo *, sis->capa);
     return sis;
@@ -539,8 +959,7 @@ SegmentInfos *sis_new()
 
 SegmentInfo *sis_new_segment(SegmentInfos *sis, int doc_cnt, Store *store)
 {
-    return sis_add_si(sis, si_new(new_segment(sis->counter++), doc_cnt,
-                                     store));
+    return sis_add_si(sis, si_new(new_segment(sis->counter++), doc_cnt, store));
 }
 
 void sis_destroy(SegmentInfos *sis)
@@ -548,8 +967,9 @@ void sis_destroy(SegmentInfos *sis)
     int i;
     const int sis_size = sis->size;
     for (i = 0; i < sis_size; i++) {
-        si_destroy(sis->segs[i]);
+        si_deref(sis->segs[i]);
     }
+    if (sis->fis) fis_deref(sis->fis);
     free(sis->segs);
     free(sis);
 }
@@ -557,11 +977,10 @@ void sis_destroy(SegmentInfos *sis)
 SegmentInfo *sis_add_si(SegmentInfos *sis, SegmentInfo *si)
 {
     if (sis->size >= sis->capa) {
-        sis->capa = sis->size * 2;
+        sis->capa <<= 1;
         REALLOC_N(sis->segs, SegmentInfo *, sis->capa);
     }
-    sis->segs[sis->size] = si;
-    sis->size++;
+    sis->segs[sis->size++] = si;
     return si;
 }
 
@@ -569,7 +988,7 @@ void sis_del_at(SegmentInfos *sis, int at)
 {
     int i;
     const int sis_size = --(sis->size);
-    si_destroy(sis->segs[at]);
+    si_deref(sis->segs[at]);
     for (i = at; i < sis_size; i++) {
         sis->segs[i] = sis->segs[i+1];
     }
@@ -580,7 +999,7 @@ void sis_del_from_to(SegmentInfos *sis, int from, int to)
     int i, num_to_del = to - from;
     const int sis_size = sis->size -= num_to_del;
     for (i = from; i < to; i++) {
-        si_destroy(sis->segs[i]);
+        si_deref(sis->segs[i]);
     }
     for (i = from; i < sis_size; i++) {
         sis->segs[i] = sis->segs[i+num_to_del];
@@ -592,74 +1011,106 @@ void sis_clear(SegmentInfos *sis)
     int i;
     const int sis_size = sis->size;
     for (i = 0; i < sis_size; i++) {
-        si_destroy(sis->segs[i]);
+        si_deref(sis->segs[i]);
     }
     sis->size = 0;
 }
 
-SegmentInfos *sis_read(Store *store)
+void sis_read_i(Store *store, FindSegmentsFile *fsf)
 {
-    int doc_cnt;
     int seg_cnt;
     int i;
-    char *name;
-    InStream *is = store->open_input(store, SEGMENTS_FILENAME);
-    SegmentInfos *sis = ALLOC(SegmentInfos);
-    sis->store = store;
+    bool success = false;
+    char seg_file_name[SEGMENT_NAME_MAX_LENGTH];
+    InStream *is = NULL;
+    SegmentInfos *sis = ALLOC_AND_ZERO(SegmentInfos);
+    segfn_for_generation(seg_file_name, fsf->generation);
+    TRY
+        is = store->open_input(store, seg_file_name);
+        sis->store = store;
 
-    sis->format = is_read_u32(is); /* do nothing. it's the first version */
-    sis->version = is_read_u64(is);
-    sis->counter = is_read_u64(is);
-    seg_cnt = is_read_vint(is);
+        sis->generation = fsf->generation;
+        sis->format = is_read_u32(is); /* do nothing. it's the first version */
+        sis->version = is_read_u64(is);
+        sis->counter = is_read_u64(is);
+        seg_cnt = is_read_vint(is);
 
-    /* allocate space for segments */
-    for (sis->capa = 4; sis->capa < seg_cnt; sis->capa <<= 1) {
-    }
-    sis->size = 0;
-    sis->segs = ALLOC_N(SegmentInfo *, sis->capa);
+        /* allocate space for segments */
+        for (sis->capa = 4; sis->capa < seg_cnt; sis->capa <<= 1) {
+        }
+        sis->size = 0;
+        sis->segs = ALLOC_N(SegmentInfo *, sis->capa);
 
-    for (i = 0; i < seg_cnt; i++) {
-        name = is_read_string(is);
-        doc_cnt = is_read_vint(is);
-        sis_add_si(sis, si_new(name, doc_cnt, store));
-    }
-    is_close(is);
+        for (i = 0; i < seg_cnt; i++) {
+            sis_add_si(sis, si_read(store, is));
+        }
+        sis->fis = fis_read(is);
+        success = true;
+    XFINALLY
+        if (is) is_close(is);
+        if (!success) {
+            sis_destroy(sis);
+        }
+    XENDTRY
+    fsf->p_return = sis;
+}
 
-    return sis;
+SegmentInfos *sis_read(Store *store)
+{
+    FindSegmentsFile fsf;
+    sis_find_segments_file(store, &fsf, &sis_read_i);
+    return fsf.p_return;
 }
 
-void sis_write(SegmentInfos *sis, Store *store)
+void sis_write(SegmentInfos *sis, Store *store, Deleter *deleter)
 {
     int i;
-    SegmentInfo *si;
-    OutStream *os = store->new_output(store, TEMPORARY_SEGMENTS_FILENAME);
+    OutStream *os = NULL;
     const int sis_size = sis->size;
+    char buf[SEGMENT_NAME_MAX_LENGTH];
+    sis->generation++;
 
-    os_write_u32(os, FORMAT);
-    os_write_u64(os, ++(sis->version)); /* every write changes the index */
-    os_write_u64(os, sis->counter);
-    os_write_vint(os, sis->size); 
-    for (i = 0; i < sis_size; i++) {
-        si = sis->segs[i];
-        os_write_string(os, si->name);
-        os_write_vint(os, si->doc_cnt);
-    }
-    os_close(os);
+    TRY
+        os = store->new_output(store,
+                               segfn_for_generation(buf, sis->generation));
+        os_write_u32(os, FORMAT);
+        os_write_u64(os, ++(sis->version)); /* every write changes the index */
+        os_write_u64(os, sis->counter);
+        os_write_vint(os, sis->size); 
+        for (i = 0; i < sis_size; i++) {
+            si_write(sis->segs[i], os);
+        }
+        fis_write(sis->fis, os);
+    XFINALLY
+        os_close(os);
+    XENDTRY
+
+    TRY
+        os = store->new_output(store, SEGMENTS_GEN_FILE_NAME);
+        os_write_u64(os, sis->generation);
+        os_write_u64(os, sis->generation);
+    XFINALLY
+        /* It's OK if we fail to write this file since it's
+         * used only as one of the retry fallbacks. */
+        HANDLED();
+        os_close(os);
+    XENDTRY
 
-    /* install new segment info */
-    store->rename(store, TEMPORARY_SEGMENTS_FILENAME, SEGMENTS_FILENAME);
+    if (deleter && sis->generation > 0) {
+        deleter_delete_file(deleter,
+                            segfn_for_generation(buf, sis->generation - 1));
+    }
 }
 
-f_u64 sis_read_current_version(Store *store)
+void sis_read_ver_i(Store *store, FindSegmentsFile *fsf)
 {
     InStream *is;
     f_u32 format = 0;
     f_u64 version = 0;
+    char seg_file_name[SEGMENT_NAME_MAX_LENGTH];
 
-    if (!store->exists(store, SEGMENTS_FILENAME)) {
-        return 0;
-    }
-    is = store->open_input(store, SEGMENTS_FILENAME);
+    segfn_for_generation(seg_file_name, (f_u64)fsf->generation);
+    is = store->open_input(store, seg_file_name);
 
     TRY
         format = is_read_u32(is);
@@ -668,7 +1119,14 @@ f_u64 sis_read_current_version(Store *store)
         is_close(is);
     XENDTRY
 
-    return version;
+    fsf->u64_return = version;
+}
+
+f_u64 sis_read_current_version(Store *store)
+{
+    FindSegmentsFile fsf;
+    sis_find_segments_file(store, &fsf, &sis_read_ver_i);
+    return fsf.u64_return;
 }
 
 /****************************************************************************
@@ -704,7 +1162,7 @@ char *lazy_df_get_data(LazyDocField *self, int i)
     char *text = NULL;
     if (i < self->size && i >= 0) {
         text = self->data[i].text;
-        if (text == NULL) {
+        if (NULL == text) {
             const int read_len = self->data[i].length + 1;
             self->data[i].text = text = ALLOC_N(char, read_len);
             is_seek(self->doc->fields_in, self->data[i].start);
@@ -1220,8 +1678,8 @@ char *te_skip_to(TermEnum *te, const char *term)
 {
     char *curr_term = te->curr_term;
     if (strcmp(curr_term, term) < 0) {
-        while (((curr_term = te->next(te)) != NULL) &&
-               (strcmp(curr_term, term) < 0)) {
+        while (NULL != ((curr_term = te->next(te)))
+               && (strcmp(curr_term, term) < 0)) {
         }
     }
     return curr_term;
@@ -1258,7 +1716,7 @@ static void sti_destroy(SegmentTermIndex *sti)
 static void sti_ensure_index_is_read(SegmentTermIndex *sti,
                                      TermEnum *index_te)
 {
-    if (sti->index_terms == NULL) {
+    if (NULL == sti->index_terms) {
         int i;
         int index_size = sti->index_size;
         off_t index_ptr = 0;
@@ -1314,7 +1772,7 @@ static int sti_get_index_offset(SegmentTermIndex *sti, const char *term)
  ****************************************************************************/
 
 #define SFI_ENSURE_INDEX_IS_READ(sfi, sti) do {\
-    if (sti->index_terms == NULL) {\
+    if (NULL == sti->index_terms) {\
         mutex_lock(&sfi->mutex);\
         sti_ensure_index_is_read(sti, sfi->index_te);\
         mutex_unlock(&sfi->mutex);\
@@ -1351,7 +1809,7 @@ SegmentFieldIndex *sfi_open(Store *store, const char *segment)
 
     sprintf(file_name, "%s.tix", segment);
     is = store->open_input(store, file_name);
-    sfi->index_te = ste_new(is, NULL);
+    sfi->index_te = ste_new(is, sfi);
     return sfi;
 }
 
@@ -1394,8 +1852,8 @@ static char *ste_next(TermEnum *te)
 
     ti = &(te->curr_ti);
     ti->doc_freq = is_read_vint(is);     /* read doc freq */
-    ti->frq_ptr += is_read_voff_t(is);/* read freq ptr */
-    ti->prx_ptr += is_read_voff_t(is);/* read prox ptr */
+    ti->frq_ptr += is_read_voff_t(is);   /* read freq ptr */
+    ti->prx_ptr += is_read_voff_t(is);   /* read prox ptr */
     if (ti->doc_freq >= STE(te)->skip_interval) {
         ti->skip_offset = is_read_voff_t(is);
     }
@@ -1497,7 +1955,7 @@ static TermInfo *ste_scan_for_term_info(SegmentTermEnum *ste, const char *term)
 {
     ste_scan_to(ste, term);
 
-    if (strcmp(TE(ste)->curr_term, term) == 0) {
+    if (0 == strcmp(TE(ste)->curr_term, term)) {
         return te_get_ti((TermEnum *)ste);
     }
     else {
@@ -1521,7 +1979,7 @@ static char *ste_get_term(TermEnum *te, int pos)
             ste_index_seek(te, sti, pos / idx_int);
         }
         while (ste->pos < pos) {
-            if (ste_next(te) == NULL) {
+            if (NULL == ste_next(te)) {
                 return NULL;
             }
         }
@@ -1575,7 +2033,7 @@ typedef struct MultiTermEnum
 static bool tew_lt(const TermEnumWrapper *tew1, const TermEnumWrapper *tew2)
 {
     int cmpres = strcmp(tew1->term, tew2->term);
-    if (cmpres == 0) {
+    if (0 == cmpres) {
         return tew1->index < tew2->index;
     }
     else {
@@ -1637,7 +2095,7 @@ static char *mte_next(TermEnum *te)
     TermEnumWrapper *top =
         (TermEnumWrapper *)pq_top(MTE(te)->tew_queue);
 
-    if (top == NULL) {
+    if (NULL == top) {
         te->curr_term[0] = '\0';
         te->curr_term_len = 0;
         return false;
@@ -1650,7 +2108,7 @@ static char *mte_next(TermEnum *te)
     te->curr_ti.doc_freq = 0;
 
     MTE(te)->ti_cnt = 0;
-    while ((top != NULL) && (strcmp(te->curr_term, top->term) == 0)) {
+    while ((NULL != top) && (0 == strcmp(te->curr_term, top->term))) {
         pq_pop(MTE(te)->tew_queue);
         te->curr_ti.doc_freq += top->te->curr_ti.doc_freq;/* increment freq */
         MTE(te)->ti_indexes[MTE(te)->ti_cnt] = top->index;
@@ -1752,7 +2210,7 @@ TermEnum *mte_new(MultiReader *mr, int field_num, const char *term)
         if (fnum >= 0) {
             TermEnumWrapper *tew;
 
-            if (term != NULL) {
+            if (NULL != term) {
                 sub_te = reader->terms_from(reader, fnum, term);
             }
             else {
@@ -1760,7 +2218,7 @@ TermEnum *mte_new(MultiReader *mr, int field_num, const char *term)
             }
 
             tew = tew_setup(&(mte->tews[i]), i, sub_te, reader);
-            if (((term == NULL) && tew_next(tew))
+            if (((NULL == term) && tew_next(tew))
                 || (tew->term && (tew->term[0] != '\0'))) {
                 pq_push(mte->tew_queue, tew);          /* initialize queue */
             }
@@ -1772,7 +2230,7 @@ TermEnum *mte_new(MultiReader *mr, int field_num, const char *term)
         }
     }
 
-    if ((term != NULL) && (mte->tew_queue->size > 0)) {
+    if ((NULL != term) && (0 < mte->tew_queue->size)) {
         mte_next(TE(mte));
     }
 
@@ -1804,7 +2262,7 @@ TermInfosReader *tir_open(Store *store,
 static __inline TermEnum *tir_enum(TermInfosReader *tir)
 {
     TermEnum *te;
-    if ((te = thread_getspecific(tir->thread_te)) == NULL) {
+    if (NULL == (te = thread_getspecific(tir->thread_te))) {
         te = ste_clone(tir->orig_te);
         ste_set_field(te, tir->field_num);
         ary_push(tir->te_bucket, te);
@@ -1827,8 +2285,8 @@ TermInfo *tir_get_ti(TermInfosReader *tir, const char *term)
     TermEnum *te = tir_enum(tir);
     char *match;
 
-    if ((match = ste_scan_to(te, term)) != NULL && 
-        strcmp(match, term) == 0) {
+    if (NULL != (match = ste_scan_to(te, term))
+        && 0 == strcmp(match, term)) {
         return &(te->curr_ti);
     }
     return NULL;
@@ -1845,8 +2303,8 @@ TermInfo *tir_get_ti_field(TermInfosReader *tir, int field_num,
         tir->field_num = field_num;
     }
 
-    if ((match = ste_scan_to(te, term)) != NULL && 
-        strcmp(match, term) == 0) {
+    if (NULL != (match = ste_scan_to(te, term))
+        && 0 == strcmp(match, term)) {
         return &(te->curr_ti);
     }
     return NULL;
@@ -1937,7 +2395,7 @@ static __inline void tw_write_term(TermWriter *tw,
 
     os_write_vint(os, start);                   /* write shared prefix length */
     os_write_vint(os, length);                  /* write delta length */
-    os_write_bytes(os, (uchar *)(term + start), length);   /* write delta chars */
+    os_write_bytes(os, (uchar *)(term + start), length); /* write delta chars */
 
     tw->last_term = term;
 }
@@ -1945,13 +2403,15 @@ static __inline void tw_write_term(TermWriter *tw,
 static void tw_add(TermWriter *tw,
                    const char *term,
                    int term_len,
-                   TermInfo *ti)
+                   TermInfo *ti,
+                   int skip_interval)
 {
     OutStream *os = tw->os;
 
 #ifdef DEBUG
     if (strcmp(tw->last_term, term) > 0) {
-        RAISE(STATE_ERROR, "\"%s\" > \"%s\" %d > %d", tw->last_term, term, *tw->last_term, *term);
+        RAISE(STATE_ERROR, "\"%s\" > \"%s\" %d > %d",
+              tw->last_term, term, *tw->last_term, *term);
     }
     if (ti->frq_ptr < tw->last_term_info.frq_ptr) {
         RAISE(STATE_ERROR, "%"F_OFF_T_PFX"d > %"F_OFF_T_PFX"d", ti->frq_ptr,
@@ -1967,6 +2427,9 @@ static void tw_add(TermWriter *tw,
     os_write_vint(os, ti->doc_freq);        /* write doc freq */
     os_write_voff_t(os, ti->frq_ptr - tw->last_term_info.frq_ptr);
     os_write_voff_t(os, ti->prx_ptr - tw->last_term_info.prx_ptr);
+    if (ti->doc_freq >= skip_interval) {
+        os_write_voff_t(os, ti->skip_offset);
+    }
 
     tw->last_term_info = *ti;
     tw->counter++;
@@ -1983,22 +2446,19 @@ void tiw_add(TermInfosWriter *tiw,
     printf("%s:%d:%d:%d:%d\n", term, term_len, ti->doc_freq,
            ti->frq_ptr, ti->prx_ptr);
     */
-    if ((tiw->tis_writer->counter % tiw->index_interval) == 0) {
+    if (0 == (tiw->tis_writer->counter % tiw->index_interval)) {
         /* add an index term */
         tw_add(tiw->tix_writer,
                tiw->tis_writer->last_term,
                strlen(tiw->tis_writer->last_term),
-               &(tiw->tis_writer->last_term_info));
+               &(tiw->tis_writer->last_term_info),
+               tiw->skip_interval);
         tis_pos = os_pos(tiw->tis_writer->os);
         os_write_voff_t(tiw->tix_writer->os, tis_pos - tiw->last_index_ptr);
         tiw->last_index_ptr = tis_pos;  /* write ptr */
     }
 
-    tw_add(tiw->tis_writer, term, term_len, ti);
-
-    if (ti->doc_freq >= tiw->skip_interval) {
-        os_write_voff_t(tiw->tis_writer->os, ti->skip_offset);
-    }
+    tw_add(tiw->tis_writer, term, term_len, ti, tiw->skip_interval);
 }
 
 static __inline void tw_reset(TermWriter *tw)
@@ -2051,7 +2511,7 @@ void tiw_close(TermInfosWriter *tiw)
 #define TDE(stde) ((TermDocEnum *)(stde))
 
 #define CHECK_STATE(method) do {\
-    if (STDE(tde)->count == 0) {\
+    if (0 == STDE(tde)->count) {\
         RAISE(STATE_ERROR, "Illegal state of TermDocEnum. You must call #next "\
               "before you call #"method);\
     }\
@@ -2059,7 +2519,7 @@ void tiw_close(TermInfosWriter *tiw)
 
 static void stde_seek_ti(SegmentTermDocEnum *stde, TermInfo *ti)
 {
-    if (ti == NULL) {
+    if (NULL == ti) {
         stde->doc_freq = 0;
     }
     else {
@@ -2117,7 +2577,7 @@ static bool stde_next(TermDocEnum *tde)
 
         doc_code = is_read_vint(stde->frq_in);
         stde->doc_num += doc_code >> 1;    /* shift off low bit */
-        if ((doc_code & 1) != 0) {         /* if low bit is set */
+        if (0 != (doc_code & 1)) {         /* if low bit is set */
             stde->freq = 1;                /* freq is one */
         }
         else {
@@ -2126,8 +2586,8 @@ static bool stde_next(TermDocEnum *tde)
 
         stde->count++;
 
-        if (stde->deleted_docs == NULL ||
-            bv_get(stde->deleted_docs, stde->doc_num) == 0) {
+        if (NULL == stde->deleted_docs
+            || 0 == bv_get(stde->deleted_docs, stde->doc_num)) {
             break; /* We found an undeleted doc so return */
         }
 
@@ -2146,7 +2606,7 @@ static int stde_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
         /* manually inlined call to next() for speed */
         doc_code = is_read_vint(stde->frq_in);
         stde->doc_num += (doc_code >> 1);            /* shift off low bit */
-        if ((doc_code & 1) != 0) {                   /* if low bit is set */
+        if (0 != (doc_code & 1)) {                   /* if low bit is set */
             stde->freq = 1;                            /* freq is one */
         }
         else {
@@ -2155,8 +2615,8 @@ static int stde_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
 
         stde->count++;
 
-        if (stde->deleted_docs == NULL ||
-            bv_get(stde->deleted_docs, stde->doc_num) == 0) {
+        if (NULL == stde->deleted_docs
+            || 0 == bv_get(stde->deleted_docs, stde->doc_num)) {
             docs[i] = stde->doc_num;
             freqs[i] = stde->freq;
             i++;
@@ -2169,16 +2629,18 @@ static bool stde_skip_to(TermDocEnum *tde, int target_doc_num)
 {
     SegmentTermDocEnum *stde = STDE(tde);
 
-    if (stde->doc_freq >= stde->skip_interval) { /* optimized case */
+    if (stde->doc_freq >= stde->skip_interval
+        && target_doc_num > stde->doc_num) {       /* optimized case */
         int last_skip_doc;
-        int last_frq_ptr;
-        int last_prx_ptr;
+        off_t last_frq_ptr;
+        off_t last_prx_ptr;
         int num_skipped;
 
-        if (stde->skip_in == NULL) {
-            stde->skip_in = is_clone(stde->frq_in); /* lazily clone */
+        if (NULL == stde->skip_in) {
+            stde->skip_in = is_clone(stde->frq_in);/* lazily clone */
         }
 
+        //printf("skip_ptr = %lld\n", stde->skip_ptr);
         if (!stde->have_skipped) {                 /* lazily seek skip stream */
             is_seek(stde->skip_in, stde->skip_ptr);
             stde->have_skipped = true;
@@ -2189,13 +2651,14 @@ static bool stde_skip_to(TermDocEnum *tde, int target_doc_num)
         last_frq_ptr = is_pos(stde->frq_in);
         last_prx_ptr = -1;
         num_skipped = -1 - (stde->count % stde->skip_interval);
+        //printf("%d, %d, %d, %d\n", last_skip_doc, last_frq_ptr, last_prx_ptr, num_skipped);
 
         while (target_doc_num > stde->skip_doc) {
             last_skip_doc = stde->skip_doc;
             last_frq_ptr = stde->frq_ptr;
             last_prx_ptr = stde->prx_ptr;
 
-            if (stde->skip_doc != 0 && stde->skip_doc >= stde->doc_num) {
+            if (0 != stde->skip_doc && stde->skip_doc >= stde->doc_num) {
                 num_skipped += stde->skip_interval;
             }
 
@@ -2204,13 +2667,14 @@ static bool stde_skip_to(TermDocEnum *tde, int target_doc_num)
             }
 
             stde->skip_doc += is_read_vint(stde->skip_in);
-            stde->frq_ptr += is_read_vint(stde->skip_in);
-            stde->prx_ptr += is_read_vint(stde->skip_in);
+            stde->frq_ptr  += is_read_vint(stde->skip_in);
+            stde->prx_ptr  += is_read_vint(stde->skip_in);
+            //printf("inner-> skip_doc:%d, frq_ptr:%d, prx_ptr:%d\n", stde->skip_doc, stde->frq_ptr, stde->prx_ptr);
 
             stde->skip_count++;
         }
 
-        /* if we found something to skip, so skip it */
+        /* if we found something to skip, skip it */
         if (last_frq_ptr > is_pos(stde->frq_in)) {
             is_seek(stde->frq_in, last_frq_ptr);
             stde->seek_prox(stde, last_prx_ptr);
@@ -2233,7 +2697,7 @@ static void stde_close(TermDocEnum *tde)
 {
     is_close(STDE(tde)->frq_in);
 
-    if (STDE(tde)->skip_in != NULL) {
+    if (NULL != STDE(tde)->skip_in) {
         is_close(STDE(tde)->skip_in);
     }
 
@@ -2245,7 +2709,7 @@ static void stde_skip_prox(SegmentTermDocEnum *stde)
     (void)stde;
 }
 
-static void stde_seek_prox(SegmentTermDocEnum *stde, int prx_ptr)
+static void stde_seek_prox(SegmentTermDocEnum *stde, off_t prx_ptr)
 { 
     (void)stde;
     (void)prx_ptr;
@@ -2290,7 +2754,7 @@ TermDocEnum *stde_new(TermInfosReader *tir,
 
 static void stpe_seek_ti(SegmentTermDocEnum *stde, TermInfo *ti)
 {
-    if (ti == NULL) {
+    if (NULL == ti) {
         stde->doc_freq = 0;
     }
     else {
@@ -2351,7 +2815,7 @@ static void stpe_skip_prox(SegmentTermDocEnum *stde)
     is_skip_vints(stde->prx_in, stde->freq);
 }
 
-static void stpe_seek_prox(SegmentTermDocEnum *stde, int prx_ptr)
+static void stpe_seek_prox(SegmentTermDocEnum *stde, off_t prx_ptr)
 {
     is_seek(stde->prx_in, prx_ptr);
     stde->prx_cnt = 0;
@@ -2422,7 +2886,7 @@ static TermDocEnum *mtde_next_tde(MultiTermDocEnum *mtde)
 }
 
 #define CHECK_CURR_TDE(method) do {\
-    if (MTDE(tde)->curr_tde == NULL) {\
+    if (NULL == MTDE(tde)->curr_tde) {\
         RAISE(STATE_ERROR, "Illegal state of TermDocEnum. You must call #next "\
               "before you call #"method);\
     }\
@@ -2456,7 +2920,7 @@ static void mtde_seek(TermDocEnum *tde, int field_num, const char *term)
     TermEnum *te = mtde->te;
     char *t;
     te->set_field(te, field_num);
-    if ((t = te->skip_to(te, term)) != NULL && strcmp(term, t) == 0) {
+    if (NULL != (t = te->skip_to(te, term)) && 0 == strcmp(term, t)) {
         mtde_seek_te(tde, te);
     } else {
         memset(mtde->state, 0, mtde->ir_cnt);
@@ -2478,7 +2942,7 @@ static int mtde_freq(TermDocEnum *tde)
 static bool mtde_next(TermDocEnum *tde)
 {
     MultiTermDocEnum *mtde = MTDE(tde);
-    if (mtde->curr_tde != NULL && mtde->curr_tde->next(mtde->curr_tde)) {
+    if (NULL != mtde->curr_tde && mtde->curr_tde->next(mtde->curr_tde)) {
         return true;
     }
     else if (mtde_next_tde(mtde)) {
@@ -2494,7 +2958,7 @@ static int mtde_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
     int i, end = 0, last_end = 0, b;
     MultiTermDocEnum *mtde = MTDE(tde);
     while (true) {
-        if (mtde->curr_tde == NULL) return end;
+        if (NULL == mtde->curr_tde) return end;
         end += mtde->curr_tde->read(mtde->curr_tde, docs + last_end,
                                     freqs + last_end, req_num - last_end);
         if (end == last_end) {              /* none left in segment */
@@ -2527,13 +2991,7 @@ static bool mtde_skip_to(TermDocEnum *tde, int target_doc_num)
 
         mtde_next_tde(mtde);
     }
-
-    if (curr_tde) {
-        return curr_tde->skip_to(curr_tde, target_doc_num - mtde->base);
-    }
-    else {
-        return false;
-    }
+    return false;
 }
 
 static void mtde_close(TermDocEnum *tde)
@@ -2660,7 +3118,7 @@ static bool mtdpe_next(TermDocEnum *tde)
     int doc;
     MultipleTermDocPosEnum *mtdpe = MTDPE(tde);
 
-    if (mtdpe->pq->size == 0) {
+    if (0 == mtdpe->pq->size) {
         return false;
     }
 
@@ -2710,7 +3168,7 @@ bool mtdpe_skip_to(TermDocEnum *tde, int target_doc_num)
     TermDocEnum *sub_tde;
     PriorityQueue *mtdpe_pq = MTDPE(tde)->pq;
 
-    while ((sub_tde = (TermDocEnum *)pq_top(mtdpe_pq)) != NULL
+    while (NULL != (sub_tde = (TermDocEnum *)pq_top(mtdpe_pq))
            && (target_doc_num > sub_tde->doc_num(sub_tde))) {
         if (sub_tde->skip_to(sub_tde, target_doc_num)) {
             pq_down(mtdpe_pq);
@@ -2779,6 +3237,256 @@ TermDocEnum *mtdpe_new(IndexReader *ir, int field_num, char **terms, int t_cnt)
     return tde;
 }
 
+/****************************************************************************
+ *
+ * FileNameFilter
+ *
+ ****************************************************************************/
+
+static HashTable *fn_extensions = NULL;
+static void file_name_filter_init()
+{
+    if (NULL == fn_extensions) {
+        int i;
+        fn_extensions = h_new_str((free_ft)NULL, (free_ft)NULL);
+        for (i = 0; i < NELEMS(INDEX_EXTENSIONS); i++) {
+          h_set(fn_extensions, INDEX_EXTENSIONS[i], (char *)INDEX_EXTENSIONS[i]);
+        }
+        register_for_cleanup(fn_extensions, (free_ft)&h_destroy);
+    }
+}
+
+static bool file_name_filter_accept(char *file_name)
+{
+    char *p = strrchr(file_name, '.');
+    if (NULL != p) {
+        char *extension = p + 1;
+        if (NULL != h_get(fn_extensions, extension)) {
+            return true;
+        }
+        else if ((*extension == 'f' || *extension == 's')
+                 && *(extension + 1) >= '0'
+                 && *(extension + 1) <= '9') {
+            return true;
+        }
+    }
+    else if (0 == strncmp(SEGMENTS_FILE_NAME, file_name,
+                          sizeof(SEGMENTS_FILE_NAME) - 1)) {
+        return true;
+    }
+    return false;
+}
+
+/*
+ * Returns true if this is a file that would be contained in a CFS file.  This
+ * function should only be called on files that pass the above "accept" (ie,
+ * are already known to be a Lucene index file).
+ */
+static bool file_name_filter_is_cfs_file(char *file_name) {
+    char *p = strrchr(file_name, '.');
+    if (NULL != p) {
+        char *extension = p + 1;
+        if (NULL != h_get(fn_extensions, extension)
+            && 0 != strcmp(extension, "del")
+            && 0 != strcmp(extension, "gen")
+            && 0 != strcmp(extension, "cfs")) {
+            return true;
+        }
+        else if ('f' == *extension
+                 && '0' <= *(extension + 1)
+                 && '9' >= *(extension + 1)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/****************************************************************************
+ *
+ * Deleter
+ *
+ ****************************************************************************/
+
+#define DELETABLE_START_CAPA 8
+Deleter *deleter_new(SegmentInfos *sis, Store *store)
+{
+    Deleter *dlr = ALLOC(Deleter);
+    dlr->sis = sis;
+    dlr->store = store;
+    dlr->pending = hs_new_str(&free);
+    return dlr;
+}
+
+void deleter_destroy(Deleter *dlr)
+{
+    hs_destroy(dlr->pending);
+    free(dlr);
+}
+
+void deleter_queue_file(Deleter *dlr, char *file_name)
+{
+    hs_add(dlr->pending, estrdup(file_name));
+}
+
+void deleter_delete_file(Deleter *dlr, char *file_name)
+{
+    Store *store = dlr->store;
+    TRY
+        if (store->exists(store, file_name)) {
+            store->remove(store, file_name);
+        }
+        hs_del(dlr->pending, file_name);
+    XCATCHALL
+        hs_add(dlr->pending, estrdup(file_name));
+    XENDTRY
+}
+
+void deleter_commit_pending_deletions(Deleter *dlr)
+{
+    int i;
+    char **pending = (char **)dlr->pending->elems;
+    for (i = dlr->pending->size - 1; i >= 0; i--) {
+        deleter_delete_file(dlr, pending[i]);
+    }
+}
+
+void deleter_delete_files(Deleter *dlr, char **files, int file_cnt)
+{
+    int i;
+    for (i = file_cnt - 1; i >= 0; i--) {
+        deleter_queue_file(dlr, files[i]);
+    }
+    deleter_commit_pending_deletions(dlr);
+}
+
+struct DelFilesArg {
+    char  curr_seg_file_name[SEGMENT_NAME_MAX_LENGTH];
+    Deleter *dlr;
+    HashTable *current;
+};
+
+static void deleter_find_deletable_files_i(char *file_name, void *arg)
+{
+    struct DelFilesArg *dfa = (struct DelFilesArg *)arg;
+    Deleter *dlr = dfa->dlr;
+
+    if (file_name_filter_accept(file_name)
+        && 0 != strcmp(file_name, dfa->curr_seg_file_name)
+        && 0 != strcmp(file_name, SEGMENTS_GEN_FILE_NAME)) {
+
+        bool do_delete = false;
+        SegmentInfo *si;
+        char segment_name[SEGMENT_NAME_MAX_LENGTH];
+        char *extension, *p;
+        strcpy(segment_name, file_name);
+
+        p = strrchr(segment_name, '.');
+
+        /* First remove any extension: */
+        if (NULL != p) {
+            *p = '\0';
+            extension = p + 1;
+        } else {
+            extension = NULL;
+        }
+
+        /* Then, remove any generation count: */
+        p = strrchr(segment_name + 1, '_');
+        if (NULL != p) {
+            *p = '\0';
+        }
+
+        /* Delete this file if it's not a "current" segment, or, it is a
+         * single index file but there is now a corresponding compound file: */
+        if (NULL == (si = h_get(dfa->current, segment_name))) {
+            /* Delete if segment is not referenced: */
+            do_delete = true;
+        }
+        else {
+            char tmp_fn[SEGMENT_NAME_MAX_LENGTH];
+            /* OK, segment is referenced, but file may still be orphan'd: */
+            if (file_name_filter_is_cfs_file(file_name)
+                && si->use_compound_file) {
+                /* This file is stored in a CFS file for this segment: */
+                do_delete = true;
+            }
+            else if (0 == strcmp("del", extension)) {
+                /* This is a _segmentName_N.del file: */
+                if (!fn_for_generation(tmp_fn, segment_name, "del", si->del_gen)
+                    || 0 != strcmp(file_name, tmp_fn)) {
+                    /* If this is a seperate .del file, but it
+                     * doesn't match the current del file name for
+                     * this segment, then delete it: */
+                    do_delete = true;
+                }
+            }
+            else if (NULL != extension
+                     && ('s' == *extension || 'f' == *extension)
+                     && isdigit(extension[1])) {
+                si_norm_file_name(si, tmp_fn, atoi(extension + 1));
+                /* This is a _segmentName_N.sX file: */
+                if (0 != strcmp(tmp_fn, file_name)) {
+                    /* This is an orphan'd norms file: */
+                    do_delete = true;
+                }
+            }
+            else if (0 == strcmp("cfs", extension) && !si->use_compound_file) {
+                /* This is a partially written _segmentName.cfs: */
+                do_delete = true;
+            }
+        }
+
+        if (do_delete) {
+            deleter_queue_file(dlr, file_name);
+        }
+    }
+}
+
+/*
+ * Determine index files that are no longer referenced and therefore should be
+ * deleted.  This is called once (by the writer), and then subsequently we add
+ * onto deletable any files that are no longer needed at the point that we
+ * create the unused file (eg when merging segments), and we only remove from
+ * deletable when a file is successfully deleted.
+ */
+void deleter_find_deletable_files(Deleter *dlr)
+{
+    /* Gather all "current" segments: */
+    int i;
+    SegmentInfos *sis = dlr->sis;
+    Store *store = dlr->store;
+    struct DelFilesArg dfa;
+    HashTable *current = dfa.current
+                       = h_new_str((free_ft)NULL, (free_ft)si_deref);
+    dfa.dlr = dlr;
+
+    for(i = 0; i < sis->size; i++) {
+        SegmentInfo *si = (SegmentInfo *)sis->segs[i];
+        REF(si);
+        h_set(current, si->name, si);
+    }
+
+    /* Then go through all files in the Directory that are Ferret index files,
+     * and add to deletable if they are not referenced by the current segments
+     * info: */
+    sis_curr_seg_file_name(dfa.curr_seg_file_name, store);
+    file_name_filter_init();
+
+    store->each(store, &deleter_find_deletable_files_i, &dfa);
+    h_destroy(dfa.current);
+}
+
+void deleter_delete_deletable_files(Deleter *dlr)
+{
+    deleter_find_deletable_files(dlr);
+    deleter_commit_pending_deletions(dlr);
+}
+
+void deleter_clear_pending_deletions(Deleter *dlr)
+{
+    hs_clear(dlr->pending);
+}
+
 /****************************************************************************
  *
  * IndexReader
@@ -2800,7 +3508,7 @@ void ir_acquire_write_lock(IndexReader *ir)
                            "need to close and reopen the index");
     }
 
-    if (ir->write_lock == NULL) {
+    if (NULL == ir->write_lock) {
         ir->write_lock = open_lock(ir->store, WRITE_LOCK_NAME);
         if (!ir->write_lock->obtain(ir->write_lock)) {/* obtain write lock */
             RAISE(LOCK_ERROR, "Could not obtain write lock when trying to "
@@ -2811,8 +3519,8 @@ void ir_acquire_write_lock(IndexReader *ir)
                               "you can safely delete these files.");
         }
 
-        /* we have to check whether index has changed since this reader was opened.
-         * if so, this reader is no longer valid for deletion */
+        /* we have to check whether index has changed since this reader was
+         * opened.  if so, this reader is no longer valid for deletion */
         if (sis_read_current_version(ir->store) > ir->sis->version) {
             ir->is_stale = true;
             ir->write_lock->release(ir->write_lock);
@@ -2856,7 +3564,7 @@ IndexReader *ir_setup(IndexReader *ir, Store *store, SegmentInfos *sis,
 
 bool ir_index_exists(Store *store)
 {
-    return store->exists(store, "segments");
+    return sis_current_segment_generation(store) != 1;
 }
 
 int ir_get_field_num(IndexReader *ir, const char *field)
@@ -2903,7 +3611,7 @@ uchar *ir_get_norms_i(IndexReader *ir, int field_num)
         norms = ir->get_norms(ir, field_num);
     }
     if (!norms) {
-        if (ir->fake_norms == NULL) {
+        if (NULL == ir->fake_norms) {
             ir->fake_norms = (uchar *)ecalloc(ir->max_doc(ir));
         }
         norms = ir->fake_norms;
@@ -3009,34 +3717,41 @@ TermDocEnum *ir_term_positions_for(IndexReader *ir, const char *field,
 
 void ir_commit_i(IndexReader *ir)
 {
-    if (ir->has_changes && ir->is_owner) {
-        Lock *commit_lock;
-
-        mutex_lock(&ir->store->mutex);
-        commit_lock = open_lock(ir->store, COMMIT_LOCK_NAME);
-        if (!commit_lock->obtain(commit_lock)) { /* obtain write lock */
-            RAISE(LOCK_ERROR, "Error trying to commit the index. Commit "
-                              "lock already obtained");
+    if (ir->has_changes) {
+        if (NULL == ir->deleter && NULL != ir->store) {
+            /* In the MultiReader case, we share this deleter across all
+             * SegmentReaders: */
+            ir->set_deleter_i(ir, deleter_new(ir->sis, ir->store));
         }
+        if (ir->is_owner) {
+            char curr_seg_fn[MAX_FILE_PATH];
+            mutex_lock(&ir->store->mutex);
 
-        ir->commit_i(ir);
-        sis_write(ir->sis, ir->store);
+            /* Should not be necessary: no prior commit should have left
+             * pending files, so just defensive: */
+            if (ir->deleter) deleter_clear_pending_deletions(ir->deleter);
 
-        commit_lock->release(commit_lock);
-        close_lock(commit_lock);
-        mutex_unlock(&ir->store->mutex);
+            sis_curr_seg_file_name(curr_seg_fn, ir->store);
+            
+            ir->commit_i(ir);
+            sis_write(ir->sis, ir->store, ir->deleter);
 
-        if (ir->write_lock != NULL) {
-            /* release write lock */
-            ir->write_lock->release(ir->write_lock);
-            close_lock(ir->write_lock);
-            ir->write_lock = NULL;
+            if (ir->deleter) deleter_delete_file(ir->deleter, curr_seg_fn);
+
+            mutex_unlock(&ir->store->mutex);
+
+            if (NULL != ir->write_lock) {
+                /* release write lock */
+                ir->write_lock->release(ir->write_lock);
+                close_lock(ir->write_lock);
+                ir->write_lock = NULL;
+            }
+        }
+        else {
+            ir->commit_i(ir);
         }
-        ir->has_changes = false;
-    }
-    else {
-        ir->commit_i(ir);
     }
+    ir->has_changes = false;
 }
 
 void ir_commit(IndexReader *ir)
@@ -3049,15 +3764,14 @@ void ir_commit(IndexReader *ir)
 void ir_close(IndexReader *ir)
 {
     mutex_lock(&ir->mutex);
-    if (--(ir->ref_cnt) == 0) {
+    if (0 == --(ir->ref_cnt)) {
         ir_commit_i(ir);
         ir->close_i(ir);
         if (ir->store) {
             store_deref(ir->store);
         }
-        if (ir->is_owner) {
+        if (ir->is_owner && ir->sis) {
             sis_destroy(ir->sis);
-            fis_deref(ir->fis);
         }
         if (ir->cache) {
             h_destroy(ir->cache);
@@ -3065,6 +3779,9 @@ void ir_close(IndexReader *ir)
         if (ir->sort_cache) {
             h_destroy(ir->sort_cache);
         }
+        if (ir->deleter && ir->is_owner) {
+            deleter_destroy(ir->deleter);
+        }
         free(ir->fake_norms);
 
         mutex_destroy(&ir->mutex);
@@ -3080,26 +3797,14 @@ void ir_close(IndexReader *ir)
  **/
 void ir_add_cache(IndexReader *ir)
 {
-    if (ir->cache == NULL) {
+    if (NULL == ir->cache) {
         ir->cache = co_hash_create();
     }
 }
 
 bool ir_is_latest(IndexReader *ir)
 {
-    volatile bool is_latest = false;
-
-    Lock *commit_lock = open_lock(ir->store, COMMIT_LOCK_NAME);
-    if (!commit_lock->obtain(commit_lock)) {
-        close_lock(commit_lock);
-        RAISE(LOCK_ERROR, "Error detecting if the current index is latest "
-              "version. Commit lock currently obtained");
-    }
-    is_latest = (sis_read_current_version(ir->store) == ir->sis->version);
-    commit_lock->release(commit_lock);
-    close_lock(commit_lock);
-
-    return is_latest;
+    return (sis_read_current_version(ir->store) == ir->sis->version);
 }
 
 /****************************************************************************
@@ -3128,35 +3833,27 @@ static Norm *norm_create(InStream *is, int field_num)
 static void norm_destroy(Norm *norm)
 {
     is_close(norm->is);
-    if (norm->bytes != NULL) {
+    if (NULL != norm->bytes) {
         free(norm->bytes);
     }
     free(norm);
 }
 
-static void norm_rewrite(Norm *norm, Store *store, char *segment,
-                  int doc_count, Store *cfs_store)
+static void norm_rewrite(Norm *norm, Store *store, Deleter *dlr,
+                         SegmentInfo *si, int doc_count)
 {
     OutStream *os;
-    char tmp_file_name[SEGMENT_NAME_MAX_LENGTH];
     char norm_file_name[SEGMENT_NAME_MAX_LENGTH];
+    const int field_num = norm->field_num;
 
-    if (norm == NULL || norm->bytes == NULL) {
-        return; /* These norms do not need to be rewritten */
+    if (si_norm_file_name(si, norm_file_name, field_num)) {
+        deleter_queue_file(dlr, norm_file_name);
     }
-
-    sprintf(tmp_file_name, "%s.tmp", segment);
-    os = store->new_output(store, tmp_file_name);
+    si_advance_norm_gen(si, field_num);
+    si_norm_file_name(si, norm_file_name, field_num);
+    os = store->new_output(store, norm_file_name);
     os_write_bytes(os, norm->bytes, doc_count);
     os_close(os);
-
-    if (cfs_store) {
-        sprintf(norm_file_name, "%s.s%d", segment, norm->field_num);
-    }
-    else {
-        sprintf(norm_file_name, "%s.f%d", segment, norm->field_num);
-    }
-    store->rename(store, tmp_file_name, norm_file_name);
     norm->is_dirty = false;
 }
 
@@ -3166,6 +3863,7 @@ static void norm_rewrite(Norm *norm, Store *store, char *segment,
 
 typedef struct SegmentReader {
     IndexReader ir;
+    SegmentInfo *si;
     char *segment;
     FieldsReader *fr;
     BitVector *deleted_docs;
@@ -3191,7 +3889,7 @@ static __inline FieldsReader *sr_fr(SegmentReader *sr)
 {
     FieldsReader *fr;
 
-    if ((fr = thread_getspecific(sr->thread_fr)) == NULL) {
+    if (NULL == (fr = thread_getspecific(sr->thread_fr))) {
         fr = fr_clone(sr->fr);
         ary_push(sr->fr_bucket, fr);
         thread_setspecific(sr->thread_fr, fr);
@@ -3201,17 +3899,17 @@ static __inline FieldsReader *sr_fr(SegmentReader *sr)
 
 static __inline bool sr_is_deleted_i(SegmentReader *sr, int doc_num)
 {
-    return (sr->deleted_docs != NULL && bv_get(sr->deleted_docs, doc_num));
+    return (NULL != sr->deleted_docs && bv_get(sr->deleted_docs, doc_num));
 }
 
 static __inline void sr_get_norms_into_i(SegmentReader *sr, int field_num,
                                        uchar *buf)
 {
     Norm *norm = h_get_int(sr->norms, field_num);
-    if (norm == NULL) {
+    if (NULL == norm) {
         memset(buf, 0, SR_SIZE(sr));
     }
-    else if (norm->bytes != NULL) { /* can copy from cache */
+    else if (NULL != norm->bytes) { /* can copy from cache */
         memcpy(buf, norm->bytes, SR_SIZE(sr));
     }
     else {
@@ -3226,11 +3924,11 @@ static __inline void sr_get_norms_into_i(SegmentReader *sr, int field_num,
 static __inline uchar *sr_get_norms_i(SegmentReader *sr, int field_num)
 {
     Norm *norm = h_get_int(sr->norms, field_num);
-    if (norm == NULL) {                           /* not an indexed field */
+    if (NULL == norm) {                           /* not an indexed field */
         return NULL;
     }
 
-    if (norm->bytes == NULL) {                    /* value not yet read */
+    if (NULL == norm->bytes) {                    /* value not yet read */
         uchar *bytes = ALLOC_N(uchar, SR_SIZE(sr));
         sr_get_norms_into_i(sr, field_num, bytes);
         norm->bytes = bytes;                        /* cache it */
@@ -3241,7 +3939,8 @@ static __inline uchar *sr_get_norms_i(SegmentReader *sr, int field_num)
 static void sr_set_norm_i(IndexReader *ir, int doc_num, int field_num, uchar b)
 {
     Norm *norm = h_get_int(SR(ir)->norms, field_num);
-    if (norm != NULL) { /* has_norms */
+    if (NULL != norm) { /* has_norms */
+        ir->has_changes = true;
         norm->is_dirty = true; /* mark it dirty */
         SR(ir)->norms_dirty = true;
         sr_get_norms_i(SR(ir), field_num)[doc_num] = b;
@@ -3250,12 +3949,13 @@ static void sr_set_norm_i(IndexReader *ir, int doc_num, int field_num, uchar b)
 
 static void sr_delete_doc_i(IndexReader *ir, int doc_num) 
 {
-    if (SR(ir)->deleted_docs == NULL) {
+    if (NULL == SR(ir)->deleted_docs) {
         SR(ir)->deleted_docs = bv_new();
     }
 
     SR(ir)->deleted_docs_dirty = true;
     SR(ir)->undelete_all = false;
+    ir->has_changes = true;
     bv_set(SR(ir)->deleted_docs, doc_num);
 }
 
@@ -3263,12 +3963,18 @@ static void sr_undelete_all_i(IndexReader *ir)
 {
     SR(ir)->undelete_all = true;
     SR(ir)->deleted_docs_dirty = false;
-    if (SR(ir)->deleted_docs != NULL) {
+    ir->has_changes = true;
+    if (NULL != SR(ir)->deleted_docs) {
         bv_destroy(SR(ir)->deleted_docs);
     }
     SR(ir)->deleted_docs = NULL;
 }
 
+static void sr_set_deleter_i(IndexReader *ir, Deleter *deleter)
+{
+  ir->deleter = deleter;
+}
+
 static void bv_write(BitVector *bv, Store *store, char *name)
 {
     int i;
@@ -3299,64 +4005,61 @@ static BitVector *bv_read(Store *store, char *name)
 
 static void sr_commit_i(IndexReader *ir)
 {
+    SegmentInfo *si = SR(ir)->si;
+    char *segment = SR(ir)->si->name;
     char tmp_file_name[SEGMENT_NAME_MAX_LENGTH];
-    char del_file_name[SEGMENT_NAME_MAX_LENGTH];
-
-    sprintf(del_file_name, "%s.del", SR(ir)->segment);
 
-    if (SR(ir)->deleted_docs_dirty) { /* re-write deleted */
-        sprintf(tmp_file_name, "%s.tmp", SR(ir)->segment);
-        bv_write(SR(ir)->deleted_docs, ir->store, tmp_file_name);
-        ir->store->rename(ir->store, tmp_file_name, del_file_name);
-    }
-    if (SR(ir)->undelete_all && ir->store->exists(ir->store, del_file_name)) {
-        ir->store->remove(ir->store, del_file_name);
+    if (SR(ir)->undelete_all || SR(ir)->deleted_docs_dirty) {
+        if (si->del_gen >= 0) {
+            fn_for_generation(tmp_file_name, segment, "del", si->del_gen);
+            deleter_queue_file(ir->deleter, tmp_file_name);
+        }
+        if (SR(ir)->undelete_all) {
+            si->del_gen = -1;
+            SR(ir)->undelete_all = false;
+        } else {
+            /* (SR(ir)->deleted_docs_dirty) re-write deleted */
+            si->del_gen++;
+            fn_for_generation(tmp_file_name, segment, "del", si->del_gen);
+            bv_write(SR(ir)->deleted_docs, ir->store, tmp_file_name);
+            SR(ir)->deleted_docs_dirty = false;
+        }
     }
     if (SR(ir)->norms_dirty) { /* re-write norms */
         int i;
         const int field_cnt = ir->fis->size;
         FieldInfo *fi;
-        for (i = 0; i < field_cnt; i++) {
+        for (i = field_cnt - 1; i >= 0; i--) {
             fi = ir->fis->fields[i];
             if (fi_is_indexed(fi)) {
-                norm_rewrite(h_get_int(SR(ir)->norms, fi->number), ir->store,
-                             SR(ir)->segment, SR_SIZE(ir), SR(ir)->cfs_store);
+                Norm *norm = h_get_int(SR(ir)->norms, fi->number);
+                if (norm && norm->is_dirty) {
+                    norm_rewrite(norm, ir->store, ir->deleter, SR(ir)->si,
+                                 SR_SIZE(ir));
+                }
             }
         }
+        SR(ir)->norms_dirty = false;
     }
-    SR(ir)->deleted_docs_dirty = false;
-    SR(ir)->norms_dirty = false;
-    SR(ir)->undelete_all = false;
 }
 
 static void sr_close_i(IndexReader *ir)
 {
     SegmentReader *sr = SR(ir);
 
-    fr_close(sr->fr);
-    tir_close(sr->tir);
-    sfi_close(sr->sfi);
-
-    if (sr->frq_in) {
-        is_close(sr->frq_in);
-    }
-    if (sr->prx_in) {
-        is_close(sr->prx_in);
-    }
-
-    h_destroy(sr->norms);
-
+    if (sr->fr)           fr_close(sr->fr);
+    if (sr->tir)          tir_close(sr->tir);
+    if (sr->sfi)          sfi_close(sr->sfi);
+    if (sr->frq_in)       is_close(sr->frq_in);
+    if (sr->prx_in)       is_close(sr->prx_in);
+    if (sr->norms)        h_destroy(sr->norms);
+    if (sr->deleted_docs) bv_destroy(sr->deleted_docs);
+    if (sr->cfs_store)    store_deref(sr->cfs_store);
     if (sr->fr_bucket) {
         thread_setspecific(sr->thread_fr, NULL);
         thread_key_delete(sr->thread_fr);
         ary_destroy(sr->fr_bucket, (free_ft)&fr_close);
     }
-    if (sr->deleted_docs) {
-        bv_destroy(sr->deleted_docs);
-    }
-    if (sr->cfs_store) {
-        store_deref(sr->cfs_store);
-    }
 }
 
 static int sr_num_docs(IndexReader *ir)
@@ -3365,7 +4068,7 @@ static int sr_num_docs(IndexReader *ir)
 
     mutex_lock(&ir->mutex);
     num_docs = SR(ir)->fr->size;
-    if (SR(ir)->deleted_docs != NULL) {
+    if (NULL != SR(ir)->deleted_docs) {
         num_docs -= SR(ir)->deleted_docs->count;
     }
     mutex_unlock(&ir->mutex);
@@ -3473,7 +4176,7 @@ static TermVector *sr_term_vector(IndexReader *ir, int doc_num,
 static HashTable *sr_term_vectors(IndexReader *ir, int doc_num)
 {
     FieldsReader *fr;
-    if (!SR(ir)->fr || (fr = sr_fr(SR(ir))) == NULL) {
+    if (!SR(ir)->fr || NULL == (fr = sr_fr(SR(ir)))) {
         return NULL;
     }
 
@@ -3493,42 +4196,32 @@ static bool sr_is_deleted(IndexReader *ir, int doc_num)
 
 static bool sr_has_deletions(IndexReader *ir)
 {
-    return (SR(ir)->deleted_docs != NULL);
+    return NULL != SR(ir)->deleted_docs;
 }
 
 static void sr_open_norms(IndexReader *ir, Store *cfs_store)
 {
     int i;
-    Store *store = ir->store;
+    SegmentInfo *si = SR(ir)->si;
     char file_name[SEGMENT_NAME_MAX_LENGTH];
-    FieldInfos *fis = ir->fis;
-    char *ext_ptr;
-    const int field_cnt = fis->size;
-
-    sprintf(file_name, "%s.", SR(ir)->segment);
-    ext_ptr = file_name + strlen(file_name);
 
-    for (i = 0; i < field_cnt; i++) {
-        if (fi_has_norms(fis->fields[i])) {
-            sprintf(ext_ptr, "s%d", i);
-            if (!store->exists(store, file_name)) {
-                sprintf(ext_ptr, "f%d", i);
-                store = cfs_store;
-            }
-            if (store->exists(store, file_name)) {
-                h_set_int(SR(ir)->norms, i,
-                          norm_create(store->open_input(store, file_name), i));
-            }
+    for (i = si->norm_gens_size - 1; i >= 0; i--) {
+        Store *store = (si->use_compound_file && si->norm_gens[i] == 0) ?
+            cfs_store : ir->store;
+        if (si_norm_file_name(si, file_name, i)) {
+            h_set_int(SR(ir)->norms, i,
+                      norm_create(store->open_input(store, file_name), i));
         }
     }
     SR(ir)->norms_dirty = false;
 }
 
-static IndexReader *sr_setup_i(SegmentReader *sr, SegmentInfo *si)
+static IndexReader *sr_setup_i(SegmentReader *sr)
 {
-    Store *store = si->store;
+    Store *store = sr->si->store;
     IndexReader *ir = IR(sr);
     char file_name[SEGMENT_NAME_MAX_LENGTH];
+    char *sr_segment = sr->si->name;
 
     ir->num_docs            = &sr_num_docs;
     ir->max_doc             = &sr_max_doc;
@@ -3549,51 +4242,56 @@ static IndexReader *sr_setup_i(SegmentReader *sr, SegmentInfo *si)
     ir->set_norm_i          = &sr_set_norm_i;
     ir->delete_doc_i        = &sr_delete_doc_i;
     ir->undelete_all_i      = &sr_undelete_all_i;
+    ir->set_deleter_i       = &sr_set_deleter_i;
     ir->commit_i            = &sr_commit_i;
     ir->close_i             = &sr_close_i;
 
-    sr->segment     = si->name;
     sr->cfs_store   = NULL;
 
-    sprintf(file_name, "%s.cfs", sr->segment);
-    if (store->exists(store, file_name)) {
-        sr->cfs_store = open_cmpd_store(store, file_name);
-        store = sr->cfs_store;
-    }
+    TRY
+        if (sr->si->use_compound_file) {
+            sprintf(file_name, "%s.cfs", sr_segment);
+            sr->cfs_store = open_cmpd_store(store, file_name);
+            store = sr->cfs_store;
+        }
 
-    sr->fr = fr_open(store, sr->segment, ir->fis);
-    sr->sfi = sfi_open(store, sr->segment);
-    sr->tir = tir_open(store, sr->sfi, sr->segment);
+        sr->fr = fr_open(store, sr_segment, ir->fis);
+        sr->sfi = sfi_open(store, sr_segment);
+        sr->tir = tir_open(store, sr->sfi, sr_segment);
 
-    sr->deleted_docs = NULL;
-    sr->deleted_docs_dirty = false;
-    sr->undelete_all = false;
-    if (si_has_deletions(si)) {
-        sprintf(file_name, "%s.del", sr->segment);
-        sr->deleted_docs = bv_read(si->store, file_name);
-    }
+        sr->deleted_docs = NULL;
+        sr->deleted_docs_dirty = false;
+        sr->undelete_all = false;
+        if (si_has_deletions(sr->si)) {
+            fn_for_generation(file_name, sr_segment, "del", sr->si->del_gen);
+            sr->deleted_docs = bv_read(sr->si->store, file_name);
+        }
 
-    sprintf(file_name, "%s.frq", sr->segment);
-    sr->frq_in = store->open_input(store, file_name);
-    sprintf(file_name, "%s.prx", sr->segment);
-    sr->prx_in = store->open_input(store, file_name);
-    sr->norms = h_new_int((free_ft)&norm_destroy);
-    sr_open_norms(ir, store);
+        sprintf(file_name, "%s.frq", sr_segment);
+        sr->frq_in = store->open_input(store, file_name);
+        sprintf(file_name, "%s.prx", sr_segment);
+        sr->prx_in = store->open_input(store, file_name);
+        sr->norms = h_new_int((free_ft)&norm_destroy);
+        sr_open_norms(ir, store);
+        if (fis_has_vectors(ir->fis)) {
+            thread_key_create(&sr->thread_fr, NULL);
+            sr->fr_bucket = ary_new();
+        }
+    XCATCHALL
+        ir_close(ir);
+    XENDTRY
 
-    if (fis_has_vectors(ir->fis)) {
-        thread_key_create(&sr->thread_fr, NULL);
-        sr->fr_bucket = ary_new();
-    }
     return ir;
 }
 
 static IndexReader *sr_open(SegmentInfos *sis, FieldInfos *fis, int si_num,
                             bool is_owner)
 {
+    IndexReader *ir;
     SegmentReader *sr = ALLOC_AND_ZERO(SegmentReader);
-    SegmentInfo *si = sis->segs[si_num];
-    IndexReader *ir = ir_setup(IR(sr), si->store, sis, fis, is_owner);
-    return sr_setup_i(SR(ir), si);
+    sr->si = sis->segs[si_num];
+    ir = ir_setup(IR(sr), sr->si->store, sis, fis, is_owner);
+    return sr_setup_i(sr);
 }
 
 /****************************************************************************
@@ -3683,7 +4381,7 @@ static uchar *mr_get_norms(IndexReader *ir, int field_num)
 
     mutex_lock(&ir->mutex);
     bytes = h_get_int(MR(ir)->norms_cache, field_num);
-    if (bytes == NULL) {
+    if (NULL == bytes) {
         int i;
         const int mr_reader_cnt = MR(ir)->r_cnt;
 
@@ -3709,7 +4407,7 @@ static uchar *mr_get_norms_into(IndexReader *ir, int field_num, uchar *buf)
 
     mutex_lock(&ir->mutex);
     bytes = h_get_int(MR(ir)->norms_cache, field_num);
-    if (bytes != NULL) {
+    if (NULL != bytes) {
         memcpy(buf, bytes, MR(ir)->max_doc);
     }
     else {
@@ -3791,6 +4489,7 @@ static void mr_set_norm_i(IndexReader *ir, int doc_num, int field_num, uchar val
     int fnum = mr_get_field_num(MR(ir), i, field_num);
     if (fnum >= 0) {
         IndexReader *reader = MR(ir)->sub_readers[i];
+        ir->has_changes = true;
         h_del_int(MR(ir)->norms_cache, fnum);/* clear cache */
         ir_set_norm_i(reader, doc_num - MR(ir)->starts[i], fnum, val);
     }
@@ -3804,6 +4503,7 @@ static void mr_delete_doc_i(IndexReader *ir, int doc_num)
     /* dispatch to segment reader */
     reader->delete_doc_i(reader, doc_num - MR(ir)->starts[i]);
     MR(ir)->has_deletions = true;
+    ir->has_changes = true;
 }
 
 static void mr_undelete_all_i(IndexReader *ir)
@@ -3817,6 +4517,17 @@ static void mr_undelete_all_i(IndexReader *ir)
         reader->undelete_all_i(reader);
     }
     MR(ir)->has_deletions = false;
+    ir->has_changes = true;
+}
+
+static void mr_set_deleter_i(IndexReader *ir, Deleter *deleter)
+{
+    int i;
+    ir->deleter = deleter;
+    for (i = MR(ir)->r_cnt - 1; i >= 0; i--) {
+        IndexReader *reader = MR(ir)->sub_readers[i];
+        reader->set_deleter_i(reader, deleter);
+    }
 }
 
 static void mr_commit_i(IndexReader *ir)
@@ -3825,7 +4536,7 @@ static void mr_commit_i(IndexReader *ir)
     const int mr_reader_cnt = MR(ir)->r_cnt;
     for (i = 0; i < mr_reader_cnt; i++) {
         IndexReader *reader = MR(ir)->sub_readers[i];
-        ir_commit(reader);
+        ir_commit_i(reader);
     }
 }
 
@@ -3887,6 +4598,7 @@ static IndexReader *mr_new(IndexReader **sub_readers, const int r_cnt)
     ir->set_norm_i          = &mr_set_norm_i;
     ir->delete_doc_i        = &mr_delete_doc_i;
     ir->undelete_all_i      = &mr_undelete_all_i;
+    ir->set_deleter_i       = &mr_set_deleter_i;
     ir->commit_i            = &mr_commit_i;
     ir->close_i             = &mr_close_i;
 
@@ -3980,33 +4692,65 @@ IndexReader *mr_open(IndexReader **sub_readers, const int r_cnt)
  * IndexReader
  ****************************************************************************/
 
+
+static void ir_open_i(Store *store, FindSegmentsFile *fsf)
+{
+    volatile bool success = false;
+    IndexReader *ir = NULL;
+    SegmentInfos *sis = NULL;
+    TRY
+    do {
+        FieldInfos *fis;
+
+        mutex_lock(&store->mutex);
+        sis_read_i(store, fsf);
+        sis = fsf->p_return;
+        fis = sis->fis;
+
+        if (sis->size == 1) {
+            ir = sr_open(sis, fis, 0, true);
+        }
+        else {
+            int i;
+            IndexReader **readers = ALLOC_N(IndexReader *, sis->size);
+            int num_segments = sis->size;
+            for (i = num_segments - 1; i >= 0; i--) {
+                TRY
+                    readers[i] = sr_open(sis, fis, i, false);
+                XCATCHALL
+                    for (i++; i < num_segments; i++) {
+                        ir_close(readers[i]);
+                    }
+                    free(readers);
+                XENDTRY
+            }
+            ir = mr_open_i(store, sis, fis, readers, sis->size);
+        }
+        fsf->p_return = ir;
+        success = true;
+    } while (0);
+    XFINALLY
+        if (!success) {
+            if (ir) {
+                ir_close(ir);
+            }
+            else if (sis) {
+                sis_destroy(sis);
+            }
+        }
+        mutex_unlock(&store->mutex);
+    XENDTRY
+}
+
 /**
  * Will keep a reference to the store. To let this method delete the store
  * make sure you deref the store that you pass to it
  */
 IndexReader *ir_open(Store *store)
 {
-    int i;
-    IndexReader *ir;
-    SegmentInfos *sis;
-    FieldInfos *fis;
-
-    mutex_lock(&store->mutex);
-    sis = sis_read(store);
-    fis = fis_read(store);
-    if (sis->size == 1) {
-        ir = sr_open(sis, fis, 0, true);
-    }
-    else {
-        IndexReader **readers = ALLOC_N(IndexReader *, sis->size);
-        for (i = sis->size; i > 0;) {
-            i--;
-            readers[i] = sr_open(sis, fis, i, false);
-        }
-        ir = mr_open_i(store, sis, fis, readers, sis->size);
-    }
-    mutex_unlock(&store->mutex);
-    return ir;
+    FindSegmentsFile fsf;
+    sis_find_segments_file(store, &fsf, &ir_open_i);
+    return (IndexReader *)fsf.p_return;
 }
 
 /****************************************************************************
@@ -4126,8 +4870,8 @@ typedef struct SkipBuffer
     OutStream *frq_out;
     OutStream *prx_out;
     int last_doc;
-    int last_frq_ptr;
-    int last_prx_ptr;
+    off_t last_frq_ptr;
+    off_t last_prx_ptr;
 } SkipBuffer;
 
 static void skip_buf_reset(SkipBuffer *skip_buf)
@@ -4149,8 +4893,8 @@ static SkipBuffer *skip_buf_new(OutStream *frq_out, OutStream *prx_out)
 
 static void skip_buf_add(SkipBuffer *skip_buf, int doc)
 {
-    int frq_ptr = os_pos(skip_buf->frq_out);
-    int prx_ptr = os_pos(skip_buf->prx_out);
+    off_t frq_ptr = os_pos(skip_buf->frq_out);
+    off_t prx_ptr = os_pos(skip_buf->prx_out);
 
     os_write_vint(skip_buf->buf, doc - skip_buf->last_doc);
     os_write_vint(skip_buf->buf, frq_ptr - skip_buf->last_frq_ptr);
@@ -4161,9 +4905,9 @@ static void skip_buf_add(SkipBuffer *skip_buf, int doc)
     skip_buf->last_prx_ptr = prx_ptr;
 }
 
-static int skip_buf_write(SkipBuffer *skip_buf)
+static off_t skip_buf_write(SkipBuffer *skip_buf)
 {
-  int skip_ptr = os_pos(skip_buf->frq_out);
+  off_t skip_ptr = os_pos(skip_buf->frq_out);
   ramo_write_to(skip_buf->buf, skip_buf->frq_out);
   return skip_ptr;
 }
@@ -4184,7 +4928,8 @@ static void dw_write_norms(DocWriter *dw, FieldInverter *fld_inv)
 {
     char file_name[SEGMENT_NAME_MAX_LENGTH];
     OutStream *norms_out;
-    sprintf(file_name, "%s.f%d", dw->segment, fld_inv->fi->number);
+    si_advance_norm_gen(dw->si, fld_inv->fi->number);
+    si_norm_file_name(dw->si, file_name, fld_inv->fi->number); 
     norms_out = dw->store->new_output(dw->store, file_name);
     os_write_bytes(norms_out, fld_inv->norms, dw->doc_num);
     os_close(norms_out);
@@ -4232,23 +4977,23 @@ static void dw_flush(DocWriter *dw)
     Posting *p;
     Occurence *occ;
     Store *store = dw->store;
-    TermInfosWriter *tiw = tiw_open(store, dw->segment,
+    TermInfosWriter *tiw = tiw_open(store, dw->si->name,
                                     dw->index_interval, skip_interval);
     TermInfo ti;
     char file_name[SEGMENT_NAME_MAX_LENGTH];
     OutStream *frq_out, *prx_out;
     SkipBuffer *skip_buf;
 
-    sprintf(file_name, "%s.frq", dw->segment);
+    sprintf(file_name, "%s.frq", dw->si->name);
     frq_out = store->new_output(store, file_name);
-    sprintf(file_name, "%s.prx", dw->segment);
+    sprintf(file_name, "%s.prx", dw->si->name);
     prx_out = store->new_output(store, file_name);
     skip_buf = skip_buf_new(frq_out, prx_out);
 
     for (i = 0; i < fields_count; i++) {
         fi = fis->fields[i];
         if (!fi_is_indexed(fi)
-            || (fld_inv = h_get_int(dw->fields, fi->number)) == NULL) {
+            || NULL == (fld_inv = h_get_int(dw->fields, fi->number))) {
             continue;
         }
         if (!fi_omit_norms(fi)) {
@@ -4265,9 +5010,9 @@ static void dw_flush(DocWriter *dw)
             last_doc = 0;
             doc_freq = 0;
             skip_buf_reset(skip_buf);
-            for (p = pl->first; p != NULL; p = p->next) {
+            for (p = pl->first; NULL != p; p = p->next) {
                 doc_freq++;
-                if ((doc_freq % dw->skip_interval) == 0) {
+                if (0 == (doc_freq % dw->skip_interval)) {
                     skip_buf_add(skip_buf, last_doc);
                 }
 
@@ -4283,7 +5028,7 @@ static void dw_flush(DocWriter *dw)
                 }
 
                 last_pos = 0;
-                for (occ = p->first_occ; occ != NULL; occ = occ->next) {
+                for (occ = p->first_occ; NULL != occ; occ = occ->next) {
                     os_write_vint(prx_out, occ->pos - last_pos);
                     last_pos = occ->pos;
                 }
@@ -4300,7 +5045,7 @@ static void dw_flush(DocWriter *dw)
     dw_flush_streams(dw);
 } 
 
-DocWriter *dw_open(IndexWriter *iw, const char *segment)
+DocWriter *dw_open(IndexWriter *iw, SegmentInfo *si)
 {
     Store *store = iw->store;
     MemoryPool *mp = mp_new_capa(iw->config.chunk_size,
@@ -4308,34 +5053,34 @@ DocWriter *dw_open(IndexWriter *iw, const char *segment)
 
     DocWriter *dw = ALLOC(DocWriter);
 
-    dw->mp = mp;
-    dw->analyzer = iw->analyzer;
-    dw->fis = iw->fis;
-    dw->store = store;
-    dw->fw = fw_open(store, segment, iw->fis);
-    dw->segment = segment;
+    dw->mp          = mp;
+    dw->analyzer    = iw->analyzer;
+    dw->fis         = iw->fis;
+    dw->store       = store;
+    dw->fw          = fw_open(store, si->name, iw->fis);
+    dw->si          = si;
 
     dw->curr_plists = h_new_str(NULL, NULL);
-    dw->fields = h_new_int((free_ft)fld_inv_destroy);
-    dw->doc_num = 0;
+    dw->fields      = h_new_int((free_ft)fld_inv_destroy);
+    dw->doc_num     = 0;
 
-    dw->index_interval = iw->config.index_interval;
-    dw->skip_interval = iw->config.skip_interval;
-    dw->max_field_length = iw->config.max_field_length;
-    dw->max_buffered_docs = iw->config.max_buffered_docs;
+    dw->index_interval      = iw->config.index_interval;
+    dw->skip_interval       = iw->config.skip_interval;
+    dw->max_field_length    = iw->config.max_field_length;
+    dw->max_buffered_docs   = iw->config.max_buffered_docs;
     
-    dw->offsets = ALLOC_AND_ZERO_N(Offset, DW_OFFSET_INIT_CAPA);
-    dw->offsets_size = 0;
-    dw->offsets_capa = DW_OFFSET_INIT_CAPA;
+    dw->offsets             = ALLOC_AND_ZERO_N(Offset, DW_OFFSET_INIT_CAPA);
+    dw->offsets_size        = 0;
+    dw->offsets_capa        = DW_OFFSET_INIT_CAPA;
 
-    dw->similarity = iw->similarity;
+    dw->similarity          = iw->similarity;
     return dw;
 }
 
-void dw_new_segment(DocWriter *dw, char *segment)
+void dw_new_segment(DocWriter *dw, SegmentInfo *si)
 {
-    dw->fw = fw_open(dw->store, segment, dw->fis);
-    dw->segment = segment;
+    dw->fw = fw_open(dw->store, si->name, dw->fis);
+    dw->si = si;
 }
 
 void dw_close(DocWriter *dw)
@@ -4536,7 +5281,7 @@ typedef struct SegmentMergeInfo {
     int base;
     int max_doc;
     int doc_cnt;
-    char *segment;
+    SegmentInfo *si;
     Store *store;
     Store *orig_store;
     BitVector *deleted_docs;
@@ -4552,7 +5297,7 @@ typedef struct SegmentMergeInfo {
 static bool smi_lt(const SegmentMergeInfo *smi1, const SegmentMergeInfo *smi2)
 {
     int cmpres = strcmp(smi1->term, smi2->term);
-    if (cmpres == 0) {
+    if (0 == cmpres) {
         return smi1->base < smi2->base;
     }
     else {
@@ -4578,12 +5323,13 @@ static void smi_load_doc_map(SegmentMergeInfo *smi)
     smi->doc_cnt = j;
 }
 
-static SegmentMergeInfo *smi_new(int base, Store *store, char *segment)
+static SegmentMergeInfo *smi_new(int base, Store *store, SegmentInfo *si)
 {
     SegmentMergeInfo *smi = ALLOC_AND_ZERO(SegmentMergeInfo);
     char file_name[SEGMENT_NAME_MAX_LENGTH];
+    char *segment = si->name;
     smi->base = base;
-    smi->segment = segment;
+    smi->si = si;
     smi->orig_store = smi->store = store;
     sprintf(file_name, "%s.cfs", segment);
     if (store->exists(store, file_name)) {
@@ -4595,8 +5341,8 @@ static SegmentMergeInfo *smi_new(int base, Store *store, char *segment)
     smi->doc_cnt = smi->max_doc
         = smi->store->length(smi->store, file_name) / FIELDS_IDX_PTR_SIZE;
 
-    sprintf(file_name, "%s.del", segment);
-    if (store->exists(store, file_name)) {
+    if (si->del_gen >= 0) {
+        fn_for_generation(file_name, segment, "del", si->del_gen);
         smi->deleted_docs = bv_read(store, file_name);
         smi_load_doc_map(smi);
     }
@@ -4606,13 +5352,14 @@ static SegmentMergeInfo *smi_new(int base, Store *store, char *segment)
 static void smi_load_term_input(SegmentMergeInfo *smi)
 {
     Store *store = smi->store;
+    char *segment = smi->si->name;
     char file_name[SEGMENT_NAME_MAX_LENGTH];
-    smi->sfi = sfi_open(store, smi->segment);
-    sprintf(file_name, "%s.tis", smi->segment);
+    smi->sfi = sfi_open(store, segment);
+    sprintf(file_name, "%s.tis", segment);
     smi->te = TE(ste_new(store->open_input(store, file_name), smi->sfi));
-    sprintf(file_name, "%s.frq", smi->segment);
+    sprintf(file_name, "%s.frq", segment);
     smi->frq_in = store->open_input(store, file_name);
-    sprintf(file_name, "%s.prx", smi->segment);
+    sprintf(file_name, "%s.prx", segment);
     smi->prx_in = store->open_input(store, file_name);
     smi->tde = stpe_new(NULL, smi->frq_in, smi->prx_in, smi->deleted_docs,
                         STE(smi->te)->skip_interval);
@@ -4652,7 +5399,7 @@ typedef struct SegmentMerger {
     TermInfo ti;
     Store *store;
     FieldInfos *fis;
-    char *segment;
+    SegmentInfo *si;
     SegmentMergeInfo **smis;
     int seg_cnt;
     int doc_cnt;
@@ -4667,19 +5414,19 @@ typedef struct SegmentMerger {
     OutStream *prx_out;
 } SegmentMerger;
 
-static SegmentMerger *sm_create(IndexWriter *iw, char *segment,
+static SegmentMerger *sm_create(IndexWriter *iw, SegmentInfo *si,
                                 SegmentInfo **seg_infos, const int seg_cnt)
 {
     int i;
     SegmentMerger *sm = ALLOC_AND_ZERO_N(SegmentMerger, seg_cnt);
     sm->store = iw->store;
     sm->fis = iw->fis;
-    sm->segment = estrdup(segment);
+    sm->si = si;
     sm->doc_cnt = 0;
     sm->smis = ALLOC_N(SegmentMergeInfo *, seg_cnt);
     for (i = 0; i < seg_cnt; i++) {
         sm->smis[i] = smi_new(sm->doc_cnt, seg_infos[i]->store,
-                              seg_infos[i]->name);
+                              seg_infos[i]);
         sm->doc_cnt += sm->smis[i]->doc_cnt;
     }
     sm->seg_cnt = seg_cnt;
@@ -4695,7 +5442,6 @@ static void sm_destroy(SegmentMerger *sm)
         smi_destroy(sm->smis[i]);
     }
     free(sm->smis);
-    free(sm->segment);
     free(sm);
 }
 
@@ -4708,20 +5454,21 @@ static void sm_merge_fields(SegmentMerger *sm)
     Store *store = sm->store;
     const int seg_cnt = sm->seg_cnt;
 
-    sprintf(file_name, "%s.fdt", sm->segment);
+    sprintf(file_name, "%s.fdt", sm->si->name);
     fdt_out = store->new_output(store, file_name);
 
-    sprintf(file_name, "%s.fdx", sm->segment);
+    sprintf(file_name, "%s.fdx", sm->si->name);
     fdx_out = store->new_output(store, file_name);
 
     for (i = 0; i < seg_cnt; i++) {
         SegmentMergeInfo *smi = sm->smis[i];
         const int max_doc = smi->max_doc;
         InStream *fdt_in, *fdx_in;
+        char *segment = smi->si->name;
         store = smi->store;
-        sprintf(file_name, "%s.fdt", smi->segment);
+        sprintf(file_name, "%s.fdt", segment);
         fdt_in = store->open_input(store, file_name);
-        sprintf(file_name, "%s.fdx", smi->segment);
+        sprintf(file_name, "%s.fdx", segment);
         fdx_in = store->open_input(store, file_name);
 
         if (max_doc > 0) {
@@ -4775,7 +5522,7 @@ static int sm_append_postings(SegmentMerger *sm, SegmentMergeInfo **matches,
          * stde_next rather than stpe_next here */
         while (stde_next(tde)) {
             doc = stde_doc_num(tde);
-            if (doc_map != NULL) {
+            if (NULL != doc_map) {
                 doc = doc_map[doc]; /* work around deletions */
             }
             doc += base;          /* convert to merged space */
@@ -4787,7 +5534,7 @@ static int sm_append_postings(SegmentMerger *sm, SegmentMergeInfo **matches,
 #endif
             df++;
 
-            if ((df % skip_interval) == 0) {
+            if (0 == (df % skip_interval)) {
                 skip_buf_add(skip_buf, last_doc);
             }
 
@@ -4823,12 +5570,12 @@ static char *sm_cache_term(SegmentMerger *sm, char *term, int term_len)
 static void sm_merge_term_info(SegmentMerger *sm, SegmentMergeInfo **matches,
                                int match_size)
 {
-    int frq_ptr = os_pos(sm->frq_out);
-    int prx_ptr = os_pos(sm->prx_out);
+    off_t frq_ptr = os_pos(sm->frq_out);
+    off_t prx_ptr = os_pos(sm->prx_out);
 
     int df = sm_append_postings(sm, matches, match_size); /* append posting data */
 
-    int skip_ptr = skip_buf_write(sm->skip_buf);
+    off_t skip_ptr = skip_buf_write(sm->skip_buf);
 
     if (df > 0) {
         /* add an entry to the dictionary with ptrs to prox and freq files */
@@ -4861,7 +5608,7 @@ static void sm_merge_term_infos(SegmentMerger *sm)
         for (j = 0; j < seg_cnt; j++) {
             smi = sm->smis[j];
             ste_set_field(smi->te, i);
-            if (smi_next(smi) != NULL) {
+            if (NULL != smi_next(smi)) {
                 pq_push(sm->queue, smi); /* initialize @queue */
             }
         }
@@ -4877,7 +5624,7 @@ static void sm_merge_term_infos(SegmentMerger *sm)
             match_size++;
             term = matches[0]->term;
             top = pq_top(sm->queue);
-            while ((top != NULL) && (strcmp(term, top->term) == 0)) {
+            while ((NULL != top) && (0 == strcmp(term, top->term))) {
                 matches[match_size] = pq_pop(sm->queue);
                 match_size++;
                 top = pq_top(sm->queue);
@@ -4889,7 +5636,7 @@ static void sm_merge_term_infos(SegmentMerger *sm)
             while (match_size > 0) {
                 match_size--;
                 smi = matches[match_size];
-                if (smi_next(smi) != NULL) {
+                if (NULL != smi_next(smi)) {
                     pq_push(sm->queue, smi);   /* restore queue */
                 }
             }
@@ -4905,12 +5652,12 @@ static void sm_merge_terms(SegmentMerger *sm)
 {
     char file_name[SEGMENT_NAME_MAX_LENGTH];
 
-    sprintf(file_name, "%s.frq", sm->segment);
+    sprintf(file_name, "%s.frq", sm->si->name);
     sm->frq_out = sm->store->new_output(sm->store, file_name);
-    sprintf(file_name, "%s.prx", sm->segment);
+    sprintf(file_name, "%s.prx", sm->si->name);
     sm->prx_out = sm->store->new_output(sm->store, file_name);
 
-    sm->tiw = tiw_open(sm->store, sm->segment, sm->config->index_interval,
+    sm->tiw = tiw_open(sm->store, sm->si->name, sm->config->index_interval,
                        sm->config->skip_interval);
     sm->skip_buf = skip_buf_new(sm->frq_out, sm->prx_out);
 
@@ -4936,6 +5683,7 @@ static void sm_merge_terms(SegmentMerger *sm)
 
 static void sm_merge_norms(SegmentMerger *sm)
 {
+    SegmentInfo *si;
     int i, j, k;
     Store *store;
     uchar byte;
@@ -4945,23 +5693,21 @@ static void sm_merge_norms(SegmentMerger *sm)
     char file_name[SEGMENT_NAME_MAX_LENGTH];
     SegmentMergeInfo *smi;
     const int seg_cnt = sm->seg_cnt;
-    const int fis_size = sm->fis->size;
-    for (i = 0; i < fis_size; i++) {
+    for (i = sm->fis->size - 1; i >= 0; i--) {
         fi = sm->fis->fields[i];
         if (fi_has_norms(fi))  {
-            sprintf(file_name, "%s.f%d", sm->segment, i);
+            si = sm->si;
+            si_advance_norm_gen(si, i);
+            si_norm_file_name(si, file_name, i);
             os = sm->store->new_output(sm->store, file_name);
             for (j = 0; j < seg_cnt; j++) {
                 smi = sm->smis[j];
-                store = smi->orig_store;
-                sprintf(file_name, "%s.s%d", smi->segment, i);
-                if (!store->exists(store, file_name)) {
-                    sprintf(file_name, "%s.f%d", smi->segment, i);
-                    store = smi->store;
-                }
-                if (store->exists(store, file_name)) {
+                si = smi->si;
+                if (si_norm_file_name(si, file_name, i)) {
                     const int max_doc = smi->max_doc;
                     BitVector *deleted_docs =  smi->deleted_docs;
+                    store = (si->use_compound_file && si->norm_gens[i])
+                             ? smi->orig_store : smi->store;
                     is = store->open_input(store, file_name);
                     if (deleted_docs) {
                         for (k = 0; k < max_doc; k++) {
@@ -5004,11 +5750,18 @@ static int sm_merge(SegmentMerger *sm)
 /* prepare an index ready for writing */
 void index_create(Store *store, FieldInfos *fis)
 {
-    SegmentInfos *sis = sis_new();
+    SegmentInfos *sis = sis_new(fis);
     store->clear_all(store);
-    sis_write(sis, store);
+    sis_write(sis, store, NULL);
     sis_destroy(sis);
-    fis_write(fis, store);
+}
+
+bool index_is_locked(Store *store)
+{
+    Lock *write_lock = open_lock(store, WRITE_LOCK_NAME);
+    bool is_locked = write_lock->is_locked(write_lock);
+    close_lock(write_lock);
+    return is_locked;
 }
 
 int iw_doc_count(IndexWriter *iw)
@@ -5025,158 +5778,79 @@ int iw_doc_count(IndexWriter *iw)
     return doc_cnt;
 }
 
-static void delete_files(char **file_names, Store *store)
+#define MOVE_TO_COMPOUND_DIR(file_name)\
+    deleter_queue_file(dlr, file_name);\
+    cw_add_file(cw, file_name)
+    
+static void iw_create_compound_file(Store *store, FieldInfos *fis,
+                                    SegmentInfo *si, char *cfs_file_name,
+                                    Deleter *dlr)
 {
     int i;
-    for (i = ary_size(file_names) - 1; i >= 0; i--) {
-        store->remove(store, file_names[i]);
-    }
-    ary_destroy((void **)file_names, &free);
-}
-
-static char **iw_create_compound_file(Store *store, FieldInfos *fis,
-                                      char *segment, char *cfs_file_name)
-{
-    char **file_names = (char **)ary_new_capa(16);
     CompoundWriter *cw;
-    FieldInfo *fi;
-    int i;
     char file_name[SEGMENT_NAME_MAX_LENGTH];
-    const int fis_size = fis->size;
-    int file_names_size;
+    char *ext;
+    int seg_len = strlen(si->name);
+
+    memcpy(file_name, si->name, seg_len);
+    file_name[seg_len] = '.';
+    ext = file_name + seg_len + 1;
 
     cw = open_cw(store, cfs_file_name);
     for (i = 0; i < NELEMS(COMPOUND_EXTENSIONS); i++) {
-        sprintf(file_name, "%s.%s",
-                segment, COMPOUND_EXTENSIONS[i]);
-        ary_push(file_names, estrdup(file_name));
+        memcpy(ext, COMPOUND_EXTENSIONS[i], 4);
+        MOVE_TO_COMPOUND_DIR(file_name);
     }
 
     /* Field norm file_names */
-    for (i = 0; i < fis_size; i++) {
-        fi = fis->fields[i];
-        if (fi_has_norms(fi)) {
-            sprintf(file_name, "%s.f%d", segment, i);
-            if (!store->exists(store, file_name)) {
-                continue;
-            }
-            ary_push(file_names, estrdup(file_name));
+    for (i = fis->size - 1; i >= 0; i--) {
+        if (fi_has_norms(fis->fields[i])
+            && si_norm_file_name(si, file_name, i)) {
+            MOVE_TO_COMPOUND_DIR(file_name);
         }
     }
 
-    /* Now merge all added file_names */
-    file_names_size = ary_size(file_names);
-    for (i = 0; i < file_names_size; i++) {
-        cw_add_file(cw, file_names[i]);
-    }
-
     /* Perform the merge */
     cw_close(cw);
-
-    return file_names;
 }
 
-static void iw_commit_compound_file(IndexWriter *iw, char *segment,
-                                    Lock *commit_lock)
+static void iw_commit_compound_file(IndexWriter *iw, SegmentInfo *si)
 {
-    char tmp_name[SEGMENT_NAME_MAX_LENGTH];
     char cfs_name[SEGMENT_NAME_MAX_LENGTH];
-    char **files_to_delete;
-    sprintf(tmp_name, "%s.tmp", segment);
-    sprintf(cfs_name, "%s.cfs", segment);
-
-    files_to_delete =
-        iw_create_compound_file(iw->store, iw->fis, segment, tmp_name);
-    if (!commit_lock->obtain(commit_lock)) {
-        RAISE(LOCK_ERROR,
-              "Couldn't obtain commit lock to write compound file");
-    }
+    sprintf(cfs_name, "%s.cfs", si->name);
 
-    delete_files(files_to_delete, iw->store);
-    iw->store->rename(iw->store, tmp_name, cfs_name);
-
-    commit_lock->release(commit_lock);
-}
-
-#define ADD_IF_EXISTS_FMT(fmt, ext) do {\
-    sprintf(file_name, fmt, segment, ext);\
-    if (store->exists(store, file_name)) {\
-        ary_push(file_names, estrdup(file_name));\
-    }\
-} while (0)
-
-#define ADD_IF_EXISTS(ext) ADD_IF_EXISTS_FMT("%s.%s", ext)
-
-static char **iw_seg_file_names(FieldInfos *fis, Store *store, char *segment)
-{
-    char **file_names = (char **)ary_new_capa(16);
-    int i;
-    char file_name[SEGMENT_NAME_MAX_LENGTH];
-    const int fis_size = fis->size;
-
-
-    sprintf(file_name, "%s.cfs", segment);
-    if (store->exists(store, file_name)) {
-        ary_push(file_names, estrdup(file_name));
-        ADD_IF_EXISTS("del");
-        for (i = 0; i < fis_size; i++) {
-            if (fi_has_norms(fis->fields[i])) {
-                ADD_IF_EXISTS_FMT("%s.s%d", i);
-            }
-        }
-    }
-    else {
-        for (i = 0; i < NELEMS(INDEX_EXTENSIONS); i++) {
-            ADD_IF_EXISTS(INDEX_EXTENSIONS[i]);
-        }
-        for (i = 0; i < fis_size; i++) {
-            if (fi_has_norms(fis->fields[i])) {
-                ADD_IF_EXISTS_FMT("%s.f%d", i);
-            }
-        }
-    }
-    return file_names;
+    iw_create_compound_file(iw->store, iw->fis, si, cfs_name, iw->deleter);
+    deleter_commit_pending_deletions(iw->deleter);
 }
 
 static void iw_merge_segments(IndexWriter *iw, const int min_seg,
                               const int max_seg)
 {
     int i;
-    Lock *commit_lock;
     SegmentInfos *sis = iw->sis;
     SegmentInfo *si = sis_new_segment(sis, 0, iw->store);
 
-    SegmentMerger *merger = sm_create(iw, si->name, &sis->segs[min_seg],
+    SegmentMerger *merger = sm_create(iw, si, &sis->segs[min_seg],
                                       max_seg - min_seg);
 
     /* This is where all the action happens. */
     si->doc_cnt = sm_merge(merger);
 
     mutex_lock(&iw->store->mutex);
-    commit_lock = open_lock(iw->store, COMMIT_LOCK_NAME);
-
-    /* *** OBTAIN COMMIT LOCK *** */
-    if (!commit_lock->obtain(commit_lock)) {
-        RAISE(LOCK_ERROR, "Couldn't obtain commit lock to commit merged segment "
-              "%s", si->name);
-    }
     /* delete merged segments */
     for (i = min_seg; i < max_seg; i++) {
-        delete_files(
-            iw_seg_file_names(iw->fis, sis->segs[i]->store, sis->segs[i]->name),
-            iw->store);
+        si_delete_files(sis->segs[i], iw->fis, iw->deleter);
     }
+    deleter_commit_pending_deletions(iw->deleter);
+
     sis_del_from_to(sis, min_seg, max_seg);
-    /* commit the segments file */
-    sis_write(sis, iw->store);
-    commit_lock->release(commit_lock);
-    /* RELEASE COMMIT LOCK */
 
     if (iw->config.use_compound_file) {
-        iw_commit_compound_file(iw, si->name, commit_lock);
+        iw_commit_compound_file(iw, si);
+        si->use_compound_file = true;
     }
 
-    close_lock(commit_lock);
+    sis_write(sis, iw->store, iw->deleter);
 
     mutex_unlock(&iw->store->mutex);
 
@@ -5223,28 +5897,20 @@ static void iw_flush_ram_segment(IndexWriter *iw)
 {
     SegmentInfos *sis = iw->sis;
     SegmentInfo *si;
-    Lock *commit_lock;
 
     si = sis->segs[sis->size - 1];
     si->doc_cnt = iw->dw->doc_num;
     dw_flush(iw->dw);
 
     mutex_lock(&iw->store->mutex);
-    commit_lock = open_lock(iw->store, COMMIT_LOCK_NAME);
 
-    if (!commit_lock->obtain(commit_lock)) {
-        RAISE(LOCK_ERROR, "Couldn't obtain commit lock to write segments file");
+    if (iw->config.use_compound_file) {
+        iw_commit_compound_file(iw, si);
+        si->use_compound_file = true;
     }
     /* commit the segments file and the fields file */
-    fis_write(iw->fis, iw->store);
-    sis_write(iw->sis, iw->store);
-    commit_lock->release(commit_lock);
+    sis_write(iw->sis, iw->store, iw->deleter);
 
-
-    if (iw->config.use_compound_file) {
-        iw_commit_compound_file(iw, si->name, commit_lock);
-    }
-    close_lock(commit_lock);
     mutex_unlock(&iw->store->mutex);
 
     iw_maybe_merge_segments(iw);
@@ -5253,11 +5919,11 @@ static void iw_flush_ram_segment(IndexWriter *iw)
 void iw_add_doc(IndexWriter *iw, Document *doc)
 {
     mutex_lock(&iw->mutex);
-    if (!iw->dw) {
-        iw->dw = dw_open(iw, sis_new_segment(iw->sis, 0, iw->store)->name);
+    if (NULL == iw->dw) {
+        iw->dw = dw_open(iw, sis_new_segment(iw->sis, 0, iw->store));
     }
-    else if (iw->dw->fw == NULL) {
-        dw_new_segment(iw->dw, sis_new_segment(iw->sis, 0, iw->store)->name);
+    else if (NULL == iw->dw->fw) {
+        dw_new_segment(iw->dw, sis_new_segment(iw->sis, 0, iw->store));
     }
     dw_add_doc(iw->dw, doc);
     if (mp_used(iw->dw->mp) > iw->config.max_buffer_memory
@@ -5291,17 +5957,25 @@ void iw_delete_term(IndexWriter *iw, const char *field, const char *term)
         do {
             SegmentInfos *sis = iw->sis;
             const int seg_cnt = sis->size;
+            bool did_delete = false;
             for (i = 0; i < seg_cnt; i++) {
                 IndexReader *ir = sr_open(sis, iw->fis, i, false);
                 TermDocEnum *tde = ir->term_docs(ir);
+                ir->deleter = iw->deleter;
                 stde_seek(tde, field_num, term);
                 while (tde->next(tde)) {
+                    did_delete = true;
                     sr_delete_doc_i(ir, STDE(tde)->doc_num);
                 }
                 tde_destroy(tde);
                 sr_commit_i(ir);
                 ir_close(ir);
             }
+            if (did_delete) {
+                mutex_lock(&iw->store->mutex);
+                sis_write(iw->sis, iw->store, iw->deleter);
+                mutex_unlock(&iw->store->mutex);
+            }
         } while (0);
         mutex_unlock(&iw->mutex);
     }
@@ -5316,7 +5990,7 @@ static void iw_optimize_i(IndexWriter *iw)
                && (si_has_deletions(iw->sis->segs[0])
                    || (iw->sis->segs[0]->store != iw->store)
                    || (iw->config.use_compound_file
-                       && (!si_uses_compound_file(iw->sis->segs[0])
+                       && (!iw->sis->segs[0]->use_compound_file
                            || si_has_separate_norms(iw->sis->segs[0])))))) {
         min_segment = iw->sis->size - iw->config.merge_factor;
         iw_merge_segments_from(iw, min_segment < 0 ? 0 : min_segment);
@@ -5344,7 +6018,9 @@ void iw_close(IndexWriter *iw)
 
     iw->write_lock->release(iw->write_lock);
     close_lock(iw->write_lock);
+    iw->write_lock = NULL;
     store_deref(iw->store);
+    deleter_destroy(iw->deleter);
 
     mutex_destroy(&iw->mutex);
     free(iw);
@@ -5368,16 +6044,17 @@ IndexWriter *iw_open(Store *store, volatile Analyzer *analyzer,
                   "Couldn't obtain write lock when opening IndexWriter");
         }
 
-
         iw->sis = sis_read(store);
-        iw->fis = fis_read(store);
+        iw->fis = iw->sis->fis;
+        REF(iw->fis);
     XCATCHALL
         if (iw->write_lock) {
             iw->write_lock->release(iw->write_lock);
             close_lock(iw->write_lock);
+            iw->write_lock = NULL;
         }
         if (iw->sis) sis_destroy(iw->sis);
-        if (iw->fis) fis_deref(iw->fis);
+        if (analyzer) a_deref((Analyzer *)analyzer);
         free(iw);
     XENDTRY
 
@@ -5385,6 +6062,9 @@ IndexWriter *iw_open(Store *store, volatile Analyzer *analyzer,
     iw->analyzer = analyzer ? (Analyzer *)analyzer
                             : mb_standard_analyzer_new(true);
 
+    iw->deleter = deleter_new(iw->sis, store);
+    deleter_delete_deletable_files(iw->deleter);
+
     REF(store);
     return iw;
 }
@@ -5400,18 +6080,19 @@ static void iw_cp_fields(IndexWriter *iw, SegmentReader *sr,
     InStream *fdt_in, *fdx_in;
     Store *store_in = sr->cfs_store ? sr->cfs_store : sr->ir.store;
     Store *store_out = iw->store;
+    char *sr_segment = sr->si->name;
 
     sprintf(file_name, "%s.fdt", segment);
     fdt_out = store_out->new_output(store_out, file_name);
     sprintf(file_name, "%s.fdx", segment);
     fdx_out = store_out->new_output(store_out, file_name);
 
-    sprintf(file_name, "%s.fdt", sr->segment);
+    sprintf(file_name, "%s.fdt", sr_segment);
     fdt_in = store_in->open_input(store_in, file_name);
-    sprintf(file_name, "%s.fdx", sr->segment);
+    sprintf(file_name, "%s.fdx", sr_segment);
     fdx_in = store_in->open_input(store_in, file_name);
 
-    sprintf(file_name, "%s.del", sr->segment);
+    sprintf(file_name, "%s.del", sr_segment);
     if (store_in->exists(store_in, file_name)) {
         OutStream *del_out;
         InStream *del_in = store_in->open_input(store_in, file_name);
@@ -5487,30 +6168,31 @@ static void iw_cp_terms(IndexWriter *iw, SegmentReader *sr,
     InStream *tix_in, *tis_in, *tfx_in, *frq_in, *prx_in;
     Store *store_out = iw->store;
     Store *store_in = sr->cfs_store ? sr->cfs_store : sr->ir.store;
+    char *sr_segment = sr->si->name;
 
     sprintf(file_name, "%s.tix", segment);
     tix_out = store_out->new_output(store_out, file_name);
-    sprintf(file_name, "%s.tix", sr->segment);
+    sprintf(file_name, "%s.tix", sr_segment);
     tix_in = store_in->open_input(store_in, file_name);
     
     sprintf(file_name, "%s.tis", segment);
     tis_out = store_out->new_output(store_out, file_name);
-    sprintf(file_name, "%s.tis", sr->segment);
+    sprintf(file_name, "%s.tis", sr_segment);
     tis_in = store_in->open_input(store_in, file_name);
 
     sprintf(file_name, "%s.tfx", segment);
     tfx_out = store_out->new_output(store_out, file_name);
-    sprintf(file_name, "%s.tfx", sr->segment);
+    sprintf(file_name, "%s.tfx", sr_segment);
     tfx_in = store_in->open_input(store_in, file_name);
 
     sprintf(file_name, "%s.frq", segment);
     frq_out = store_out->new_output(store_out, file_name);
-    sprintf(file_name, "%s.frq", sr->segment);
+    sprintf(file_name, "%s.frq", sr_segment);
     frq_in = store_in->open_input(store_in, file_name);
 
     sprintf(file_name, "%s.prx", segment);
     prx_out = store_out->new_output(store_out, file_name);
-    sprintf(file_name, "%s.prx", sr->segment);
+    sprintf(file_name, "%s.prx", sr_segment);
     prx_in = store_in->open_input(store_in, file_name);
 
     if (map) {
@@ -5548,47 +6230,38 @@ static void iw_cp_terms(IndexWriter *iw, SegmentReader *sr,
 }
 
 static void iw_cp_norms(IndexWriter *iw, SegmentReader *sr,
-                        const char *segment, int *map)
+                        SegmentInfo *si, int *map)
 {
     int i;
     FieldInfos *fis = IR(sr)->fis;
     const int field_cnt = fis->size;
     InStream *norms_in;
     OutStream *norms_out;
-    Store *store_in = sr->ir.store;
-    Store *cfs_store_in = sr->cfs_store;
     Store *store_out = iw->store;
     char file_name_in[SEGMENT_NAME_MAX_LENGTH];
-    char *ext_ptr_in;
     char file_name_out[SEGMENT_NAME_MAX_LENGTH];
-    char *ext_ptr_out;
-    sprintf(file_name_in, "%s.", sr->segment);
-    ext_ptr_in = file_name_in + strlen(file_name_in);
-    sprintf(file_name_out, "%s.", segment);
-    ext_ptr_out = file_name_out + strlen(file_name_out);
 
     for (i = 0; i < field_cnt; i++) {
-        if (fi_has_norms(fis->fields[i])) {
-            Store *store = store_in;
-            sprintf(ext_ptr_in, "s%d", i);
-            if (!store->exists(store, file_name_in)) {
-                sprintf(ext_ptr_in, "f%d", i);
-                store = cfs_store_in;
-            }
-            if (store->exists(store, file_name_in)) {
-                norms_in = store->open_input(store, file_name_in);
-                sprintf(ext_ptr_out, "f%d", map ? map[i] : i);
-                norms_out = store_out->new_output(store_out, file_name_out);
-                is2os_copy_bytes(norms_in, norms_out, is_length(norms_in));
-                os_close(norms_out);
-                is_close(norms_in);
-            }
+        if (fi_has_norms(fis->fields[i])
+            && si_norm_file_name(sr->si, file_name_in, i)) {
+            Store *store = (sr->si->use_compound_file
+                            && sr->si->norm_gens[i] == 0) ? sr->cfs_store
+                                                          : IR(sr)->store;
+            int field_num = map ? map[i] : i;
+
+            norms_in = store->open_input(store, file_name_in);
+            si_advance_norm_gen(si, field_num);
+            si_norm_file_name(si, file_name_out, field_num);
+            norms_out = store_out->new_output(store_out, file_name_out);
+            is2os_copy_bytes(norms_in, norms_out, is_length(norms_in));
+            os_close(norms_out);
+            is_close(norms_in);
         }
     }
 }
 
 static void iw_cp_map_files(IndexWriter *iw, SegmentReader *sr,
-                            const char *segment)
+                            SegmentInfo *si)
 {
     int i;
     FieldInfos *from_fis = IR(sr)->fis;
@@ -5600,19 +6273,19 @@ static void iw_cp_map_files(IndexWriter *iw, SegmentReader *sr,
         field_map[i] = fis_get_field_num(to_fis, from_fis->fields[i]->name);
     }
 
-    iw_cp_fields(iw, sr, segment, field_map);
-    iw_cp_terms(iw, sr, segment, field_map);
-    iw_cp_norms(iw, sr, segment, field_map);
+    iw_cp_fields(iw, sr, si->name, field_map);
+    iw_cp_terms( iw, sr, si->name, field_map);
+    iw_cp_norms( iw, sr, si,       field_map);
 
     free(field_map);
 }
 
 static void iw_cp_files(IndexWriter *iw, SegmentReader *sr,
-                        const char *segment)
+                        SegmentInfo *si)
 {
-    iw_cp_fields(iw, sr, segment, NULL);
-    iw_cp_terms(iw, sr, segment, NULL);
-    iw_cp_norms(iw, sr, segment, NULL);
+    iw_cp_fields(iw, sr, si->name, NULL);
+    iw_cp_terms( iw, sr, si->name, NULL);
+    iw_cp_norms( iw, sr, si,       NULL);
 }
 
 static void iw_add_segment(IndexWriter *iw, SegmentReader *sr)
@@ -5641,10 +6314,10 @@ static void iw_add_segment(IndexWriter *iw, SegmentReader *sr)
     }
 
     if (must_map_fields) {
-        iw_cp_map_files(iw, sr, si->name);
+        iw_cp_map_files(iw, sr, si);
     }
     else {
-        iw_cp_files(iw, sr, si->name);
+        iw_cp_files(iw, sr, si);
     }
 }
 
@@ -5666,8 +6339,6 @@ static void iw_add_segments(IndexWriter *iw, IndexReader *ir)
 void iw_add_readers(IndexWriter *iw, IndexReader **readers, const int r_cnt)
 {
     int i;
-    Lock *commit_lock;
-
     mutex_lock(&iw->mutex);
     iw_optimize_i(iw);
 
@@ -5676,16 +6347,9 @@ void iw_add_readers(IndexWriter *iw, IndexReader **readers, const int r_cnt)
     }
 
     mutex_lock(&iw->store->mutex);
-    commit_lock = open_lock(iw->store, COMMIT_LOCK_NAME);
 
-    if (!commit_lock->obtain(commit_lock)) {
-        RAISE(LOCK_ERROR, "Couldn't obtain commit lock to write segments file");
-    }
     /* commit the segments file and the fields file */
-    fis_write(iw->fis, iw->store);
-    sis_write(iw->sis, iw->store);
-    commit_lock->release(commit_lock);
-    close_lock(commit_lock);
+    sis_write(iw->sis, iw->store, iw->deleter);
     mutex_unlock(&iw->store->mutex);
 
     iw_optimize_i(iw);