whistlepig 0.9.1 → 0.10

data/ext/whistlepig/lock.h

@@ -0,0 +1,18 @@
+#ifndef wp_lock_h_
+#define wp_lock_h_
+
+// whistlepig locks
+// (c) 2011 william morgan. see copying for license terms.
+
+#include <pthread.h>
+
+#include "error.h"
+
+#define WP_LOCK_READLOCK 0
+#define WP_LOCK_WRITELOCK 1
+
+wp_error* wp_lock_setup(pthread_rwlock_t* lock) RAISES_ERROR;
+wp_error* wp_lock_grab(pthread_rwlock_t* lock, int lock_type) RAISES_ERROR;
+wp_error* wp_lock_release(pthread_rwlock_t* lock) RAISES_ERROR;
+
+#endif

data/ext/whistlepig/mmap-obj.c

@@ -18,10 +18,10 @@ wp_error* mmap_obj_create(mmap_obj* o, const char* magic, const char* pathname,
   lseek(o->fd, size - 1, SEEK_SET);
   ssize_t num_bytes = write(o->fd, "", 1);
   if(num_bytes == -1) RAISE_SYSERROR("write");
-  o->header = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
-  if(o->header == MAP_FAILED) RAISE_SYSERROR("mmap");
-  strncpy(o->header->magic, magic, MMAP_OBJ_MAGIC_SIZE);
-  o->header->size = initial_size;
+  o->content = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
+  if(o->content == MAP_FAILED) RAISE_SYSERROR("mmap");
+  strncpy(o->content->magic, magic, MMAP_OBJ_MAGIC_SIZE);
+  o->content->size = o->loaded_size = initial_size;
   DEBUG("created new %s object with %u bytes", magic, size);
 
   return NO_ERROR;
@@ -33,44 +33,60 @@ wp_error* mmap_obj_load(mmap_obj* o, const char* magic, const char* pathname) {
   if(o->fd == -1) RAISE_SYSERROR("cannot open %s", pathname);
 
   // load header
-  o->header = mmap(NULL, sizeof(mmap_obj_header), PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
-  if(o->header == MAP_FAILED) RAISE_SYSERROR("header mmap");
+  o->content = mmap(NULL, sizeof(mmap_obj_header), PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
+  if(o->content == MAP_FAILED) RAISE_SYSERROR("header mmap");
   DEBUG("loaded header of %u bytes for %s object", sizeof(mmap_obj_header), magic);
 
-  RELAY_ERROR(validate(o->header, magic));
+  RELAY_ERROR(validate(o->content, magic));
 
-  uint32_t size = o->header->size + (uint32_t)sizeof(mmap_obj_header);
+  o->loaded_size = o->content->size;
+
+  uint32_t size = o->content->size + (uint32_t)sizeof(mmap_obj_header);
   DEBUG("full size is %u bytes (including %u-byte header)", size, sizeof(mmap_obj_header));
-  if(munmap(o->header, sizeof(mmap_obj_header)) == -1) RAISE_SYSERROR("munmap");
+  if(munmap(o->content, sizeof(mmap_obj_header)) == -1) RAISE_SYSERROR("munmap");
 
-  o->header = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
-  if(o->header == MAP_FAILED) RAISE_SYSERROR("full mmap");
+  o->content = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
+  if(o->content == MAP_FAILED) RAISE_SYSERROR("full mmap");
   DEBUG("loaded full %s object of %u bytes", magic, size);
 
   return NO_ERROR;
 }
 
+wp_error* mmap_obj_reload(mmap_obj* o) {
+  if(o->loaded_size != o->content->size) {
+    DEBUG("need to reload %s because size of %u is now %u", o->content->magic, o->loaded_size, o->content->size);
+    uint32_t new_size = o->content->size + (uint32_t)sizeof(mmap_obj_header);
+    if(munmap(o->content, sizeof(mmap_obj_header) + o->loaded_size) == -1) RAISE_SYSERROR("munmap");
+    o->content = mmap(NULL, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
+    if(o->content == MAP_FAILED) RAISE_SYSERROR("mmap of %uk", new_size / 1024);
+    o->loaded_size = o->content->size;
+    DEBUG("loaded %u bytes for %s. header is at %p", o->content->size, o->content->magic, o->content);
+  }
+
+  return NO_ERROR;
+}
+
 wp_error* mmap_obj_resize(mmap_obj* o, uint32_t data_size) {
-  DEBUG("going to expand from %u to %u bytes. current header is at %p", o->header->size, data_size, o->header);
+  DEBUG("going to expand from %u to %u bytes. current header is at %p", o->content->size, data_size, o->content);
 
-  if(munmap(o->header, sizeof(mmap_obj_header) + o->header->size) == -1) RAISE_SYSERROR("munmap");
+  if(munmap(o->content, sizeof(mmap_obj_header) + o->content->size) == -1) RAISE_SYSERROR("munmap");
   uint32_t size = data_size + (uint32_t)sizeof(mmap_obj_header);
 
   lseek(o->fd, size - 1, SEEK_SET);
   ssize_t num_bytes = write(o->fd, "", 1);
   if(num_bytes == -1) RAISE_SYSERROR("write");
   //lseek(fd, 0, SEEK_SET); // not necessary!
-  o->header = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
-  if(o->header == MAP_FAILED) RAISE_SYSERROR("mmap");
-  o->header->size = data_size;
-  DEBUG("loaded %u bytes after resize. header is at %p", o->header->size, o->header);
+  o->content = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
+  if(o->content == MAP_FAILED) RAISE_SYSERROR("mmap");
+  o->content->size = o->loaded_size = data_size;
+  DEBUG("loaded %u bytes after resize. header is at %p", o->content->size, o->content);
 
   return NO_ERROR;
 }
 
 wp_error* mmap_obj_unload(mmap_obj* o) {
-  DEBUG("unloading %u bytes", sizeof(mmap_obj_header) + o->header->size);
-  if(munmap(o->header, sizeof(mmap_obj_header) + o->header->size) == -1) RAISE_SYSERROR("munmap");
-  o->header = NULL;
+  DEBUG("unloading %u bytes", sizeof(mmap_obj_header) + o->content->size);
+  if(munmap(o->content, sizeof(mmap_obj_header) + o->content->size) == -1) RAISE_SYSERROR("munmap");
+  o->content = NULL;
   return NO_ERROR;
 }

data/ext/whistlepig/mmap-obj.h

@@ -7,7 +7,7 @@
 // wrappers around the logic of loading, unloading, and resizing
 // arbitrary-sized objects using mmap.
 //
-// note that aany of the mmap_obj_* functions may change the object pointer, so
+// note that any of the mmap_obj_* functions may change the object pointer, so
 // use MMAP_OBJ or MAP_OBJ_PTR to dereference (again) after calling them.
 
 #define MMAP_OBJ_MAGIC_SIZE 15
@@ -15,26 +15,27 @@
 #include <stdint.h>
 #include "error.h"
 
-// the header, with a magic string
+// what's actually mmap'd
 typedef struct mmap_obj_header {
   char magic[MMAP_OBJ_MAGIC_SIZE];
-  uint32_t size;
-  char obj[];
+  uint32_t size; // size of payload, not including this header
+  char obj[];    // the payload itself
 } mmap_obj_header;
 
 // what we pass around at runtime
 typedef struct mmap_obj {
   int fd;
-  mmap_obj_header* header;
+  uint32_t loaded_size; // compare against header->sizer
+  mmap_obj_header* content;
 } mmap_obj;
 
 // public API
 
 // public: get the actual object from an mmap_obj
-#define MMAP_OBJ(v, type) ((type*)&v.header->obj)
+#define MMAP_OBJ(v, type) ((type*)&v.content->obj)
 
 // public: get the object from an mmap_obj*
-#define MMAP_OBJ_PTR(v, type) (type*)v->header->obj
+#define MMAP_OBJ_PTR(v, type) (type*)v->content->obj
 
 // public: create an object with an initial size
 wp_error* mmap_obj_create(mmap_obj* o, const char* magic, const char* pathname, uint32_t initial_size) RAISES_ERROR;
@@ -43,6 +44,10 @@ wp_error* mmap_obj_create(mmap_obj* o, const char* magic, const char* pathname,
 // magic doesn't match)
 wp_error* mmap_obj_load(mmap_obj* o, const char* magic, const char* pathname) RAISES_ERROR;
 
+// public: load an object, but only if the size has changed since the
+// first load.
+wp_error* mmap_obj_reload(mmap_obj* o) RAISES_ERROR;
+
 // public: resize an object. note that the obj pointer might change after this call.
 wp_error* mmap_obj_resize(mmap_obj* o, uint32_t new_size) RAISES_ERROR;
 

data/ext/whistlepig/search.c

@@ -184,15 +184,16 @@ static wp_error* term_init_search_state(wp_query* q, wp_segment* seg) {
   term t;
   stringmap* sh = MMAP_OBJ(seg->stringmap, stringmap);
   termhash* th = MMAP_OBJ(seg->termhash, termhash);
+  stringpool* sp = MMAP_OBJ(seg->stringpool, stringpool);
 
   term_search_state* state = q->search_data = malloc(sizeof(term_search_state));
   state->started = 0;
 
   state->label = q->type == WP_QUERY_LABEL ? 1 : 0;
   if(state->label) t.field_s = 0;
-  else t.field_s = stringmap_string_to_int(sh, q->field); // will be -1 if not found
+  else t.field_s = stringmap_string_to_int(sh, sp, q->field); // will be -1 if not found
 
-  t.word_s = stringmap_string_to_int(sh, q->word);
+  t.word_s = stringmap_string_to_int(sh, sp, q->word);
 
   uint32_t offset = termhash_get_val(th, t);
   if(offset == (uint32_t)-1) offset = OFFSET_NONE;
@@ -268,10 +269,10 @@ static wp_error* neg_init_search_state(wp_query* q, wp_segment* seg) {
 
   RELAY_ERROR(wp_search_init_search_state(q->children, seg));
 
-  postings_region* pr = MMAP_OBJ(seg->postings, postings_region);
+  segment_info* si = MMAP_OBJ(seg->seginfo, segment_info);
   neg_search_state* state = q->search_data = malloc(sizeof(neg_search_state));
 
-  state->cur = pr->num_docs + 1;
+  state->cur = si->num_docs + 1;
   search_result result;
   int done;
   RELAY_ERROR(query_next_doc(q->children, seg, &result, &done));
@@ -294,8 +295,8 @@ static wp_error* neg_release_search_state(wp_query* q) {
 static wp_error* every_init_search_state(wp_query* q, wp_segment* seg) {
   q->search_data = malloc(sizeof(docid_t));
 
-  postings_region* pr = MMAP_OBJ(seg->postings, postings_region);
-  *(docid_t*)q->search_data = pr->num_docs;
+  segment_info* si = MMAP_OBJ(seg->seginfo, segment_info);
+  *(docid_t*)q->search_data = si->num_docs;
 
   return NO_ERROR;
 }

data/ext/whistlepig/segment.c

@@ -6,18 +6,50 @@
 #define POSTINGS_REGION_TYPE_IMMUTABLE_VBE   1
 #define POSTINGS_REGION_TYPE_MUTABLE_NO_POSITIONS 2 // bigger, mutable
 
+#define SEGMENT_VERSION 3
+
 #define wp_segment_label_posting_at(posting_region, offset) ((label_posting*)(posting_region->postings + offset))
 
-static void postings_region_init(postings_region* pr, uint32_t initial_size, uint32_t index_type_and_flags) {
-  pr->index_type_and_flags = index_type_and_flags;
-  pr->num_docs = 0;
+wp_error* wp_segment_grab_readlock(wp_segment* seg) {
+  segment_info* si = MMAP_OBJ(seg->seginfo, segment_info);
+  RELAY_ERROR(wp_lock_grab(&si->lock, WP_LOCK_READLOCK));
+  return NO_ERROR;
+}
+
+wp_error* wp_segment_grab_writelock(wp_segment* seg) {
+  segment_info* si = MMAP_OBJ(seg->seginfo, segment_info);
+  RELAY_ERROR(wp_lock_grab(&si->lock, WP_LOCK_WRITELOCK));
+  return NO_ERROR;
+}
+
+wp_error* wp_segment_release_lock(wp_segment* seg) {
+  segment_info* si = MMAP_OBJ(seg->seginfo, segment_info);
+  RELAY_ERROR(wp_lock_release(&si->lock));
+  return NO_ERROR;
+}
+
+static void postings_region_init(postings_region* pr, uint32_t initial_size, uint32_t postings_type_and_flags) {
+  pr->postings_type_and_flags = postings_type_and_flags;
   pr->num_postings = 0;
   pr->postings_head = 1; // skip one byte, which is reserved as OFFSET_NONE
   pr->postings_tail = initial_size;
 }
 
-RAISING_STATIC(postings_region_validate(postings_region* pr, uint32_t index_type_and_flags)) {
-  if(pr->index_type_and_flags != index_type_and_flags) RAISE_ERROR("segment has index type %u; expecting type %u", pr->index_type_and_flags, index_type_and_flags);
+RAISING_STATIC(segment_info_init(segment_info* si, uint32_t segment_version)) {
+  si->segment_version = segment_version;
+  si->num_docs = 0;
+
+  RELAY_ERROR(wp_lock_setup(&si->lock));
+  return NO_ERROR;
+}
+
+RAISING_STATIC(segment_info_validate(segment_info* si, uint32_t segment_version)) {
+  if(si->segment_version != segment_version) RAISE_ERROR("segment has type %u; expecting type %u", si->segment_version, segment_version);
+  return NO_ERROR;
+}
+
+RAISING_STATIC(postings_region_validate(postings_region* pr, uint32_t postings_type_and_flags)) {
+  if(pr->postings_type_and_flags != postings_type_and_flags) RAISE_ERROR("postings region has type %u; expecting type %u", pr->postings_type_and_flags, postings_type_and_flags);
   return NO_ERROR;
 }
 
@@ -27,59 +59,78 @@ RAISING_STATIC(postings_region_validate(postings_region* pr, uint32_t index_type
 wp_error* wp_segment_load(wp_segment* segment, const char* pathname_base) {
   char fn[FN_SIZE];
 
+  // open the segment info
+  snprintf(fn, 128, "%s.si", pathname_base);
+  RELAY_ERROR(mmap_obj_load(&segment->seginfo, "wp/seginfo", fn));
+  RELAY_ERROR(segment_info_validate(MMAP_OBJ(segment->seginfo, segment_info), SEGMENT_VERSION));
+
   // open the string pool
   snprintf(fn, 128, "%s.sp", pathname_base);
-  RELAY_ERROR(mmap_obj_load(&segment->stringpool, "ti/stringpool", fn));
+  RELAY_ERROR(mmap_obj_load(&segment->stringpool, "wp/stringpool", fn));
 
   // open the string hash
-  snprintf(fn, 128, "%s.sh_", pathname_base);
-  RELAY_ERROR(mmap_obj_load(&segment->stringmap, "ti/stringmap", fn));
-  stringmap_setup(MMAP_OBJ(segment->stringmap, stringmap), MMAP_OBJ(segment->stringpool, stringpool));
+  snprintf(fn, 128, "%s.sh", pathname_base);
+  RELAY_ERROR(mmap_obj_load(&segment->stringmap, "wp/stringmap", fn));
 
   // open the term hash
   snprintf(fn, 128, "%s.th", pathname_base);
-  RELAY_ERROR(mmap_obj_load(&segment->termhash, "ti/termhash", fn));
-  termhash_setup(MMAP_OBJ(segment->termhash, termhash));
+  RELAY_ERROR(mmap_obj_load(&segment->termhash, "wp/termhash", fn));
 
   // open the postings region
   snprintf(fn, 128, "%s." WP_SEGMENT_POSTING_REGION_PATH_SUFFIX, pathname_base);
-  RELAY_ERROR(mmap_obj_load(&segment->postings, "ti/postings", fn));
+  RELAY_ERROR(mmap_obj_load(&segment->postings, "wp/postings", fn));
   RELAY_ERROR(postings_region_validate(MMAP_OBJ(segment->postings, postings_region), POSTINGS_REGION_TYPE_IMMUTABLE_VBE));
 
   // open the labels postings region
   snprintf(fn, 128, "%s.lb", pathname_base);
-  RELAY_ERROR(mmap_obj_load(&segment->labels, "ti/labels", fn));
+  RELAY_ERROR(mmap_obj_load(&segment->labels, "wp/labels", fn));
   RELAY_ERROR(postings_region_validate(MMAP_OBJ(segment->labels, postings_region), POSTINGS_REGION_TYPE_MUTABLE_NO_POSITIONS));
 
   return NO_ERROR;
 }
 
+wp_error* wp_segment_reload(wp_segment* segment) {
+  RELAY_ERROR(mmap_obj_reload(&segment->seginfo));
+  RELAY_ERROR(mmap_obj_reload(&segment->stringpool));
+  RELAY_ERROR(mmap_obj_reload(&segment->stringmap));
+  RELAY_ERROR(mmap_obj_reload(&segment->termhash));
+  RELAY_ERROR(mmap_obj_reload(&segment->postings));
+  RELAY_ERROR(mmap_obj_reload(&segment->labels));
+
+  return NO_ERROR;
+}
+
 wp_error* wp_segment_create(wp_segment* segment, const char* pathname_base) {
   char fn[FN_SIZE];
 
+  // create the segment info
+  snprintf(fn, 128, "%s.si", pathname_base);
+  RELAY_ERROR(mmap_obj_create(&segment->seginfo, "wp/seginfo", fn, sizeof(segment_info)));
+  RELAY_ERROR(segment_info_init(MMAP_OBJ(segment->seginfo, segment_info), SEGMENT_VERSION));
+
   // create the string pool
   snprintf(fn, 128, "%s.sp", pathname_base);
-  RELAY_ERROR(mmap_obj_create(&segment->stringpool, "ti/stringpool", fn, stringpool_initial_size()));
+  RELAY_ERROR(mmap_obj_create(&segment->stringpool, "wp/stringpool", fn, stringpool_initial_size()));
   stringpool_init(MMAP_OBJ(segment->stringpool, stringpool));
 
   // create the string hash
-  snprintf(fn, 128, "%s.sh_", pathname_base);
-  RELAY_ERROR(mmap_obj_create(&segment->stringmap, "ti/stringmap", fn, stringmap_initial_size()));
-  stringmap_init(MMAP_OBJ(segment->stringmap, stringmap), MMAP_OBJ(segment->stringpool, stringpool));
+  snprintf(fn, 128, "%s.sh", pathname_base);
+  RELAY_ERROR(mmap_obj_create(&segment->stringmap, "wp/stringmap", fn, stringmap_initial_size()));
+  stringmap_init(MMAP_OBJ(segment->stringmap, stringmap));
 
   // create the term hash
   snprintf(fn, 128, "%s.th", pathname_base);
-  RELAY_ERROR(mmap_obj_create(&segment->termhash, "ti/termhash", fn, termhash_initial_size()));
+  RELAY_ERROR(mmap_obj_create(&segment->termhash, "wp/termhash", fn, termhash_initial_size()));
   termhash_init(MMAP_OBJ(segment->termhash, termhash));
 
   // create the postings region
   snprintf(fn, 128, "%s." WP_SEGMENT_POSTING_REGION_PATH_SUFFIX, pathname_base);
-  RELAY_ERROR(mmap_obj_create(&segment->postings, "ti/postings", fn, sizeof(postings_region) + INITIAL_POSTINGS_SIZE));
+  RELAY_ERROR(mmap_obj_create(&segment->postings, "wp/postings", fn, sizeof(postings_region) + INITIAL_POSTINGS_SIZE));
   postings_region_init(MMAP_OBJ(segment->postings, postings_region), INITIAL_POSTINGS_SIZE, POSTINGS_REGION_TYPE_IMMUTABLE_VBE);
 
   // create the labels postings region
   snprintf(fn, 128, "%s.lb", pathname_base);
-  RELAY_ERROR(mmap_obj_create(&segment->labels, "ti/labels", fn, sizeof(postings_region) + INITIAL_POSTINGS_SIZE));
+  RELAY_ERROR(mmap_obj_create(&segment->labels, "wp/labels", fn, sizeof(postings_region) + INITIAL_POSTINGS_SIZE));
   postings_region_init(MMAP_OBJ(segment->labels, postings_region), INITIAL_POSTINGS_SIZE, POSTINGS_REGION_TYPE_MUTABLE_NO_POSITIONS);
 
   return NO_ERROR;
@@ -96,11 +147,13 @@ int wp_segment_exists(const char* pathname_base) {
 wp_error* wp_segment_delete(const char* pathname_base) {
   char fn[FN_SIZE];
 
+  snprintf(fn, 128, "%s.si", pathname_base);
+  unlink(fn);
   snprintf(fn, 128, "%s." WP_SEGMENT_POSTING_REGION_PATH_SUFFIX, pathname_base);
   unlink(fn);
   snprintf(fn, 128, "%s.sp", pathname_base);
   unlink(fn);
-  snprintf(fn, 128, "%s.sh_", pathname_base);
+  snprintf(fn, 128, "%s.sh", pathname_base);
   unlink(fn);
   snprintf(fn, 128, "%s.th", pathname_base);
   unlink(fn);
@@ -132,9 +185,7 @@ RAISING_STATIC(bump_stringmap(wp_segment* s, int* success)) {
     }
     else {
       RELAY_ERROR(mmap_obj_resize(&s->stringmap, next_size));
-      sh = MMAP_OBJ(s->stringmap, stringmap); // this could have changed!
-      stringmap_setup(sh, MMAP_OBJ(s->stringpool, stringpool));
-      RELAY_ERROR(stringmap_bump_size(sh));
+      RELAY_ERROR(stringmap_bump_size(MMAP_OBJ(s->stringmap, stringmap), MMAP_OBJ(s->stringpool, stringpool)));
     }
   }
 
@@ -154,10 +205,7 @@ RAISING_STATIC(bump_stringpool(wp_segment* s, int* success)) {
     }
     else {
       RELAY_ERROR(mmap_obj_resize(&s->stringpool, next_size));
-      sp = MMAP_OBJ(s->stringpool, stringpool); // may have changed!
-      stringmap* sh = MMAP_OBJ(s->stringmap, stringmap);
-      sh->pool = sp; // need to update it here too
-      stringpool_bump_size(sp);
+      stringpool_bump_size(MMAP_OBJ(s->stringpool, stringpool));
     }
   }
 
@@ -177,9 +225,7 @@ RAISING_STATIC(bump_termhash(wp_segment* s, int* success)) {
     }
     else {
       RELAY_ERROR(mmap_obj_resize(&s->termhash, next_size));
-      th = MMAP_OBJ(s->termhash, termhash); // could have changed!
-      termhash_setup(th);
-      RELAY_ERROR(termhash_bump_size(th));
+      RELAY_ERROR(termhash_bump_size(MMAP_OBJ(s->termhash, termhash)));
       *success = 1;
     }
   }
@@ -196,7 +242,7 @@ RAISING_STATIC(postings_region_ensure_fit(mmap_obj* mmopr, uint32_t postings_byt
   uint32_t new_tail = pr->postings_tail;
   while(new_tail <= new_head) new_tail = new_tail * 2;
 
-  if(new_tail > MAX_POSTINGS_REGION_SIZE) new_tail = MAX_POSTINGS_REGION_SIZE;
+  if(new_tail > MAX_POSTINGS_REGION_SIZE - sizeof(mmap_obj_header)) new_tail = MAX_POSTINGS_REGION_SIZE - sizeof(mmap_obj_header);
   DEBUG("new tail will be %u, current is %u, max is %u", new_tail, pr->postings_tail, MAX_POSTINGS_REGION_SIZE);
 
   if(new_tail <= new_head) { // can't increase enough
@@ -362,7 +408,7 @@ wp_error* wp_segment_read_posting(wp_segment* s, uint32_t offset, posting* po, i
 
   RELAY_ERROR(read_multibyte(&pr->postings[offset], &po->next_offset, &size));
   //DEBUG("read next_offset %u -> %u (%u bytes)", po->next_offset, orig_offset - po->next_offset, size);
-  if((po->next_offset == 0) || (po->next_offset > orig_offset)) RAISE_ERROR("read invalid next_offset %u (must be > 0 and < %u", po->next_offset, orig_offset);
+  if((po->next_offset == 0) || (po->next_offset > orig_offset)) RAISE_ERROR("read invalid next_offset %u (must be > 0 and < %u)", po->next_offset, orig_offset);
   po->next_offset = orig_offset - po->next_offset;
   offset += size;
 
@@ -408,11 +454,12 @@ wp_error* wp_segment_add_posting(wp_segment* s, const char* field, const char* w
   postings_region* pr = MMAP_OBJ(s->postings, postings_region);
   stringmap* sh = MMAP_OBJ(s->stringmap, stringmap);
   termhash* th = MMAP_OBJ(s->termhash, termhash);
+  stringpool* sp = MMAP_OBJ(s->stringpool, stringpool);
 
   // construct the term object
   term t;
-  RELAY_ERROR(stringmap_add(sh, field, &t.field_s));
-  RELAY_ERROR(stringmap_add(sh, word, &t.word_s));
+  RELAY_ERROR(stringmap_add(sh, sp, field, &t.field_s));
+  RELAY_ERROR(stringmap_add(sh, sp, word, &t.word_s));
 
   // find the offset of the next posting
   posting po;
@@ -480,12 +527,13 @@ wp_error* wp_segment_add_label(wp_segment* s, const char* label, docid_t doc_id)
   postings_region* pr = MMAP_OBJ(s->labels, postings_region);
   stringmap* sh = MMAP_OBJ(s->stringmap, stringmap);
   termhash* th = MMAP_OBJ(s->termhash, termhash);
+  stringpool* sp = MMAP_OBJ(s->stringpool, stringpool);
 
   // construct the term object. term objects for labels have the special
   // sentinel field value 0
   term t;
   t.field_s = 0; // label sentinel value
-  RELAY_ERROR(stringmap_add(sh, label, &t.word_s)); // get word key
+  RELAY_ERROR(stringmap_add(sh, sp, label, &t.word_s)); // get word key
 
   // find the previous and next label postings, between which we'll insert this
   // posting
@@ -558,12 +606,13 @@ wp_error* wp_segment_remove_label(wp_segment* s, const char* label, docid_t doc_
   postings_region* pr = MMAP_OBJ(s->labels, postings_region);
   stringmap* sh = MMAP_OBJ(s->stringmap, stringmap);
   termhash* th = MMAP_OBJ(s->termhash, termhash);
+  stringpool* sp = MMAP_OBJ(s->stringpool, stringpool);
 
   // construct the term object. term objects for labels have the special
   // sentinel field value 0
   term t;
   t.field_s = 0; // label sentinel value
-  t.word_s = stringmap_string_to_int(sh, label); // will be -1 if not there
+  t.word_s = stringmap_string_to_int(sh, sp, label); // will be -1 if not there
 
   // find the posting and the previous posting in the list, if any
   uint32_t prev_offset = OFFSET_NONE;
@@ -613,12 +662,13 @@ wp_error* wp_segment_remove_label(wp_segment* s, const char* label, docid_t doc_
 }
 
 wp_error* wp_segment_grab_docid(wp_segment* segment, docid_t* doc_id) {
-  postings_region* pr = MMAP_OBJ(segment->postings, postings_region);
-  *doc_id = ++pr->num_docs;
+  segment_info* si = MMAP_OBJ(segment->seginfo, segment_info);
+  *doc_id = ++si->num_docs;
   return NO_ERROR;
 }
 
 wp_error* wp_segment_dumpinfo(wp_segment* segment, FILE* stream) {
+  segment_info* si = MMAP_OBJ(segment->seginfo, segment_info);
   postings_region* pr = MMAP_OBJ(segment->postings, postings_region);
   stringmap* sh = MMAP_OBJ(segment->stringmap, stringmap);
   stringpool* sp = MMAP_OBJ(segment->stringpool, stringpool);
@@ -626,17 +676,17 @@ wp_error* wp_segment_dumpinfo(wp_segment* segment, FILE* stream) {
 
   #define p(a, b) 100.0 * (float)a / (float)b
 
-  fprintf(stream, "segment has type %u\n", pr->index_type_and_flags);
-  fprintf(stream, "segment has %u docs and %u postings\n", pr->num_docs, pr->num_postings);
-  fprintf(stream, "postings region is %6ukb at %3.1f%% saturation\n", segment->postings.header->size / 1024, p(pr->postings_head, pr->postings_tail));
-  fprintf(stream, "    string hash is %6ukb at %3.1f%% saturation\n", segment->stringmap.header->size / 1024, p(sh->n_occupied, sh->n_buckets));
-  fprintf(stream, "     stringpool is %6ukb at %3.1f%% saturation\n", segment->stringpool.header->size / 1024, p(sp->next, sp->size));
-  fprintf(stream, "     term hash has %6ukb at %3.1f%% saturation\n", segment->termhash.header->size / 1024, p(th->n_occupied, th->n_buckets));
+  fprintf(stream, "segment has type %u\n", pr->postings_type_and_flags);
+  fprintf(stream, "segment has %u docs and %u postings\n", si->num_docs, pr->num_postings);
+  fprintf(stream, "postings region is %6ukb at %3.1f%% saturation\n", segment->postings.content->size / 1024, p(pr->postings_head, pr->postings_tail));
+  fprintf(stream, "    string hash is %6ukb at %3.1f%% saturation\n", segment->stringmap.content->size / 1024, p(sh->n_occupied, sh->n_buckets));
+  fprintf(stream, "     stringpool is %6ukb at %3.1f%% saturation\n", segment->stringpool.content->size / 1024, p(sp->next, sp->size));
+  fprintf(stream, "     term hash has %6ukb at %3.1f%% saturation\n", segment->termhash.content->size / 1024, p(th->n_occupied, th->n_buckets));
 
   return NO_ERROR;
 }
 
 uint64_t wp_segment_num_docs(wp_segment* seg) {
-  postings_region* pr = MMAP_OBJ(seg->postings, postings_region);
-  return pr->num_docs;
+  segment_info* si = MMAP_OBJ(seg->seginfo, segment_info);
+  return si->num_docs;
 }

data/ext/whistlepig/segment.h

@@ -15,6 +15,8 @@
 // different, mutable format. regular text is stored in a compressed format
 // that is not amenable to later changes.
 
+#include <pthread.h>
+
 #include "defaults.h"
 #include "stringmap.h"
 #include "termhash.h"
@@ -60,21 +62,27 @@ typedef struct label_posting {
 // terms also; see termhash.h.)
 
 #define MAX_LOGICAL_DOCID 2147483646 // don't tweak me
-#define MAX_POSTINGS_REGION_SIZE (512*1024*1024) // tweak me
+#define MAX_POSTINGS_REGION_SIZE (256*1024*1024) // tweak me
 
 #define WP_SEGMENT_POSTING_REGION_PATH_SUFFIX "pr"
 
 // the header for the postings region
 typedef struct postings_region {
-  uint32_t index_type_and_flags;
-  uint32_t num_docs;
+  uint32_t postings_type_and_flags;
   uint32_t num_postings;
   uint32_t postings_head, postings_tail;
   uint8_t postings[]; // where the postings go yo
 } postings_region;
 
+typedef struct segment_info {
+  uint32_t segment_version;
+  uint32_t num_docs;
+  pthread_rwlock_t lock;
+} segment_info;
+
 // a segment is a bunch of all these things
 typedef struct wp_segment {
+  mmap_obj seginfo;
   mmap_obj stringmap;
   mmap_obj stringpool;
   mmap_obj termhash;
@@ -93,6 +101,9 @@ wp_error* wp_segment_create(wp_segment* segment, const char* pathname_base) RAIS
 // public: load a segment, raising an error unless it already exists
 wp_error* wp_segment_load(wp_segment* segment, const char* pathname_base) RAISES_ERROR;
 
+// public: reload a segment as necessary, in case an external writer has changed the mmap object sizes
+wp_error* wp_segment_reload(wp_segment* segment) RAISES_ERROR;
+
 // public: unload a segment
 wp_error* wp_segment_unload(wp_segment* s) RAISES_ERROR;
 
@@ -102,6 +113,11 @@ uint64_t wp_segment_num_docs(wp_segment* s);
 // public: delete a segment from disk
 wp_error* wp_segment_delete(const char* pathname_base) RAISES_ERROR;
 
+// public: lock grabbing and releasing
+wp_error* wp_segment_grab_readlock(wp_segment* seg) RAISES_ERROR;
+wp_error* wp_segment_grab_writelock(wp_segment* seg) RAISES_ERROR;
+wp_error* wp_segment_release_lock(wp_segment* seg) RAISES_ERROR;
+
 // private: read a posting from the postings region at a given offset
 wp_error* wp_segment_read_posting(wp_segment* s, uint32_t offset, posting* po, int include_positions) RAISES_ERROR;