whistlepig 0.11.2 → 0.12

data/README

@@ -8,7 +8,7 @@ full-text search without the frills, Whistlepig may be for you.
 Whistlepig is written in ANSI C99. It currently provides a C API and Ruby
 bindings.
 
-Latest version: 0.11.2, released 2012-05-19.
+Latest version: 0.12, released 2012-06-09.
         Status: beta
           News: http://all-thing.net/label/whistlepig/
       Homepage: http://masanjin.net/whistlepig/
@@ -16,7 +16,7 @@ Latest version: 0.11.2, released 2012-05-19.
 
 = Getting it
 
-       Tarball:  http://masanjin.net/whistlepig/whistlepig-0.11.2.tar.gz
+       Tarball:  http://masanjin.net/whistlepig/whistlepig-0.12.tar.gz
        Rubygem:  gem install whistlepig
            Git:  git clone git://github.com/wmorgan/whistlepig.git
 

data/ext/whistlepig/error.c

@@ -1,9 +1,10 @@
 #include <stdlib.h>
 #include "error.h"
 
-wp_error* wp_error_new(const char* msg, const char* src) {
+wp_error* wp_error_new(const char* msg, const char* src, unsigned char type) {
   wp_error* ret = malloc(sizeof(wp_error));
   ret->msg = msg;
+  ret->type = type;
   ret->size = 1;
   ret->srcs = malloc(sizeof(const char*));
   ret->srcs[0] = src;

data/ext/whistlepig/error.h

@@ -25,8 +25,13 @@
 #include <stdio.h>
 #include <string.h>
 
+#define WP_ERROR_TYPE_BASIC 1
+#define WP_ERROR_TYPE_SYSTEM 2
+#define WP_ERROR_TYPE_VERSION 3
+
 // pseudo-backtrace
 typedef struct wp_error {
+  unsigned char type;
   unsigned int size;
   const char* msg;
   const char** srcs;
@@ -39,25 +44,30 @@ typedef struct wp_error {
 // API methods
 
 // private: make a new error object with a message and source line
-wp_error* wp_error_new(const char* msg, const char* src) RAISES_ERROR;
+wp_error* wp_error_new(const char* msg, const char* src, unsigned char type) RAISES_ERROR;
 // private: add a source line to a pre-existing error
 wp_error* wp_error_chain(wp_error* e, const char* src) RAISES_ERROR;
 
 // public: free an error, once handled
 void wp_error_free(wp_error* e);
 
-// public: raise an error with a printf-style message
-#define RAISE_ERROR(fmt, ...) do { \
+// private: internal mechanics for raising an error
+#define RAISE_ERROR_OF_TYPE(type, fmt, ...) do { \
   char* msg = malloc(1024); \
   char* src = malloc(1024); \
   snprintf(msg, 1024, fmt, ## __VA_ARGS__); \
   snprintf(src, 1024, "%s (%s:%d)", __PRETTY_FUNCTION__, __FILE__, __LINE__); \
-  return wp_error_new(msg, src); \
+  return wp_error_new(msg, src, type); \
 } while(0)
 
-// public: raise an error with a printf-style message and have strerror() autoamtically
-// appended
-#define RAISE_SYSERROR(fmt, ...) RAISE_ERROR(fmt ": %s", ## __VA_ARGS__, strerror(errno))
+// public: raise a basic error
+#define RAISE_ERROR(fmt, ...) RAISE_ERROR_OF_TYPE(WP_ERROR_TYPE_BASIC, fmt, ## __VA_ARGS__)
+
+// public: raise a version error
+#define RAISE_VERSION_ERROR(fmt, ...) RAISE_ERROR_OF_TYPE(WP_ERROR_TYPE_VERSION, fmt, ## __VA_ARGS__)
+
+// public: raise a system error with strerror() automatically appended to the message
+#define RAISE_SYSERROR(fmt, ...) RAISE_ERROR_OF_TYPE(WP_ERROR_TYPE_SYSTEM, fmt ": %s", ## __VA_ARGS__, strerror(errno))
 
 // public: relay an error up the stack if the called function returns one.
 #define RELAY_ERROR(e) do { \

data/ext/whistlepig/index.c

@@ -32,7 +32,6 @@ RAISING_STATIC(release_lock(wp_index* index)) {
   return NO_ERROR;
 }
 
-
 RAISING_STATIC(index_info_init(index_info* ii, uint32_t index_version)) {
   ii->index_version = index_version;
   ii->num_segments = 0;
@@ -42,7 +41,7 @@ RAISING_STATIC(index_info_init(index_info* ii, uint32_t index_version)) {
 }
 
 RAISING_STATIC(index_info_validate(index_info* ii, uint32_t index_version)) {
-  if(ii->index_version != index_version) RAISE_ERROR("index has type %u; expecting type %u", ii->index_version, index_version);
+  if(ii->index_version != index_version) RAISE_VERSION_ERROR("index has type %u; expecting type %u", ii->index_version, index_version);
   return NO_ERROR;
 }
 
@@ -149,6 +148,57 @@ wp_error* wp_index_setup_query(wp_index* index, wp_query* query) {
   return NO_ERROR;
 }
 
+#define RESULT_BUF_SIZE 1024
+// count the results by running the query until it stops. slow!
+RAISING_STATIC(count_query_by_running_it(wp_index* index, wp_query* query, uint32_t* num_results)) {
+  uint64_t results[RESULT_BUF_SIZE];
+
+  *num_results = 0;
+  while(1) {
+    uint32_t this_num_results;
+    RELAY_ERROR(wp_index_run_query(index, query, RESULT_BUF_SIZE, &this_num_results, results));
+    *num_results += this_num_results;
+    if(this_num_results < RESULT_BUF_SIZE) break; // done
+  }
+
+  return NO_ERROR;
+}
+
+RAISING_STATIC(count_query_from_posting_list_header(wp_index* index, wp_query* query, uint32_t* num_results)) {
+  // make sure we have know about all segments (one could've been added by a writer)
+  RELAY_ERROR(grab_readlock(index));
+  RELAY_ERROR(ensure_all_segments(index));
+  RELAY_ERROR(release_lock(index));
+
+  *num_results = 0;
+  for(int i = 0; i < index->num_segments; i++) {
+    uint32_t this_num_results;
+
+    DEBUG("counting on segment %d", i);
+    wp_segment* seg = &index->segments[i];
+    RELAY_ERROR(wp_segment_grab_readlock(seg));
+    RELAY_ERROR(wp_segment_reload(seg));
+    RELAY_ERROR(wp_segment_count_term(seg, query->field, query->word, &this_num_results));
+    RELAY_ERROR(wp_segment_release_lock(seg));
+    *num_results += this_num_results;
+    DEBUG("got %d results from segment %d", this_num_results, i);
+  }
+
+  return NO_ERROR;
+}
+
+RAISING_STATIC(count_query(wp_index* index, wp_query* query, uint32_t* num_results)) {
+  switch(query->type) {
+    case WP_QUERY_TERM:
+    case WP_QUERY_LABEL:
+      RELAY_ERROR(count_query_from_posting_list_header(index, query, num_results));
+      break;
+    case WP_QUERY_EVERY: // TODO -- special case this
+    default:
+      RELAY_ERROR(count_query_by_running_it(index, query, num_results));
+  }
+  return NO_ERROR;
+}
 // can be called multiple times to resume
 wp_error* wp_index_run_query(wp_index* index, wp_query* query, uint32_t max_num_results, uint32_t* num_results, uint64_t* results) {
   *num_results = 0;
@@ -209,20 +259,10 @@ wp_error* wp_index_run_query(wp_index* index, wp_query* query, uint32_t max_num_
   return NO_ERROR;
 }
 
-#define RESULT_BUF_SIZE 1024
-// count the results by just running the query until it stops. slow!
+// just count the results, don't return them
 wp_error* wp_index_count_results(wp_index* index, wp_query* query, uint32_t* num_results) {
-  uint64_t results[RESULT_BUF_SIZE];
-
-  *num_results = 0;
   RELAY_ERROR(wp_index_setup_query(index, query));
-  while(1) {
-    uint32_t this_num_results;
-    RELAY_ERROR(wp_index_run_query(index, query, RESULT_BUF_SIZE, &this_num_results, results));
-    *num_results += this_num_results;
-    if(this_num_results < RESULT_BUF_SIZE) break; // done
-  }
-
+  RELAY_ERROR(count_query(index, query, num_results));
   RELAY_ERROR(wp_index_teardown_query(index, query));
 
   return NO_ERROR;
@@ -369,7 +409,7 @@ wp_error* wp_index_add_label(wp_index* index, const char* label, uint64_t doc_id
     if(doc_id > index->docid_offsets[i - 1]) {
       wp_segment* seg = &index->segments[i - 1];
 
-      DEBUG("found doc %llu in segment %u", doc_id, i - 1);
+      DEBUG("found doc %"PRIu64" in segment %u", doc_id, i - 1);
       RELAY_ERROR(wp_segment_grab_writelock(seg));
       RELAY_ERROR(wp_segment_reload(seg));
       RELAY_ERROR(wp_segment_add_label(seg, label, (docid_t)(doc_id - index->docid_offsets[i - 1])));
@@ -377,7 +417,7 @@ wp_error* wp_index_add_label(wp_index* index, const char* label, uint64_t doc_id
       found = 1;
       break;
     }
-    else DEBUG("did not find doc %llu in segment %u", doc_id, i - 1);
+    else DEBUG("did not find doc %"PRIu64" in segment %u", doc_id, i - 1);
   }
 
   if(!found) RAISE_ERROR("couldn't find doc id %"PRIu64, doc_id);
@@ -396,7 +436,7 @@ wp_error* wp_index_remove_label(wp_index* index, const char* label, uint64_t doc
     if(doc_id > index->docid_offsets[i - 1]) {
       wp_segment* seg = &index->segments[i - 1];
 
-      DEBUG("found doc %llu in segment %u", doc_id, i - 1);
+      DEBUG("found doc %"PRIu64" in segment %u", doc_id, i - 1);
       RELAY_ERROR(wp_segment_grab_writelock(seg));
       RELAY_ERROR(wp_segment_reload(seg));
       RELAY_ERROR(wp_segment_remove_label(seg, label, (docid_t)(doc_id - index->docid_offsets[i - 1])));
@@ -404,7 +444,7 @@ wp_error* wp_index_remove_label(wp_index* index, const char* label, uint64_t doc
       found = 1;
       break;
     }
-    else DEBUG("did not find doc %llu in segment %u", doc_id, i - 1);
+    else DEBUG("did not find doc %"PRIu64" in segment %u", doc_id, i - 1);
   }
 
   if(!found) RAISE_ERROR("couldn't find doc id %"PRIu64, doc_id);

data/ext/whistlepig/mmap-obj.c

@@ -14,7 +14,7 @@ wp_error* mmap_obj_create(mmap_obj* o, const char* magic, const char* pathname,
   if(o->fd == -1) RAISE_SYSERROR("cannot create %s", pathname);
 
   uint32_t size = initial_size + (uint32_t)sizeof(mmap_obj_header);
-  DEBUG("creating %s with %u + %u = %u bytes for %s object", pathname, initial_size, sizeof(mmap_obj_header), size, magic);
+  DEBUG("creating %s with %u + %lu = %u bytes for %s object", pathname, initial_size, sizeof(mmap_obj_header), size, magic);
   lseek(o->fd, size - 1, SEEK_SET);
   ssize_t num_bytes = write(o->fd, "", 1);
   if(num_bytes == -1) RAISE_SYSERROR("write");
@@ -35,14 +35,14 @@ wp_error* mmap_obj_load(mmap_obj* o, const char* magic, const char* pathname) {
   // load header
   o->content = mmap(NULL, sizeof(mmap_obj_header), PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
   if(o->content == MAP_FAILED) RAISE_SYSERROR("header mmap");
-  DEBUG("loaded header of %u bytes for %s object", sizeof(mmap_obj_header), magic);
+  DEBUG("loaded header of %lu bytes for %s object", sizeof(mmap_obj_header), magic);
 
   RELAY_ERROR(validate(o->content, magic));
 
   o->loaded_size = o->content->size;
 
   uint32_t size = o->content->size + (uint32_t)sizeof(mmap_obj_header);
-  DEBUG("full size is %u bytes (including %u-byte header)", size, sizeof(mmap_obj_header));
+  DEBUG("full size is %u bytes (including %lu-byte header)", size, sizeof(mmap_obj_header));
   if(munmap(o->content, sizeof(mmap_obj_header)) == -1) RAISE_SYSERROR("munmap");
 
   o->content = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, o->fd, 0);
@@ -85,7 +85,7 @@ wp_error* mmap_obj_resize(mmap_obj* o, uint32_t data_size) {
 }
 
 wp_error* mmap_obj_unload(mmap_obj* o) {
-  DEBUG("unloading %u bytes", sizeof(mmap_obj_header) + o->content->size);
+  DEBUG("unloading %lu bytes", sizeof(mmap_obj_header) + o->content->size);
   if(munmap(o->content, sizeof(mmap_obj_header) + o->content->size) == -1) RAISE_SYSERROR("munmap");
   o->content = NULL;
   return NO_ERROR;

data/ext/whistlepig/search.c

@@ -195,8 +195,14 @@ static wp_error* term_init_search_state(wp_query* q, wp_segment* seg) {
 
   t.word_s = stringmap_string_to_int(sh, sp, q->word);
 
-  uint32_t offset = termhash_get_val(th, t);
-  if(offset == (uint32_t)-1) offset = OFFSET_NONE;
+  uint32_t offset;
+  posting_list_header* plh = termhash_get_val(th, t);
+
+  DEBUG("posting list header for %s:%s (-> %u:%u) is %p", q->field, q->word, t.field_s, t.word_s, plh);
+  if(plh == NULL) offset = OFFSET_NONE;
+  else offset = plh->next_offset;
+
+  if(plh) DEBUG("posting list header has count=%u next_offset=%u", plh->count, plh->next_offset);
 
   if(offset == OFFSET_NONE) state->done = 1; // no entry in term hash
   else {
@@ -475,8 +481,8 @@ static wp_error* conj_next_doc(wp_query* q, wp_segment* seg, search_result* resu
 
   while(!found && !*done) {
     RELAY_ERROR(query_next_doc(master, seg, result, done));
-    DEBUG("master reports doc %u done %d", result->doc_id, *done);
     if(!*done) {
+      DEBUG("master reports doc %u done %d", result->doc_id, *done);
       search_doc = result->doc_id;
       wp_search_result_free(result); // sigh
       RELAY_ERROR(conj_advance_to_doc(q, seg, search_doc, result, &found, done));
@@ -564,8 +570,8 @@ static wp_error* phrase_next_doc(wp_query* q, wp_segment* seg, search_result* re
 
   while(!found && !*done) {
     RELAY_ERROR(query_next_doc(master, seg, result, done));
-    DEBUG("master reports doc %u done %d", result->doc_id, *done);
     if(!*done) {
+      DEBUG("master reports doc %u done %d", result->doc_id, *done);
       search_doc = result->doc_id;
       wp_search_result_free(result); // sigh
       RELAY_ERROR(phrase_advance_to_doc(q, seg, search_doc, result, &found, done));
@@ -784,7 +790,6 @@ static wp_error* every_advance_to_doc(wp_query* q, wp_segment* seg, docid_t doc_
 
 wp_error* wp_search_run_query_on_segment(struct wp_query* q, struct wp_segment* s, uint32_t max_num_results, uint32_t* num_results, search_result* results) {
   int done;
-
   *num_results = 0;
 
 #ifdef DEBUG

data/ext/whistlepig/segment.c

@@ -3,13 +3,16 @@
 #include <unistd.h>
 #include "whistlepig.h"
 
-#define POSTINGS_REGION_TYPE_IMMUTABLE_VBE   1
+#define POSTINGS_REGION_TYPE_IMMUTABLE_VBE 1
 #define POSTINGS_REGION_TYPE_MUTABLE_NO_POSITIONS 2 // bigger, mutable
 
-#define SEGMENT_VERSION 3
+#define SEGMENT_VERSION 4
 
 #define wp_segment_label_posting_at(posting_region, offset) ((label_posting*)(posting_region->postings + offset))
 
+static posting_list_header blank_plh = { .count = 0, .next_offset = OFFSET_NONE };
+static term dead_term = { .field_s = 0, .word_s = 0 };
+
 wp_error* wp_segment_grab_readlock(wp_segment* seg) {
   segment_info* si = MMAP_OBJ(seg->seginfo, segment_info);
   RELAY_ERROR(wp_lock_grab(&si->lock, WP_LOCK_READLOCK));
@@ -28,6 +31,23 @@ wp_error* wp_segment_release_lock(wp_segment* seg) {
   return NO_ERROR;
 }
 
+wp_error* wp_segment_count_term(wp_segment* seg, const char* field, const char* word, uint32_t* num_results) {
+  stringmap* sh = MMAP_OBJ(seg->stringmap, stringmap);
+  stringpool* sp = MMAP_OBJ(seg->stringpool, stringpool);
+  termhash* th = MMAP_OBJ(seg->termhash, termhash);
+
+  term t;
+  if(field == NULL) t.field_s = 0; // label sentinel
+  else t.field_s = stringmap_string_to_int(sh, sp, field);
+  t.word_s = stringmap_string_to_int(sh, sp, word);
+
+  posting_list_header* plh = termhash_get_val(th, t);
+  if(plh == NULL) *num_results = 0;
+  else *num_results = plh->count;
+
+  return NO_ERROR;
+}
+
 static void postings_region_init(postings_region* pr, uint32_t initial_size, uint32_t postings_type_and_flags) {
   pr->postings_type_and_flags = postings_type_and_flags;
   pr->num_postings = 0;
@@ -44,7 +64,7 @@ RAISING_STATIC(segment_info_init(segment_info* si, uint32_t segment_version)) {
 }
 
 RAISING_STATIC(segment_info_validate(segment_info* si, uint32_t segment_version)) {
-  if(si->segment_version != segment_version) RAISE_ERROR("segment has type %u; expecting type %u", si->segment_version, segment_version);
+  if(si->segment_version != segment_version) RAISE_VERSION_ERROR("segment has type %u; expecting type %u", si->segment_version, segment_version);
   return NO_ERROR;
 }
 
@@ -449,7 +469,7 @@ wp_error* wp_segment_add_posting(wp_segment* s, const char* field, const char* w
   RELAY_ERROR(bump_stringpool(s, &success));
   RELAY_ERROR(bump_termhash(s, &success));
 
-  DEBUG("adding posting for %s:%s and doc %u", field, word, doc_id);
+  DEBUG("adding posting for %s:%s and doc %u with %u positions", field, word, doc_id, num_positions);
 
   postings_region* pr = MMAP_OBJ(s->postings, postings_region);
   stringmap* sh = MMAP_OBJ(s->stringmap, stringmap);
@@ -461,26 +481,38 @@ wp_error* wp_segment_add_posting(wp_segment* s, const char* field, const char* w
   RELAY_ERROR(stringmap_add(sh, sp, field, &t.field_s));
   RELAY_ERROR(stringmap_add(sh, sp, word, &t.word_s));
 
+  DEBUG("%s:%s maps to %u:%u", field, word, t.field_s, t.word_s);
+
   // find the offset of the next posting
+  posting_list_header* plh = termhash_get_val(th, t);
+  if(plh == NULL) {
+    RELAY_ERROR(termhash_put_val(th, t, &blank_plh));
+    plh = termhash_get_val(th, t);
+  }
+  DEBUG("posting list header for %s:%s is at %p", field, word, plh);
+
   posting po;
-  uint32_t next_offset = termhash_get_val(th, t);
-  if(next_offset == (uint32_t)-1) next_offset = OFFSET_NONE;
-  if(next_offset != OFFSET_NONE) { // TODO remove this check for speed once happy
+  uint32_t next_offset = plh->next_offset;
+
+  if(next_offset != OFFSET_NONE) { // TODO remove this check for speed once happy [PERFORMANCE]
     RELAY_ERROR(wp_segment_read_posting(s, next_offset, &po, 0));
     if(po.doc_id >= doc_id) RAISE_ERROR("cannot add a doc_id out of sorted order");
   }
 
   // write the entry to the postings region
   uint32_t entry_offset = pr->postings_head;
-  //DEBUG("entry will be at offset %u, prev offset is %u and next offset is %u", entry_offset, prev_offset, next_offset);
+  DEBUG("writing posting at offset %u. next offset is %u.", entry_offset, next_offset);
+
   po.doc_id = doc_id;
   po.next_offset = next_offset;
   po.num_positions = num_positions;
   RELAY_ERROR(write_posting(s, &po, positions)); // prev_docid is 0 for th
-  DEBUG("postings list head now at %u", pr->postings_head);
+  DEBUG("posting list head now at %u", pr->postings_head);
 
   // really finally, update the tail pointer so that readers can access this posting
-  RELAY_ERROR(termhash_put_val(th, t, entry_offset));
+  plh->count++;
+  plh->next_offset = entry_offset;
+  DEBUG("posting list header for %s:%s now reads count=%u offset=%u", field, word, plh->count, plh->next_offset);
 
   return NO_ERROR;
 }
@@ -537,22 +569,25 @@ wp_error* wp_segment_add_label(wp_segment* s, const char* label, docid_t doc_id)
 
   // find the previous and next label postings, between which we'll insert this
   // posting
-  uint32_t prev_offset = OFFSET_NONE;
-  uint32_t next_offset = termhash_get_val(th, t);
+  posting_list_header* plh = termhash_get_val(th, t);
+  if(plh == NULL) {
+    RELAY_ERROR(termhash_put_val(th, t, &blank_plh));
+    plh = termhash_get_val(th, t);
+  }
+
+  uint32_t next_offset = plh->next_offset;
   docid_t last_docid = DOCID_NONE;
+  uint32_t prev_offset = OFFSET_NONE;
 
-  if(next_offset == (uint32_t)-1) next_offset = OFFSET_NONE;
   DEBUG("start offset is %u (none is %u)", next_offset, OFFSET_NONE);
 
   while(next_offset != OFFSET_NONE) {
     label_posting* lp = wp_segment_label_posting_at(pr, next_offset);
 
-    if((last_docid != DOCID_NONE) && (lp->doc_id >= last_docid)) {
+    if((last_docid != DOCID_NONE) && (lp->doc_id >= last_docid))
       RAISE_ERROR("whistlepig index corruption! lp %u has docid %u but last docid at lp %u was %u", next_offset, lp->doc_id, prev_offset, last_docid);
-    }
-    else {
+    else
       last_docid = lp->doc_id;
-    }
 
     DEBUG("got doc id %u next_offset %u at offset %u (looking for doc id %u)", lp->doc_id, lp->next_offset, next_offset, doc_id);
     if(lp->doc_id == doc_id) {
@@ -567,18 +602,23 @@ wp_error* wp_segment_add_label(wp_segment* s, const char* label, docid_t doc_id)
   // find a space for the posting by first checking for a free postings in the
   // dead list. the dead list is the list stored under the sentinel term with
   // field 0 and word 0.
-  term dead_term = { .field_s = 0, .word_s = 0 };
+  posting_list_header* dead_plh = termhash_get_val(th, dead_term);
+  if(dead_plh == NULL) {
+    RELAY_ERROR(termhash_put_val(th, dead_term, &blank_plh));
+    dead_plh = termhash_get_val(th, t);
+  }
+
   uint32_t entry_offset;
-  uint32_t dead_offset = termhash_get_val(th, dead_term);
-  if(dead_offset == (uint32_t)-1) dead_offset = OFFSET_NONE;
+  uint32_t dead_offset = dead_plh->next_offset;
 
   if(dead_offset == OFFSET_NONE) { // make a new posting
     entry_offset = pr->postings_head;
   }
   else { // we'll use this one; remove it from the linked list
     DEBUG("offset from dead list is %u, using it for the new posting!", dead_offset);
-    entry_offset = dead_offset;
-    RELAY_ERROR(termhash_put_val(th, dead_term, wp_segment_label_posting_at(pr, dead_offset)->next_offset));
+    entry_offset = dead_plh->next_offset;
+    dead_plh->next_offset = wp_segment_label_posting_at(pr, dead_offset)->next_offset;
+    dead_plh->count--;
   }
 
   // finally, write the entry to the label postings region
@@ -588,11 +628,12 @@ wp_error* wp_segment_add_label(wp_segment* s, const char* label, docid_t doc_id)
   po->next_offset = next_offset;
 
   pr->postings_head += (uint32_t)sizeof(label_posting);
-  DEBUG("label postings list head now at %u", pr->postings_head);
+  DEBUG("label posting list head now at %u", pr->postings_head);
 
   // really finally, update either the previous offset or the tail pointer
   // for this label so that readers can access this posting
-  if(prev_offset == OFFSET_NONE) RELAY_ERROR(termhash_put_val(th, t, entry_offset));
+  plh->count++;
+  if(prev_offset == OFFSET_NONE) plh->next_offset = entry_offset;
   else wp_segment_label_posting_at(pr, prev_offset)->next_offset = entry_offset;
 
   return NO_ERROR;
@@ -615,13 +656,16 @@ wp_error* wp_segment_remove_label(wp_segment* s, const char* label, docid_t doc_
   t.word_s = stringmap_string_to_int(sh, sp, label); // will be -1 if not there
 
   // find the posting and the previous posting in the list, if any
-  uint32_t prev_offset = OFFSET_NONE;
-  uint32_t offset = termhash_get_val(th, t);
   docid_t last_docid = DOCID_NONE;
+  uint32_t prev_offset = OFFSET_NONE;
+  posting_list_header* plh = termhash_get_val(th, t);
+  if(plh == NULL) {
+    DEBUG("no such label %s", label);
+    return NO_ERROR;
+  }
 
-  if(offset == (uint32_t)-1) offset = OFFSET_NONE;
+  uint32_t offset = plh->next_offset;
   label_posting* lp = NULL;
-
   while(offset != OFFSET_NONE) {
     lp = wp_segment_label_posting_at(pr, offset);
 
@@ -646,17 +690,22 @@ wp_error* wp_segment_remove_label(wp_segment* s, const char* label, docid_t doc_
   }
 
   // we've found the posting; now remove it from the list
-  if(prev_offset == OFFSET_NONE) RELAY_ERROR(termhash_put_val(th, t, lp->next_offset));
+  if(prev_offset == OFFSET_NONE) plh->next_offset = lp->next_offset;
   else wp_segment_label_posting_at(pr, prev_offset)->next_offset = lp->next_offset;
+  plh->count--;
 
   // now add it to the dead list for later reclamation
-  term dead_term = { .field_s = 0, .word_s = 0 };
-  uint32_t dead_offset = termhash_get_val(th, dead_term);
-  if(dead_offset == (uint32_t)-1) dead_offset = OFFSET_NONE;
+  posting_list_header* dead_plh = termhash_get_val(th, dead_term);
+  if(dead_plh == NULL) {
+    RELAY_ERROR(termhash_put_val(th, dead_term, &blank_plh));
+    dead_plh = termhash_get_val(th, t);
+  }
 
-  lp->next_offset = dead_offset;
   DEBUG("adding dead label posting %u to head of deadlist with next_offset %u", offset, lp->next_offset);
-  RELAY_ERROR(termhash_put_val(th, dead_term, offset));
+
+  uint32_t dead_offset = dead_plh->next_offset;
+  lp->next_offset = dead_offset;
+  dead_plh->next_offset = offset;
 
   return NO_ERROR;
 }

data/ext/whistlepig/segment.h

@@ -150,4 +150,7 @@ wp_error* wp_segment_ensure_fit(wp_segment* seg, uint32_t postings_bytes, uint32
 // private: return the size on disk of a position array
 wp_error* wp_segment_sizeof_posarray(wp_segment* seg, uint32_t num_positions, pos_t* positions, uint32_t* size) RAISES_ERROR;
 
+// private: count the number of occurences of a particular term
+wp_error* wp_segment_count_term(wp_segment* seg, const char* field, const char* term, uint32_t* num_results);
+
 #endif

data/ext/whistlepig/snippeter.c

@@ -87,7 +87,7 @@ RAISING_STATIC(snippetize_from_lexer(wp_query* query, lexinfo* charpos, yyscan_t
 
   RARRAY_INIT(pword, words);
   while(yylex(*scanner) != TOK_DONE) {
-    pword pw = { strdup(yyget_text(*scanner)), charpos->start, charpos->end };
+    pword pw = { .token = strdup(yyget_text(*scanner)), .start = charpos->start, .end = charpos->end };
     RARRAY_ADD(pword, words, pw);
   }
 

data/ext/whistlepig/termhash.c

@@ -106,15 +106,15 @@ wp_error* termhash_bump_size(termhash *h) {
 
   // get pointers to the old locations
   term* oldkeys = TERMHASH_KEYS(h);
-  uint32_t* oldvals = TERMHASH_VALS(h);
+  posting_list_header* oldvals = TERMHASH_VALS(h);
 
   // set pointers to the new locations
   uint32_t* newflags = (uint32_t*)h->boundary;
   term* newkeys = (term*)(newflags + ((new_n_buckets >> 4) + 1));
-  uint32_t* newvals = (uint32_t*)(newkeys + new_n_buckets);
+  posting_list_header* newvals = (posting_list_header*)(newkeys + new_n_buckets);
 
   // move the vals and keys
-  memmove(newvals, oldvals, h->n_buckets * sizeof(uint32_t));
+  memmove(newvals, oldvals, h->n_buckets * sizeof(posting_list_header));
   memmove(newkeys, oldkeys, h->n_buckets * sizeof(term));
 
   // clear the new flags
@@ -124,8 +124,7 @@ wp_error* termhash_bump_size(termhash *h) {
   for (unsigned int j = 0; j != h->n_buckets; ++j) {
     if (iseither(flagbaks, j) == 0) {
       term key = newkeys[j];
-      uint32_t val;
-      val = newvals[j];
+      posting_list_header val = newvals[j];
       set_isdel_true(flagbaks, j);
       while (1) {
         uint32_t inc, k, i;
@@ -139,7 +138,7 @@ wp_error* termhash_bump_size(termhash *h) {
         set_isempty_false(newflags, i);
         if (i < h->n_buckets && iseither(flagbaks, i) == 0) {
           { term tmp = newkeys[i]; newkeys[i] = key; key = tmp; }
-          { uint32_t tmp = newvals[i]; newvals[i] = val; val = tmp; }
+          { posting_list_header tmp = newvals[i]; newvals[i] = val; val = tmp; }
           set_isdel_true(flagbaks, i);
         } else {
           newkeys[i] = key;
@@ -235,20 +234,20 @@ void termhash_del(termhash *h, uint32_t x) {
   }
 }
 
-uint32_t termhash_get_val(termhash* h, term t) {
-  uint32_t* vals = TERMHASH_VALS(h);
+posting_list_header* termhash_get_val(termhash* h, term t) {
+  posting_list_header* vals = TERMHASH_VALS(h);
   uint32_t idx = termhash_get(h, t);
-  if(idx == h->n_buckets) return (uint32_t)-1;
-  return vals[idx];
+  if(idx == h->n_buckets) return NULL;
+  return &vals[idx];
 }
 
-wp_error* termhash_put_val(termhash* h, term t, uint32_t val) {
+wp_error* termhash_put_val(termhash* h, term t, posting_list_header* val) {
   int status;
-  uint32_t* vals = TERMHASH_VALS(h);
+  posting_list_header* vals = TERMHASH_VALS(h);
   uint32_t loc = termhash_put(h, t, &status);
   DEBUG("put(%u,%u) has status %d and loc %u (error val is %u)", t.field_s, t.word_s, status, loc, h->n_buckets);
   if(status == -1) RAISE_ERROR("out of space in hash");
-  vals[loc] = val;
+  memcpy(&vals[loc], val, sizeof(posting_list_header));
   return NO_ERROR;
 }
 
@@ -257,22 +256,22 @@ int termhash_needs_bump(termhash* h) {
 }
 
 // returns the total size in bytes
-//   memory layout: termhash, then:
+//   memory layout: termhash struct, then:
 //   ((n_buckets >> 4) + 1) uint32_t's for the flags
 //   n_buckets terms for the keys
-//   n_buckets uint32_t's for the vals (offsets into postings lists)
+//   n_buckets posting_list_header for the vals (offsets into postings lists)
 static uint32_t size(uint32_t n_buckets) {
   uint32_t size = (uint32_t)sizeof(termhash) +
     (((n_buckets >> 4) + 1) * (uint32_t)sizeof(uint32_t)) +
     (n_buckets * (uint32_t)sizeof(term)) +
-    (n_buckets * (uint32_t)sizeof(uint32_t));
+    (n_buckets * (uint32_t)sizeof(posting_list_header));
 
   DEBUG("size of a termhash with %u buckets is %lu + %lu + %lu + %lu = %u",
     n_buckets,
     (long)sizeof(termhash),
     (long)(((n_buckets >> 4) + 1) * sizeof(uint32_t)),
     (long)(n_buckets * sizeof(term)),
-    (long)(n_buckets * sizeof(uint32_t)),
+    (long)(n_buckets * sizeof(posting_list_header)),
     size);
 
   return size;

data/ext/whistlepig/termhash.h

@@ -22,6 +22,18 @@ typedef struct term {
   uint32_t word_s;
 } term;
 
+typedef struct posting_list_header {
+  uint32_t count;
+  uint32_t next_offset;
+} posting_list_header;
+
+typedef struct block_header {
+  uint32_t max_docid;
+  uint32_t next_offset;
+  uint32_t block_start;
+  uint8_t data[];
+} block_header;
+
 #define INITIAL_N_BUCKETS_IDX 1
 
 typedef struct termhash {
@@ -31,12 +43,12 @@ typedef struct termhash {
   // in memory at this point
   // ((n_buckets >> 4) + 1) uint32_t's for the flags
   // n_buckets terms for the keys
-  // n_buckets uint32_t's for the vals (offsets into postings lists)
+  // n_buckets posting_list_header for the vals
 } termhash;
 
 #define TERMHASH_FLAGS(h) ((uint32_t*)(h)->boundary)
 #define TERMHASH_KEYS(h) ((term*)((uint32_t*)(h)->boundary + (((h)->n_buckets >> 4) + 1)))
-#define TERMHASH_VALS(h) ((uint32_t*)(TERMHASH_KEYS(h) + (h)->n_buckets))
+#define TERMHASH_VALS(h) ((posting_list_header*)(TERMHASH_KEYS(h) + (h)->n_buckets))
 
 // API methods
 
@@ -50,14 +62,14 @@ uint32_t termhash_get(termhash *h, term t);
 
 // public: get an int given a term. returns (uint32_t)-1 if the term is not in
 // the hash.
-uint32_t termhash_get_val(termhash* h, term t); // convenience
+posting_list_header* termhash_get_val(termhash* h, term t); // convenience
 
 // private: khash-style setter: insert a term into the hash. see the code
 // for details on what all the return values mean.
 uint32_t termhash_put(termhash* h, term t, int *ret); // khash-style
 
 // public: adds a term to the hash with the given value
-wp_error* termhash_put_val(termhash* h, term t, uint32_t val) RAISES_ERROR; // convenience
+wp_error* termhash_put_val(termhash* h, term t, posting_list_header* val) RAISES_ERROR; // convenience
 
 // public: returns the byte size of the termhash
 uint32_t termhash_size(termhash* h);

data/ext/whistlepig/whistlepig.c

@@ -7,7 +7,9 @@ static VALUE c_index;
 static VALUE c_entry;
 static VALUE c_query;
 static VALUE c_error;
-static VALUE c_parseerror;
+static VALUE c_parse_error;
+static VALUE c_sys_error;
+static VALUE c_version_error;
 
 static void index_free(wp_index* index) {
   wp_error* e = wp_index_free(index);
@@ -20,7 +22,12 @@ static void index_free(wp_index* index) {
 
 #define RAISE_IF_NECESSARY(e) do { \
   if(e != NULL) { \
-    VALUE exc = rb_exc_new2(c_error, e->msg); \
+    VALUE exc; \
+    switch(e->type) { \
+      case WP_ERROR_TYPE_SYSTEM: exc = rb_exc_new2(c_sys_error, e->msg); break; \
+      case WP_ERROR_TYPE_VERSION: exc = rb_exc_new2(c_version_error, e->msg); break; \
+      default: exc = rb_exc_new2(c_error, e->msg); break; \
+    } \
     wp_error_free(e); \
     rb_exc_raise(exc); \
   }  \
@@ -331,7 +338,7 @@ static VALUE query_new(VALUE class, VALUE default_field, VALUE string) {
   wp_query* query;
   wp_error* e = wp_query_parse(RSTRING_PTR(string), RSTRING_PTR(default_field), &query);
   if(e != NULL) {
-    VALUE exc = rb_exc_new2(c_parseerror, e->msg);
+    VALUE exc = rb_exc_new2(c_parse_error, e->msg);
     wp_error_free(e);
     rb_exc_raise(exc);
   }
@@ -614,5 +621,7 @@ void Init_whistlepig() {
   rb_define_attr(c_query, "query", 1, 0);
 
   c_error = rb_define_class_under(m_whistlepig, "Error", rb_eStandardError);
-  c_parseerror = rb_define_class_under(m_whistlepig, "ParseError", rb_eStandardError);
+  c_parse_error = rb_define_class_under(m_whistlepig, "ParseError", c_error);
+  c_sys_error = rb_define_class_under(m_whistlepig, "SystemError", c_error);
+  c_version_error = rb_define_class_under(m_whistlepig, "VersionError", c_error);
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: whistlepig
 version: !ruby/object:Gem::Version
-  version: 0.11.2
+  version: '0.12'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-19 00:00:00.000000000 Z
+date: 2012-06-09 00:00:00.000000000 Z
 dependencies: []
 description: Whistlepig is a minimalist realtime full-text search index. Its goal
   is to be as small and minimally-featured as possible, while still remaining useful,