whistlepig 0.9.1 → 0.10

data/ext/whistlepig/stringmap.c

@@ -33,20 +33,15 @@ static inline int string_equals(const char* a, const char* b) {
   return strcmp(a, b) == 0;
 }
 
-// set flags, keys and vals to correct locations based on h->n_buckets
-void stringmap_setup(stringmap* h, stringpool* p) {
-  h->pool = p;
-  h->flags = (uint32_t*)h->boundary;
-  h->keys = (uint32_t*)((uint32_t*)h->boundary + ((h->n_buckets >> 4) + 1));
-}
+#define STRINGMAP_FLAGS(h) ((uint32_t*)(h)->boundary)
+#define STRINGMAP_KEYS(h) ((uint32_t*)((uint32_t*)(h)->boundary + (((h)->n_buckets >> 4) + 1)))
 
-void stringmap_init(stringmap* h, stringpool* p) {
+void stringmap_init(stringmap* h) {
   h->n_buckets_idx = INITIAL_N_BUCKETS_IDX;
   h->n_buckets = prime_list[h->n_buckets_idx];
   h->upper_bound = (uint32_t)(h->n_buckets * HASH_UPPER + 0.5);
   h->size = h->n_occupied = 0;
-  stringmap_setup(h, p);
-  memset(h->flags, 0xaa, ((h->n_buckets>>4) + 1) * sizeof(uint32_t));
+  memset(STRINGMAP_FLAGS(h), 0xaa, ((h->n_buckets>>4) + 1) * sizeof(uint32_t));
 }
 
 /*
@@ -66,22 +61,25 @@ static void kh_clear_##name(kh_##name##_t *h) {
 }
 */
 
-uint32_t stringmap_get(stringmap *h, const char* key) {
+uint32_t stringmap_get(stringmap *h, stringpool* pool, const char* key) {
+  uint32_t* flags = STRINGMAP_FLAGS(h);
+  uint32_t* keys = STRINGMAP_KEYS(h);
+
   if(h->n_buckets) {
     uint32_t inc, k, i, last;
     k = string_hash(key); i = k % h->n_buckets;
     inc = 1 + k % (h->n_buckets - 1); last = i;
-    while (!isempty(h->flags, i) && (isdel(h->flags, i) || !string_equals(stringpool_lookup(h->pool, h->keys[i]), key))) {
+    while (!isempty(flags, i) && (isdel(flags, i) || !string_equals(stringpool_lookup(pool, keys[i]), key))) {
       if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets;
       else i += inc;
       if (i == last) return h->n_buckets;
     }
-    return iseither(h->flags, i)? h->n_buckets : i;
+    return iseither(flags, i)? h->n_buckets : i;
   }
   else return 0;
 }
 
-wp_error* stringmap_bump_size(stringmap *h) {
+wp_error* stringmap_bump_size(stringmap *h, stringpool* pool) {
   DEBUG("bumping size for string hash at %p with size %u and boundary %p", h, stringmap_size(h), h->boundary);
 
   if(h->n_buckets_idx >= (HASH_PRIME_SIZE - 1)) RAISE_ERROR("stringmap can't be this big");
@@ -89,51 +87,54 @@ wp_error* stringmap_bump_size(stringmap *h) {
   h->n_buckets_idx++;
   uint32_t new_n_buckets = prime_list[h->n_buckets_idx];
 
-  // first make a backup of the oldflags
-  size_t oldflagsize = ((h->n_buckets >> 4) + 1) * sizeof(uint32_t);
-  uint32_t* oldflags = malloc(oldflagsize);
-  memcpy(oldflags, h->flags, oldflagsize);
+  // get pointers to the old locations
+  uint32_t* oldkeys = STRINGMAP_KEYS(h);
+  uint32_t* oldflags = STRINGMAP_FLAGS(h);
 
-  // keep pointers to the old locations
-  uint32_t* oldkeys = h->keys;
+  // make a backup of the old flags in a separate memory region
+  size_t flagbaksize = ((h->n_buckets >> 4) + 1) * sizeof(uint32_t);
+  uint32_t* flagbaks = malloc(flagbaksize);
+  memcpy(flagbaks, oldflags, flagbaksize);
 
-  // set pointers to the new locations
-  h->keys = (uint32_t*)((uint32_t*)h->boundary + ((new_n_buckets >> 4) + 1));
+  // get a pointer pointers to the new locations
+  //h->keys = (uint32_t*)((uint32_t*)h->boundary + ((new_n_buckets >> 4) + 1));
+  uint32_t* newflags = (uint32_t*)h->boundary; // unchanged, actually
+  uint32_t* newkeys = (uint32_t*)((uint32_t*)h->boundary + ((new_n_buckets >> 4) + 1));
 
   // move the keys
-  memmove(h->keys, oldkeys, h->n_buckets * sizeof(uint32_t));
+  memmove(newkeys, oldkeys, h->n_buckets * sizeof(uint32_t));
 
   // clear the new flags
-  memset(h->flags, 0xaa, ((new_n_buckets>>4) + 1) * sizeof(uint32_t));
+  memset(STRINGMAP_FLAGS(h), 0xaa, ((new_n_buckets>>4) + 1) * sizeof(uint32_t));
 
   // do the complicated stuff from khash.h
   for (unsigned int j = 0; j != h->n_buckets; ++j) {
-    if (iseither(oldflags, j) == 0) {
-      uint32_t key = h->keys[j];
-      set_isdel_true(oldflags, j);
+    if (iseither(flagbaks, j) == 0) {
+      uint32_t key = newkeys[j];
+      set_isdel_true(flagbaks, j);
       while (1) {
         uint32_t inc, k, i;
-        k = string_hash(stringpool_lookup(h->pool, key));
+        k = string_hash(stringpool_lookup(pool, key));
         i = k % new_n_buckets;
         inc = 1 + k % (new_n_buckets - 1);
-        while (!isempty(h->flags, i)) {
+        while (!isempty(newflags, i)) {
           if (i + inc >= new_n_buckets) i = i + inc - new_n_buckets;
           else i += inc;
         }
-        set_isempty_false(h->flags, i);
-        if (i < h->n_buckets && iseither(oldflags, i) == 0) {
-          { uint32_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; }
-          set_isdel_true(oldflags, i);
+        set_isempty_false(newflags, i);
+        if (i < h->n_buckets && iseither(flagbaks, i) == 0) {
+          { uint32_t tmp = newkeys[i]; newkeys[i] = key; key = tmp; }
+          set_isdel_true(flagbaks, i);
         } else {
-          h->keys[i] = key;
+          newkeys[i] = key;
           break;
         }
       }
     }
   }
 
-  free(oldflags);
-  h->n_buckets = new_n_buckets;
+  free(flagbaks);
+  h->n_buckets = new_n_buckets; // STRINGMAP_KEYS now works
   h->n_occupied = h->size;
   h->upper_bound = (uint32_t)(h->n_buckets * HASH_UPPER + 0.5);
 
@@ -144,8 +145,10 @@ wp_error* stringmap_bump_size(stringmap *h) {
   return NO_ERROR;
 }
 
-uint32_t stringmap_put(stringmap *h, const char* key, int *ret) {
+uint32_t stringmap_put(stringmap *h, stringpool* pool, const char* key, int *ret) {
   uint32_t x;
+  uint32_t* flags = STRINGMAP_FLAGS(h);
+  uint32_t* keys = STRINGMAP_KEYS(h);
 
   {
 #ifdef DEBUGOUTPUT
@@ -154,27 +157,27 @@ int num_loops = 0;
     uint32_t inc, k, i, site, last;
     x = site = h->n_buckets; k = string_hash(key); i = k % h->n_buckets;
     //DEBUG("asked to hash '%s'. initial hash is %u => %u and n_occupied is %u", key, k, i, h->n_occupied);
-    if (isempty(h->flags, i)) x = i;
+    if (isempty(flags, i)) x = i;
     else {
       inc = 1 + k % (h->n_buckets - 1); last = i;
-      while (!isempty(h->flags, i) && (isdel(h->flags, i) || !string_equals(stringpool_lookup(h->pool, h->keys[i]), key))) {
+      while (!isempty(flags, i) && (isdel(flags, i) || !string_equals(stringpool_lookup(pool, keys[i]), key))) {
 #ifdef DEBUGOUTPUT
 num_loops++;
 #endif
-        if (isdel(h->flags, i)) site = i;
+        if (isdel(flags, i)) site = i;
         if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets;
         else i += inc;
         if (i == last) { x = site; break; }
       }
       if ((x == h->n_buckets) && (i == last)) { // out of space
-        if(!string_equals(stringpool_lookup(h->pool, h->keys[i]), key)) {
+        if(!string_equals(stringpool_lookup(pool, keys[i]), key)) {
           DEBUG("out of space!");
           *ret = -1;
           return x;
         }
       }
       if (x == h->n_buckets) { // didn't find it on the first try
-        if (isempty(h->flags, i) && site != h->n_buckets) x = site;
+        if (isempty(flags, i) && site != h->n_buckets) x = site;
         else x = i;
       }
     }
@@ -185,15 +188,15 @@ num_loops++;
   //DEBUG("for pos %u, isempty? %d and isdel %d", x, isempty(h->flags, x), isdel(h->flags, x));
 
   uint32_t idx;
-  if(isempty(h->flags, x) || isdel(h->flags, x)) {
-    idx = stringpool_add(h->pool, key);
+  if(isempty(flags, x) || isdel(flags, x)) {
+    idx = stringpool_add(pool, key);
     if(idx == (uint32_t)-1) {
       *ret = -2;
       return x;
     }
-    if (isempty(h->flags, x)) ++h->n_occupied;
-    h->keys[x] = idx;
-    set_isboth_false(h->flags, x);
+    if (isempty(flags, x)) ++h->n_occupied;
+    keys[x] = idx;
+    set_isboth_false(flags, x);
     ++h->size;
     *ret = 1;
   }
@@ -203,8 +206,9 @@ num_loops++;
 }
 
 void stringmap_del(stringmap *h, uint32_t x) {
-  if (x != h->n_buckets && !iseither(h->flags, x)) {
-    set_isdel_true(h->flags, x);
+  uint32_t* flags = STRINGMAP_FLAGS(h);
+  if (x != h->n_buckets && !iseither(flags, x)) {
+    set_isdel_true(flags, x);
     --h->size;
   }
 }
@@ -255,24 +259,25 @@ uint32_t stringmap_next_size(stringmap* h) {
   return size(prime_list[next_idx]);
 }
 
-const char* stringmap_int_to_string(stringmap* h, uint32_t i) {
-  return stringpool_lookup(h->pool, i);
+const char* stringmap_int_to_string(stringmap* h, stringpool* p, uint32_t i) {
+  (void)h;
+  return stringpool_lookup(p, i);
 }
 
 // returns -1 if not found
-uint32_t stringmap_string_to_int(stringmap* h, const char* s) {
-  uint32_t idx = stringmap_get(h, s);
+uint32_t stringmap_string_to_int(stringmap* h, stringpool* pool, const char* s) {
+  uint32_t idx = stringmap_get(h, pool, s);
   if(idx == h->n_buckets) return (uint32_t)-1; // not there
-  return h->keys[idx];
+  return STRINGMAP_KEYS(h)[idx];
 }
 
-wp_error* stringmap_add(stringmap *h, const char* s, uint32_t* id) {
+wp_error* stringmap_add(stringmap *h, stringpool* pool, const char* s, uint32_t* id) {
   int status;
-  uint32_t idx = stringmap_put(h, s, &status);
+  uint32_t idx = stringmap_put(h, pool, s, &status);
   if(status == -1) RAISE_ERROR("out of space in hash put");
   if(status == -2) RAISE_ERROR("out of space in pool put");
 
-  *id = h->keys[idx];
+  *id = STRINGMAP_KEYS(h)[idx];
 
   return NO_ERROR;
 }

data/ext/whistlepig/stringmap.h

@@ -10,9 +10,8 @@
 // and stringpool, it uses a slightly funny API that never allocates memory,
 // but instead operates on pointers to preallocated blocks of memory.
 //
-// uses a stringpool internally to do the int->string mapping. so if you're so
-// you shouldn't have to interact with the stringpool directly; you can just
-// use this object.
+// uses a stringpool internally to do the int->string mapping.  you shouldn't
+// have to interact with the stringpool directly; you can just use this object.
 //
 // like termhash and pool, has a slightly funny API that is designed to work on
 // a pre-allocated chunk of memory rather than allocate any of its own.
@@ -36,9 +35,6 @@
 typedef struct stringmap {
   uint8_t n_buckets_idx;
   uint32_t n_buckets, size, n_occupied, upper_bound;
-  uint32_t *flags;
-  uint32_t *keys;
-  stringpool* pool;
   uint8_t boundary[];
   // in memory at this point
   // ((n_buckets >> 4) + 1) uint32_t's for the flags
@@ -48,21 +44,18 @@ typedef struct stringmap {
 // API methods
 
 // public: write a new stringmap to memory
-void stringmap_init(stringmap* h, stringpool* p);
-
-// public: set up an existing stringmap in memory
-void stringmap_setup(stringmap* h, stringpool* p);
+void stringmap_init(stringmap* h);
 
 // public: add a string. sets id to its id. dupes are fine; will just set the
 // id correctly.
-wp_error* stringmap_add(stringmap *h, const char* s, uint32_t* id) RAISES_ERROR;
+wp_error* stringmap_add(stringmap *h, stringpool* p, const char* s, uint32_t* id) RAISES_ERROR;
 
 // public: get the int value given a string. returns (uint32_t)-1 if not found.
-uint32_t stringmap_string_to_int(stringmap* h, const char* s);
+uint32_t stringmap_string_to_int(stringmap* h, stringpool* pool, const char* s);
 
 // public: get the string value given an int. returns corrupt data if the int
 // is invalid.
-const char* stringmap_int_to_string(stringmap* h, uint32_t i);
+const char* stringmap_int_to_string(stringmap* h, stringpool* p, uint32_t i);
 
 // public: returns the byte size of the stringmap
 uint32_t stringmap_size(stringmap* h);
@@ -77,6 +70,6 @@ uint32_t stringmap_next_size(stringmap* h);
 int stringmap_needs_bump(stringmap* h);
 
 // public: increases the size of the stringmap
-wp_error* stringmap_bump_size(stringmap *h) RAISES_ERROR;
+wp_error* stringmap_bump_size(stringmap *h, stringpool* pool) RAISES_ERROR;
 
 #endif

data/ext/whistlepig/termhash.c

@@ -35,23 +35,11 @@ void termhash_init(termhash* h) {
   h->n_buckets = prime_list[h->n_buckets_idx];
   h->upper_bound = (uint32_t)(h->n_buckets * HASH_UPPER + 0.5);
   h->size = h->n_occupied = 0;
-  termhash_setup(h);
-  memset(h->flags, 0xaa, ((h->n_buckets>>4) + 1) * sizeof(uint32_t));
+  memset(TERMHASH_FLAGS(h), 0xaa, ((h->n_buckets>>4) + 1) * sizeof(uint32_t));
 }
 
 #define OFFSET(a, b) (long)((uint8_t*)a - (uint8_t*)b)
 // set flags, keys and vals to correct locations based on h->n_buckets
-void termhash_setup(termhash* h) {
-  DEBUG("term hash ranges from %p to %p (size %u)", h, (char*)h + termhash_size(h), termhash_size(h));
-  DEBUG("boundary is at %p (+%ld)", h->boundary, OFFSET(h->boundary, h));
-  h->flags = (uint32_t*)h->boundary;
-  h->keys = (term*)((uint32_t*)h->boundary + ((h->n_buckets >> 4) + 1));
-  h->vals = (uint32_t*)((term*)h->keys + h->n_buckets);
-  DEBUG("flags are at %p (+%ld)", h->flags, OFFSET(h->flags, h->boundary));
-  DEBUG(" keys are at %p (+%ld)", h->keys, OFFSET(h->keys, h->boundary));
-  DEBUG(" vals are at %p (+%ld)", h->vals, OFFSET(h->vals, h->boundary));
-}
-
 /*
 static void termhash_dump(termhash* h) {
   for(uint32_t i = 0; i < h->n_buckets; i++) {
@@ -83,89 +71,94 @@ static void kh_clear_##name(kh_##name##_t *h) {
 */
 
 uint32_t termhash_get(termhash *h, term key) {
+  uint32_t* flags = TERMHASH_FLAGS(h);
+  term* keys = TERMHASH_KEYS(h);
+
   if(h->n_buckets) {
     uint32_t inc, k, i, last;
     k = hash_term(key); i = k % h->n_buckets;
     inc = 1 + k % (h->n_buckets - 1); last = i;
-    while (!isempty(h->flags, i) && (isdel(h->flags, i) || !term_equals(h->keys[i], key))) {
+    while (!isempty(flags, i) && (isdel(flags, i) || !term_equals(keys[i], key))) {
       if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets;
       else i += inc;
       if (i == last) return h->n_buckets;
     }
-    return iseither(h->flags, i)? h->n_buckets : i;
+    return iseither(flags, i)? h->n_buckets : i;
   }
   else return 0;
 }
 
 wp_error* termhash_bump_size(termhash *h) {
   DEBUG("bumping size for term hash at %p with size %u and boundary %p (+%ld)", h, termhash_size(h), h->boundary, (long)((uint8_t*)h->boundary - (uint8_t*)h));
-  DEBUG("flags are at %p (+%ld)", h->flags, OFFSET(h->flags, h->boundary));
-  DEBUG(" keys are at %p (+%ld)", h->keys, OFFSET(h->keys, h->boundary));
-  DEBUG(" vals are at %p (+%ld)", h->vals, OFFSET(h->vals, h->boundary));
+  DEBUG("flags are at %p (+%ld)", TERMHASH_FLAGS(h), OFFSET(TERMHASH_FLAGS(h), h->boundary));
+  DEBUG(" keys are at %p (+%ld)", TERMHASH_KEYS(h), OFFSET(TERMHASH_KEYS(h), h->boundary));
+  DEBUG(" vals are at %p (+%ld)", TERMHASH_VALS(h), OFFSET(TERMHASH_VALS(h), h->boundary));
+
+  if(h->n_buckets_idx >= (HASH_PRIME_SIZE - 1)) RAISE_ERROR("termhash can't be this big");
 
   h->n_buckets_idx++;
-  if(h->n_buckets_idx > HASH_PRIME_SIZE) exit(1); // die horribly TODO fixme
   uint32_t new_n_buckets = prime_list[h->n_buckets_idx];
 
-  // first make a backup of the oldflags
-  size_t oldflagsize = ((h->n_buckets >> 4) + 1) * sizeof(uint32_t);
-  uint32_t* oldflags = malloc(oldflagsize);
-  memcpy(oldflags, h->flags, oldflagsize);
+  // first make a backup of the old flags in a separate memory region
+  size_t flagbaksize = ((h->n_buckets >> 4) + 1) * sizeof(uint32_t);
+  uint32_t* flagbaks = malloc(flagbaksize);
+  memcpy(flagbaks, TERMHASH_FLAGS(h), flagbaksize);
 
-  // keep pointers to the old locations
-  term* oldkeys = h->keys;
-  uint32_t* oldvals = h->vals;
+  // get pointers to the old locations
+  term* oldkeys = TERMHASH_KEYS(h);
+  uint32_t* oldvals = TERMHASH_VALS(h);
 
   // set pointers to the new locations
-  h->keys = (term*)((uint32_t*)h->boundary + ((new_n_buckets >> 4) + 1));
-  h->vals = (uint32_t*)((term*)h->keys + new_n_buckets);
+  uint32_t* newflags = (uint32_t*)h->boundary;
+  term* newkeys = (term*)(newflags + ((new_n_buckets >> 4) + 1));
+  uint32_t* newvals = (uint32_t*)(newkeys + new_n_buckets);
 
   // move the vals and keys
-  memmove(h->vals, oldvals, h->n_buckets * sizeof(uint32_t));
-  memmove(h->keys, oldkeys, h->n_buckets * sizeof(term));
+  memmove(newvals, oldvals, h->n_buckets * sizeof(uint32_t));
+  memmove(newkeys, oldkeys, h->n_buckets * sizeof(term));
 
   // clear the new flags
-  memset(h->flags, 0xaa, ((new_n_buckets>>4) + 1) * sizeof(uint32_t));
+  memset(newflags, 0xaa, ((new_n_buckets>>4) + 1) * sizeof(uint32_t));
 
   // do the complicated stuff from khash.h
   for (unsigned int j = 0; j != h->n_buckets; ++j) {
-    if (iseither(oldflags, j) == 0) {
-      term key = h->keys[j];
+    if (iseither(flagbaks, j) == 0) {
+      term key = newkeys[j];
       uint32_t val;
-      val = h->vals[j];
-      set_isdel_true(oldflags, j);
+      val = newvals[j];
+      set_isdel_true(flagbaks, j);
       while (1) {
         uint32_t inc, k, i;
         k = hash_term(key);
         i = k % new_n_buckets;
         inc = 1 + k % (new_n_buckets - 1);
-        while (!isempty(h->flags, i)) {
+        while (!isempty(newflags, i)) {
           if (i + inc >= new_n_buckets) i = i + inc - new_n_buckets;
           else i += inc;
         }
-        set_isempty_false(h->flags, i);
-        if (i < h->n_buckets && iseither(oldflags, i) == 0) {
-          { term tmp = h->keys[i]; h->keys[i] = key; key = tmp; }
-          { uint32_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; }
-          set_isdel_true(oldflags, i);
+        set_isempty_false(newflags, i);
+        if (i < h->n_buckets && iseither(flagbaks, i) == 0) {
+          { term tmp = newkeys[i]; newkeys[i] = key; key = tmp; }
+          { uint32_t tmp = newvals[i]; newvals[i] = val; val = tmp; }
+          set_isdel_true(flagbaks, i);
         } else {
-          h->keys[i] = key;
-          h->vals[i] = val;
+          newkeys[i] = key;
+          newvals[i] = val;
           break;
         }
       }
     }
   }
 
-  free(oldflags);
+  free(flagbaks);
   h->n_buckets = new_n_buckets;
   h->n_occupied = h->size;
   h->upper_bound = (uint32_t)(h->n_buckets * HASH_UPPER + 0.5);
 
   DEBUG("after bump, term hash at %p has size %u and boundary %p (+%ld)", h, termhash_size(h), h->boundary, (long)((uint8_t*)h->boundary - (uint8_t*)h));
-  DEBUG("flags are at %p (+%ld)", h->flags, (long)((uint8_t*)h->flags - (uint8_t*)h->boundary));
-  DEBUG(" keys are at %p (+%ld)", h->keys, (long)((uint8_t*)h->keys - (uint8_t*)h->boundary));
-  DEBUG(" vals are at %p (+%ld)", h->vals, (long)((uint8_t*)h->vals - (uint8_t*)h->boundary));
+  DEBUG("flags are at %p (+%ld)", TERMHASH_FLAGS(h), OFFSET(TERMHASH_FLAGS(h), h->boundary));
+  DEBUG(" keys are at %p (+%ld)", TERMHASH_KEYS(h), OFFSET(TERMHASH_KEYS(h), h->boundary));
+  DEBUG(" vals are at %p (+%ld)", TERMHASH_VALS(h), OFFSET(TERMHASH_VALS(h), h->boundary));
 
 #ifdef DEBUGOUTPUT
 //DEBUG("and now i look like this:");
@@ -177,6 +170,8 @@ wp_error* termhash_bump_size(termhash *h) {
 
 uint32_t termhash_put(termhash *h, term key, int *ret) {
   uint32_t x;
+  uint32_t* flags = TERMHASH_FLAGS(h);
+  term* keys = TERMHASH_KEYS(h);
 
   {
 #ifdef DEBUGOUTPUT
@@ -185,40 +180,40 @@ int num_loops = 0;
     uint32_t inc, k, i, site, last;
     x = site = h->n_buckets; k = hash_term(key); i = k % h->n_buckets;
     DEBUG("initial hash is %u", k);
-    if (isempty(h->flags, i)) x = i;
+    if (isempty(flags, i)) x = i;
     else {
       inc = 1 + k % (h->n_buckets - 1); last = i;
-      while (!isempty(h->flags, i) && (isdel(h->flags, i) || !term_equals(h->keys[i], key))) {
+      while (!isempty(flags, i) && (isdel(flags, i) || !term_equals(keys[i], key))) {
 #ifdef DEBUGOUTPUT
 num_loops++;
 #endif
-        if (isdel(h->flags, i)) site = i;
+        if (isdel(flags, i)) site = i;
         if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets;
         else i += inc;
         if (i == last) { x = site; break; }
       }
       if ((x == h->n_buckets) && (i == last)) { // out of space
-        if(!term_equals(h->keys[i], key)) {
+        if(!term_equals(keys[i], key)) {
           *ret = -1;
           return x;
         }
       }
       if (x == h->n_buckets) { // didn't find it on the first try
-        if (isempty(h->flags, i) && site != h->n_buckets) x = site;
+        if (isempty(flags, i) && site != h->n_buckets) x = site;
         else x = i;
       }
     }
     DEBUG("looped %u times to put", num_loops);
     //DEBUG("x is %u, site is %u, n_buckets is %u", x, site, h->n_buckets);
   }
-  if (isempty(h->flags, x)) {
-    h->keys[x] = key;
-    set_isboth_false(h->flags, x);
+  if (isempty(flags, x)) {
+    keys[x] = key;
+    set_isboth_false(flags, x);
     ++h->size; ++h->n_occupied;
     *ret = 1;
-  } else if (isdel(h->flags, x)) {
-    h->keys[x] = key;
-    set_isboth_false(h->flags, x);
+  } else if (isdel(flags, x)) {
+    keys[x] = key;
+    set_isboth_false(flags, x);
     ++h->size;
     *ret = 2;
   }
@@ -233,24 +228,27 @@ num_loops++;
 }
 
 void termhash_del(termhash *h, uint32_t x) {
-  if (x != h->n_buckets && !iseither(h->flags, x)) {
-    set_isdel_true(h->flags, x);
+  uint32_t* flags = TERMHASH_FLAGS(h);
+  if (x != h->n_buckets && !iseither(flags, x)) {
+    set_isdel_true(flags, x);
     --h->size;
   }
 }
 
 uint32_t termhash_get_val(termhash* h, term t) {
+  uint32_t* vals = TERMHASH_VALS(h);
   uint32_t idx = termhash_get(h, t);
   if(idx == h->n_buckets) return (uint32_t)-1;
-  return h->vals[idx];
+  return vals[idx];
 }
 
 wp_error* termhash_put_val(termhash* h, term t, uint32_t val) {
   int status;
+  uint32_t* vals = TERMHASH_VALS(h);
   uint32_t loc = termhash_put(h, t, &status);
   DEBUG("put(%u,%u) has status %d and loc %u (error val is %u)", t.field_s, t.word_s, status, loc, h->n_buckets);
   if(status == -1) RAISE_ERROR("out of space in hash");
-  h->vals[loc] = val;
+  vals[loc] = val;
   return NO_ERROR;
 }
 

data/ext/whistlepig/termhash.h

@@ -27,9 +27,6 @@ typedef struct term {
 typedef struct termhash {
   uint8_t n_buckets_idx;
   uint32_t n_buckets, size, n_occupied, upper_bound;
-  uint32_t *flags;
-  term *keys;
-  uint32_t *vals;
   uint8_t boundary[];
   // in memory at this point
   // ((n_buckets >> 4) + 1) uint32_t's for the flags
@@ -37,14 +34,15 @@ typedef struct termhash {
   // n_buckets uint32_t's for the vals (offsets into postings lists)
 } termhash;
 
+#define TERMHASH_FLAGS(h) ((uint32_t*)(h)->boundary)
+#define TERMHASH_KEYS(h) ((term*)((uint32_t*)(h)->boundary + (((h)->n_buckets >> 4) + 1)))
+#define TERMHASH_VALS(h) ((uint32_t*)(TERMHASH_KEYS(h) + (h)->n_buckets))
+
 // API methods
 
 // public: make a new termhash
 void termhash_init(termhash* h);  // makes a new one
 
-// public: set up an existing termhash
-void termhash_setup(termhash* h); // inits one from disk
-
 // private: khash-style getter: returns the slot id, if any, given a term key.
 // you can then look this up within the vals array yourself. returns
 // h->n_buckets if the term is not in the hash.

data/ext/whistlepig/whistlepig.c

@@ -143,7 +143,11 @@ static VALUE index_delete(VALUE class, VALUE v_pathname_base) {
 static VALUE index_size(VALUE self) {
   wp_index* index;
   Data_Get_Struct(self, wp_index, index);
-  return INT2NUM(wp_index_num_docs(index));
+
+  uint64_t num_docs;
+  wp_error* e = wp_index_num_docs(index, &num_docs);
+  RAISE_IF_NECESSARY(e);
+  return INT2NUM(num_docs);
 }
 
 static VALUE index_init(VALUE self, VALUE v_pathname_base) {

data/ext/whistlepig/whistlepig.h

@@ -10,6 +10,7 @@
 #include "index.h"
 #include "query.h"
 #include "query-parser.h"
+#include "lock.h"
 #include "error.h"
 
 // see comments in index.c