whistlepig 0.7 → 0.8

data/README

@@ -8,8 +8,8 @@ the frills, Whistlepig may be for you.
 Whistlepig is written in ANSI C99. It currently provides a C API and Ruby
 bindings.
 
-Latest version: 0.7, released 2011-07-19.
-        Status: alpha
+Latest version: 0.8, released 2012-03-13.
+        Status: beta
           News: http://all-thing.net/label/whistlepig/
       Homepage: http://masanjin.net/whistlepig/
    Bug reports: http://github.com/wmorgan/whistlepig/issues

data/ext/whistlepig/dump.c

@@ -0,0 +1,65 @@
+#include <stdio.h>
+#include "whistlepig.h"
+
+#define isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
+#define isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
+#define KEY(h, i) &(h->pool[h->keys[i]])
+
+RAISING_STATIC(dump_posting_list(wp_segment* s, uint32_t offset)) {
+  posting po;
+
+  while(offset != OFFSET_NONE) {
+    RELAY_ERROR(wp_segment_read_posting(s, offset, &po, 1));
+
+    printf("  @%u doc %u:", offset, po.doc_id);
+    for(uint32_t i = 0; i < po.num_positions; i++) {
+      printf(" %d", po.positions[i]);
+    }
+    printf("\n");
+
+    offset = po.next_offset;
+    free(po.positions);
+  }
+
+  return NO_ERROR;
+}
+
+RAISING_STATIC(dump(wp_segment* segment)) {
+  termhash* th = MMAP_OBJ(segment->termhash, termhash);
+  stringmap* sh = MMAP_OBJ(segment->stringmap, stringmap);
+
+  for(uint32_t i = 0; i < th->n_buckets; i++) {
+    if(isempty(th->flags, i)); // do nothing
+    else if(isdel(th->flags, i)) printf("%u: [deleted]", i);
+    else {
+      term t = th->keys[i];
+      const char* field = stringmap_int_to_string(sh, t.field_s);
+      const char* word = stringmap_int_to_string(sh, t.word_s);
+      printf("%u: %s:'%s'\n", i, field, word);
+      RELAY_ERROR(dump_posting_list(segment, th->vals[i]));
+    }
+  }
+
+  return NO_ERROR;
+}
+
+int main(int argc, char* argv[]) {
+  if(argc != 2) {
+    fprintf(stderr, "Usage: %s <segment filename>\n", argv[0]);
+    return -1;
+  }
+
+  wp_index* index;
+  DIE_IF_ERROR(wp_index_load(&index, argv[1]));
+  DIE_IF_ERROR(wp_index_dumpinfo(index, stdout));
+
+  for(int i = 0; i < index->num_segments; i++) {
+    printf("\nsegment %d details:\n", i);
+    DIE_IF_ERROR(dump(&index->segments[i]));
+  }
+
+  DIE_IF_ERROR(wp_index_unload(index));
+
+  return 0;
+}
+

data/ext/whistlepig/extconf.rb

@@ -3,4 +3,4 @@ require 'mkmf'
 $CFLAGS = "-g -O3 -std=c99 $(cflags) -D_ANSI_SOURCE"
 
 create_header
-create_makefile "whistlepigc"
+create_makefile "whistlepig"

data/ext/whistlepig/query.c

@@ -12,7 +12,17 @@ static wp_query* wp_query_new() {
   return ret;
 }
 
+static const char* identity(const char* field, const char* word) {
+  (void)field;
+  if(word) return strdup(word);
+  else return NULL;
+}
+
 wp_query* wp_query_clone(wp_query* other) {
+  return wp_query_substitute(other, identity);
+}
+
+wp_query* wp_query_substitute(wp_query* other, const char *(*substituter)(const char* field, const char* word)) {
   wp_query* ret = malloc(sizeof(wp_query));
   ret->type = other->type;
   ret->num_children = other->num_children;
@@ -21,12 +31,12 @@ wp_query* wp_query_clone(wp_query* other) {
   if(other->field) ret->field = strdup(other->field);
   else ret->field = NULL;
 
-  if(other->word) ret->word = strdup(other->word);
+  if(other->field && other->word) ret->word = substituter(other->field, other->word);
   else ret->word = NULL;
 
   ret->children = ret->next = ret->last = NULL; // set below
   for(wp_query* child = other->children; child != NULL; child = child->next) {
-    wp_query* clone = wp_query_clone(child);
+    wp_query* clone = wp_query_substitute(child, substituter);
     if(ret->last == NULL) ret->children = ret->last = clone;
     else {
       ret->last->next = clone;

data/ext/whistlepig/query.h

@@ -64,9 +64,12 @@ wp_query* wp_query_new_empty();
 // public: make an every-document query node.
 wp_query* wp_query_new_every();
 
-// public: deep clone of a query, but dropping all search state.
+// public: deep clone of a query, dropping all search state.
 wp_query* wp_query_clone(wp_query* other);
 
+// public: build a new query by substituting words from the old query, dropping all search state
+wp_query* wp_query_substitute(wp_query* other, const char *(*substituter)(const char* field, const char* word));
+
 // public: add a query node as a child of another
 wp_query* wp_query_add(wp_query* a, wp_query* b);
 

data/ext/whistlepig/search.c

@@ -793,6 +793,7 @@ wp_error* wp_search_run_query_on_segment(struct wp_query* q, struct wp_segment*
 #endif
 
   while(*num_results < max_num_results) {
+    DEBUG("got %d results so far (max is %d)", *num_results, max_num_results);
     RELAY_ERROR(query_next_doc(q, s, &results[*num_results], &done));
     if(done) break;
     DEBUG("got result %u (%u doc matches)", results[*num_results].doc_id, results[*num_results].num_doc_matches);

data/ext/whistlepig/segment.c

@@ -396,6 +396,9 @@ wp_error* wp_segment_read_posting(wp_segment* s, uint32_t offset, posting* po, i
 wp_error* wp_segment_add_posting(wp_segment* s, const char* field, const char* word, docid_t doc_id, uint32_t num_positions, pos_t positions[]) {
   // TODO move this logic up to ensure_fit()
   int success;
+
+  if(doc_id == 0) RAISE_ERROR("can't add a label to doc 0");
+
   RELAY_ERROR(bump_stringmap(s, &success));
   RELAY_ERROR(bump_stringpool(s, &success));
   RELAY_ERROR(bump_termhash(s, &success));
@@ -465,11 +468,14 @@ wp_error* wp_segment_read_label(wp_segment* s, uint32_t offset, posting* po) {
 wp_error* wp_segment_add_label(wp_segment* s, const char* label, docid_t doc_id) {
   // TODO move this logic up to ensure_fit()
   int success;
+
+  if(doc_id == 0) RAISE_ERROR("can't add a label to doc 0");
+
   RELAY_ERROR(bump_stringmap(s, &success));
   RELAY_ERROR(bump_stringpool(s, &success));
   RELAY_ERROR(bump_termhash(s, &success));
 
-  DEBUG("adding label %s to doc %u", label, doc_id);
+  DEBUG("adding label '%s' to doc %u", label, doc_id);
 
   postings_region* pr = MMAP_OBJ(s->labels, postings_region);
   stringmap* sh = MMAP_OBJ(s->stringmap, stringmap);
@@ -485,22 +491,34 @@ wp_error* wp_segment_add_label(wp_segment* s, const char* label, docid_t doc_id)
   // posting
   uint32_t prev_offset = OFFSET_NONE;
   uint32_t next_offset = termhash_get_val(th, t);
+  docid_t last_docid = DOCID_NONE;
+
   if(next_offset == (uint32_t)-1) next_offset = OFFSET_NONE;
+  DEBUG("start offset is %u (none is %u)", next_offset, OFFSET_NONE);
 
   while(next_offset != OFFSET_NONE) {
-    label_posting* po = wp_segment_label_posting_at(pr, next_offset);
-    if(po->doc_id == doc_id) {
+    label_posting* lp = wp_segment_label_posting_at(pr, next_offset);
+
+    if((last_docid != DOCID_NONE) && (lp->doc_id >= last_docid)) {
+      RAISE_ERROR("whistlepig index corruption! lp %u has docid %u but last docid at lp %u was %u", next_offset, lp->doc_id, prev_offset, last_docid);
+    }
+    else {
+      last_docid = lp->doc_id;
+    }
+
+    DEBUG("got doc id %u next_offset %u at offset %u (looking for doc id %u)", lp->doc_id, lp->next_offset, next_offset, doc_id);
+    if(lp->doc_id == doc_id) {
       DEBUG("already have label '%s' for doc %u; returning", label, doc_id);
       return NO_ERROR;
     }
-    else if(po->doc_id < doc_id) break;
+    else if(lp->doc_id < doc_id) break;
     prev_offset = next_offset;
-    next_offset = po->next_offset;
+    next_offset = lp->next_offset;
   }
 
   // find a space for the posting by first checking for a free postings in the
-  // dead list.  the dead list is the list stored under the sentinel term
-  // with field 0 and word 0.
+  // dead list. the dead list is the list stored under the sentinel term with
+  // field 0 and word 0.
   term dead_term = { .field_s = 0, .word_s = 0 };
   uint32_t entry_offset;
   uint32_t dead_offset = termhash_get_val(th, dead_term);
@@ -550,11 +568,21 @@ wp_error* wp_segment_remove_label(wp_segment* s, const char* label, docid_t doc_
   // find the posting and the previous posting in the list, if any
   uint32_t prev_offset = OFFSET_NONE;
   uint32_t offset = termhash_get_val(th, t);
+  docid_t last_docid = DOCID_NONE;
+
   if(offset == (uint32_t)-1) offset = OFFSET_NONE;
   label_posting* lp = NULL;
 
   while(offset != OFFSET_NONE) {
     lp = wp_segment_label_posting_at(pr, offset);
+
+    if((last_docid != DOCID_NONE) && (lp->doc_id >= last_docid)) {
+      RAISE_ERROR("whistlepig index corruption! lp %u has docid %u but last docid at lp %u was %u", offset, lp->doc_id, prev_offset, last_docid);
+    }
+    else {
+      last_docid = lp->doc_id;
+    }
+
     if(lp->doc_id < doc_id) offset = OFFSET_NONE; // nasty hack to induce failure
     if(lp->doc_id <= doc_id) break;
     prev_offset = offset;

data/ext/whistlepig/test-segment.c

@@ -0,0 +1,404 @@
+#include "test.h"
+#include "segment.h"
+#include "tokenizer.lex.h"
+#include "query.h"
+#include "index.h"
+
+#define SEGMENT_PATH "/tmp/segment-test"
+
+wp_error* setup(wp_segment* segment) {
+  RELAY_ERROR(wp_segment_delete(SEGMENT_PATH));
+  RELAY_ERROR(wp_segment_create(segment, SEGMENT_PATH));
+  return NO_ERROR;
+}
+
+#define ADD_DOC(word, pos) \
+  positions[0] = pos; \
+  RELAY_ERROR(wp_segment_ensure_fit(segment, postings_bytes, 0, &success)); \
+  if(success != 1) RAISE_ERROR("couldn't ensure segment fit"); \
+  RELAY_ERROR(wp_segment_add_posting(segment, "body", word, doc_id, 1, positions));
+
+wp_error* add_docs(wp_segment* segment) {
+  docid_t doc_id;
+  pos_t positions[10];
+  uint32_t postings_bytes;
+  int success;
+
+  RELAY_ERROR(wp_segment_sizeof_posarray(segment, 1, NULL, &postings_bytes));
+
+  RELAY_ERROR(wp_segment_grab_docid(segment, &doc_id));
+  ADD_DOC("one", 0);
+  ADD_DOC("two", 1);
+  ADD_DOC("three", 2);
+
+  RELAY_ERROR(wp_segment_grab_docid(segment, &doc_id));
+  ADD_DOC("two", 0);
+  ADD_DOC("three", 1);
+  ADD_DOC("four", 2);
+
+  RELAY_ERROR(wp_segment_grab_docid(segment, &doc_id));
+  ADD_DOC("three", 0);
+  ADD_DOC("four", 1);
+  ADD_DOC("five", 2);
+
+  return NO_ERROR;
+}
+
+TEST(initial_state) {
+  wp_segment segment;
+  RELAY_ERROR(setup(&segment));
+
+  postings_region* pr = MMAP_OBJ(segment.postings, postings_region);
+  ASSERT(pr->num_docs == 0);
+  ASSERT(pr->num_postings == 0);
+
+  RELAY_ERROR(wp_segment_unload(&segment));
+  return NO_ERROR;
+}
+
+TEST(adding_a_doc_increments_counts) {
+  wp_segment segment;
+  pos_t positions[10];
+  docid_t doc_id;
+
+  RELAY_ERROR(setup(&segment));
+  RELAY_ERROR(wp_segment_grab_docid(&segment, &doc_id));
+
+  positions[0] = 0;
+  RELAY_ERROR(wp_segment_add_posting(&segment, "body", "hello", doc_id, 1, positions));
+  positions[0] = 1;
+  RELAY_ERROR(wp_segment_add_posting(&segment, "body", "there", doc_id, 1, positions));
+
+  postings_region* pr = MMAP_OBJ(segment.postings, postings_region);
+  ASSERT(pr->num_docs == 1);
+  ASSERT(pr->num_postings == 2);
+
+  RELAY_ERROR(wp_segment_unload(&segment));
+  return NO_ERROR;
+}
+
+#define RUN_QUERY(query) \
+  RELAY_ERROR(wp_search_init_search_state(query, &segment)); \
+  RELAY_ERROR(wp_search_run_query_on_segment(query, &segment, 10, &num_results, &results[0])); \
+  RELAY_ERROR(wp_search_release_search_state(query));
+
+TEST(simple_term_queries) {
+  wp_segment segment;
+  uint32_t num_results;
+  search_result results[10];
+  wp_query* query;
+
+  RELAY_ERROR(setup(&segment));
+  RELAY_ERROR(add_docs(&segment));
+
+  query = wp_query_new_term("body", "one");
+  RUN_QUERY(query);
+
+  ASSERT(num_results == 1);
+  ASSERT(results[0].doc_id == 1);
+
+  query = wp_query_new_term("body", "two");
+  RUN_QUERY(query);
+
+  ASSERT(num_results == 2);
+  ASSERT(results[0].doc_id == 2);
+  ASSERT(results[1].doc_id == 1);
+
+  RELAY_ERROR(wp_segment_unload(&segment));
+  return NO_ERROR;
+}
+
+TEST(simple_conjunctive_queries) {
+  wp_segment segment;
+  uint32_t num_results;
+  search_result results[10];
+  wp_query* query;
+
+  RELAY_ERROR(setup(&segment));
+  RELAY_ERROR(add_docs(&segment));
+
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, wp_query_new_term("body", "one"));
+  query = wp_query_add(query, wp_query_new_term("body", "two"));
+
+  RUN_QUERY(query);
+
+  ASSERT(num_results == 1);
+  ASSERT(results[0].doc_id == 1);
+
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, wp_query_new_term("body", "four"));
+  query = wp_query_add(query, wp_query_new_term("body", "two"));
+
+  RUN_QUERY(query);
+
+  ASSERT(num_results == 1);
+  ASSERT(results[0].doc_id == 2);
+
+  // <empty>
+  query = wp_query_new_conjunction();
+  RUN_QUERY(query);
+  ASSERT(num_results == 0);
+
+  RELAY_ERROR(wp_segment_unload(&segment));
+  return NO_ERROR;
+}
+
+TEST(simple_phrasal_queries) {
+  wp_segment segment;
+  uint32_t num_results;
+  search_result results[10];
+  wp_query* query;
+
+  RELAY_ERROR(setup(&segment));
+  RELAY_ERROR(add_docs(&segment));
+
+  query = wp_query_new_phrase();
+  query = wp_query_add(query, wp_query_new_term("body", "one"));
+  query = wp_query_add(query, wp_query_new_term("body", "two"));
+  RUN_QUERY(query);
+  ASSERT(num_results == 1);
+  ASSERT(results[0].doc_id == 1);
+
+  query = wp_query_new_phrase();
+  query = wp_query_add(query, wp_query_new_term("body", "two"));
+  query = wp_query_add(query, wp_query_new_term("body", "one"));
+  RUN_QUERY(query);
+  ASSERT(num_results == 0);
+
+  query = wp_query_new_phrase();
+  query = wp_query_add(query, wp_query_new_term("body", "two"));
+  query = wp_query_add(query, wp_query_new_term("body", "three"));
+  RUN_QUERY(query);
+  ASSERT(num_results == 2);
+  ASSERT(results[0].doc_id == 2);
+  ASSERT(results[1].doc_id == 1);
+
+  query = wp_query_new_phrase();
+  query = wp_query_add(query, wp_query_new_term("body", "one"));
+  query = wp_query_add(query, wp_query_new_term("body", "two"));
+  query = wp_query_add(query, wp_query_new_term("body", "three"));
+  RUN_QUERY(query);
+  ASSERT(num_results == 1);
+  ASSERT(results[0].doc_id == 1);
+
+  RELAY_ERROR(wp_segment_unload(&segment));
+  return NO_ERROR;
+}
+
+TEST(segment_conjuction_of_phrase_queries) {
+  wp_segment segment;
+  uint32_t num_results;
+  search_result results[10];
+  wp_query* query;
+  wp_query* subquery;
+
+  RELAY_ERROR(setup(&segment));
+  RELAY_ERROR(add_docs(&segment));
+
+  // one "two three"
+  subquery = wp_query_new_phrase();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "two"));
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "three"));
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, wp_query_new_term("body", "one"));
+  query = wp_query_add(query, subquery);
+
+  RUN_QUERY(query);
+  ASSERT(num_results == 1);
+  ASSERT(results[0].doc_id == 1);
+
+  // "two three" one
+  subquery = wp_query_new_phrase();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "two"));
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "three"));
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, subquery);
+  query = wp_query_add(query, wp_query_new_term("body", "one"));
+
+  RUN_QUERY(query);
+  ASSERT(num_results == 1);
+  ASSERT(results[0].doc_id == 1);
+
+  // one "three two"
+  subquery = wp_query_new_phrase();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "three"));
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "two"));
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, wp_query_new_term("body", "one"));
+  query = wp_query_add(query, subquery);
+
+  RUN_QUERY(query);
+  ASSERT(num_results == 0);
+
+  // two "two three"
+  subquery = wp_query_new_phrase();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "two"));
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "three"));
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, wp_query_new_term("body", "two"));
+  query = wp_query_add(query, subquery);
+
+  RUN_QUERY(query);
+  ASSERT(num_results == 2);
+  ASSERT(results[0].doc_id == 2);
+  ASSERT(results[1].doc_id == 1);
+
+  RELAY_ERROR(wp_segment_unload(&segment));
+  return NO_ERROR;
+}
+
+TEST(negation_queries) {
+  wp_segment segment;
+  uint32_t num_results;
+  search_result results[10];
+  wp_query* query;
+  wp_query* subquery;
+
+  RELAY_ERROR(setup(&segment));
+  RELAY_ERROR(add_docs(&segment));
+
+  // one "two three"
+  subquery = wp_query_new_phrase();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "two"));
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "three"));
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, wp_query_new_term("body", "one"));
+  query = wp_query_add(query, subquery);
+
+  RUN_QUERY(query);
+  ASSERT(num_results == 1);
+  ASSERT(results[0].doc_id == 1);
+
+  // "two three" one
+  subquery = wp_query_new_phrase();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "two"));
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "three"));
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, subquery);
+  query = wp_query_add(query, wp_query_new_term("body", "one"));
+
+  RUN_QUERY(query);
+  ASSERT(num_results == 1);
+  ASSERT(results[0].doc_id == 1);
+
+  // one "three two"
+  subquery = wp_query_new_phrase();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "three"));
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "two"));
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, wp_query_new_term("body", "one"));
+  query = wp_query_add(query, subquery);
+
+  RUN_QUERY(query);
+  ASSERT(num_results == 0);
+
+  // two "two three"
+  subquery = wp_query_new_phrase();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "two"));
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "three"));
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, wp_query_new_term("body", "two"));
+  query = wp_query_add(query, subquery);
+
+  RUN_QUERY(query);
+  ASSERT(num_results == 2);
+  ASSERT(results[0].doc_id == 2);
+  ASSERT(results[1].doc_id == 1);
+
+  // <empty>
+  query = wp_query_new_conjunction();
+  RUN_QUERY(query);
+  ASSERT(num_results == 0);
+
+  // -one
+  subquery = wp_query_new_term("body", "one");
+  query = wp_query_new_negation();
+  query = wp_query_add(query, subquery);
+  RUN_QUERY(query);
+  ASSERT(num_results == 2);
+  ASSERT(results[0].doc_id == 3);
+  ASSERT(results[1].doc_id == 2);
+
+  // -two
+  subquery = wp_query_new_term("body", "two");
+  query = wp_query_new_negation();
+  query = wp_query_add(query, subquery);
+  RUN_QUERY(query);
+  ASSERT(num_results == 1);
+  ASSERT(results[0].doc_id == 3);
+
+  // -three
+  subquery = wp_query_new_term("body", "three");
+  query = wp_query_new_negation();
+  query = wp_query_add(query, subquery);
+  RUN_QUERY(query);
+  ASSERT(num_results == 0);
+
+  // -potato
+  subquery = wp_query_new_term("body", "potato");
+  query = wp_query_new_negation();
+  query = wp_query_add(query, subquery);
+  RUN_QUERY(query);
+  ASSERT(num_results == 3);
+  ASSERT(results[0].doc_id == 3);
+  ASSERT(results[1].doc_id == 2);
+  ASSERT(results[2].doc_id == 1);
+
+  // -"one two"
+  subquery = wp_query_new_conjunction();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "one"));
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "two"));
+  query = wp_query_new_negation();
+  query = wp_query_add(query, subquery);
+  RUN_QUERY(query);
+  ASSERT(num_results == 2);
+  ASSERT(results[0].doc_id == 3);
+  ASSERT(results[1].doc_id == 2);
+
+  // -(AND one three)
+  subquery = wp_query_new_conjunction();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "one"));
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "three"));
+  query = wp_query_new_negation();
+  query = wp_query_add(query, subquery);
+  RUN_QUERY(query);
+  ASSERT(num_results == 2);
+  ASSERT(results[0].doc_id == 3);
+  ASSERT(results[1].doc_id == 2);
+
+  // -"one three"
+  subquery = wp_query_new_phrase();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "one"));
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "three"));
+  query = wp_query_new_negation();
+  query = wp_query_add(query, subquery);
+  RUN_QUERY(query);
+  ASSERT(num_results == 3);
+
+  // (AND -one three)
+  subquery = wp_query_new_negation();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "one"));
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, subquery);
+  query = wp_query_add(query, wp_query_new_term("body", "three"));
+  RUN_QUERY(query);
+  ASSERT(num_results == 2);
+  ASSERT(results[0].doc_id == 3);
+  ASSERT(results[1].doc_id == 2);
+
+  // (AND three -one)
+  subquery = wp_query_new_negation();
+  subquery = wp_query_add(subquery, wp_query_new_term("body", "one"));
+  query = wp_query_new_conjunction();
+  query = wp_query_add(query, wp_query_new_term("body", "three"));
+  query = wp_query_add(query, subquery);
+  RUN_QUERY(query);
+  ASSERT(num_results == 2);
+  ASSERT(results[0].doc_id == 3);
+  ASSERT(results[1].doc_id == 2);
+
+  RELAY_ERROR(wp_segment_unload(&segment));
+  return NO_ERROR;
+}
+

data/ext/whistlepig/test-stringmap.c

@@ -0,0 +1,82 @@
+#include <string.h>
+#include "stringmap.h"
+#include "error.h"
+#include "test.h"
+
+static stringmap* setup() {
+  stringpool* p = malloc(stringpool_initial_size());
+  stringpool_init(p);
+  stringmap* q = malloc(stringmap_initial_size());
+  stringmap_init(q, p);
+  return q;
+}
+
+TEST(stringmap_initial_state) {
+  stringmap* q = setup();
+  ASSERT(q->n_occupied == 0);
+  ASSERT(!stringmap_needs_bump(q));
+
+  free(q);
+  return NO_ERROR;
+}
+
+TEST(stringmap_lookups_on_empty) {
+  stringmap* q = setup();
+
+  ASSERT(stringmap_string_to_int(q, "hot potato") == (uint32_t)-1);
+  ASSERT(stringmap_int_to_string(q, 0) == NULL);
+  ASSERT(stringmap_int_to_string(q, 1234) == NULL);
+
+  free(q);
+  return NO_ERROR;
+}
+
+TEST(stringmap_multiple_adds) {
+  stringmap* q = setup();
+
+  ASSERT(stringmap_string_to_int(q, "hot potato") == (uint32_t)-1);
+  uint32_t x, y;
+  RELAY_ERROR(stringmap_add(q, "hot potato", &x));
+  ASSERT(x != (uint32_t)-1);
+  RELAY_ERROR(stringmap_add(q, "hot potato", &y));
+  ASSERT(y != (uint32_t)-1);
+  ASSERT(x == y);
+
+  free(q);
+  return NO_ERROR;
+}
+
+TEST(stringmap_hashing_is_preserved) {
+  stringmap* q = setup();
+
+  uint32_t x, y;
+  RELAY_ERROR(stringmap_add(q, "hello there", &x));
+  ASSERT(x != (uint32_t)-1);
+  const char* a = stringmap_int_to_string(q, x);
+  ASSERT(strcmp(a, "hello there") == 0);
+
+  RELAY_ERROR(stringmap_add(q, "how are you?", &y));
+  const char* b = stringmap_int_to_string(q, y);
+  ASSERT(strcmp(b, "how are you?") == 0);
+
+  ASSERT(x != y);
+
+  free(q);
+  return NO_ERROR;
+}
+
+TEST(stringmap_detects_out_of_room) {
+  stringmap* q = setup();
+
+  uint32_t x, y, z, w;
+  RELAY_ERROR(stringmap_add(q, "one", &x));
+  RELAY_ERROR(stringmap_add(q, "two", &y));
+  RELAY_ERROR(stringmap_add(q, "three", &z));
+
+  wp_error* e = stringmap_add(q, "four", &w);
+  ASSERT(e != NULL);
+  wp_error_free(e);
+
+  free(q);
+  return NO_ERROR;
+}

data/ext/whistlepig/test-stringpool.c

@@ -0,0 +1,67 @@
+#include <string.h>
+#include "stringpool.h"
+#include "error.h"
+#include "test.h"
+
+TEST(stringpool_initial_state) {
+  stringpool* p = malloc(stringpool_initial_size());
+  stringpool_init(p);
+
+  ASSERT(!stringpool_needs_bump(p));
+
+  free(p);
+  return NO_ERROR;
+}
+
+TEST(stringpool_add_gives_unique_ids) {
+  stringpool* p = malloc(stringpool_initial_size());
+  stringpool_init(p);
+
+  uint32_t ret1 = stringpool_add(p, "potato");
+  ASSERT(ret1 > 0);
+
+  uint32_t ret2 = stringpool_add(p, "monkey");
+  ASSERT(ret2 > 0);
+
+  ASSERT(ret1 != ret2);
+
+  free(p);
+  return NO_ERROR;
+}
+
+TEST(stringpool_add_gives_ids_that_lookup_returns) {
+  stringpool* p = malloc(stringpool_initial_size());
+  stringpool_init(p);
+
+  uint32_t ret;
+  char* s;
+
+  ret = stringpool_add(p, "potato");
+  s = stringpool_lookup(p, ret);
+  ASSERT(!strcmp(s, "potato"));
+
+  ret = stringpool_add(p, "monkey");
+  s = stringpool_lookup(p, ret);
+  ASSERT(!strcmp(s, "monkey"));
+
+  free(p);
+  return NO_ERROR;
+}
+
+TEST(stringpool_detects_out_of_room) {
+  stringpool* p = malloc(stringpool_initial_size());
+  stringpool_init(p);
+
+  uint32_t ret;
+  int times = stringpool_initial_size() / 6;
+  for(int i = 0; i < times - 1; i++)  {
+    ret = stringpool_add(p, "12345");
+    ASSERT(ret != (uint32_t)-1);
+  }
+
+  ret = stringpool_add(p, "12345");
+  ASSERT(ret == (uint32_t)-1);
+
+  return NO_ERROR;
+}
+

data/ext/whistlepig/test-termhash.c

@@ -0,0 +1,95 @@
+#include "termhash.h"
+#include "test.h"
+#include "error.h"
+
+TEST(termhash_initial_state) {
+  termhash* h = malloc(termhash_initial_size());
+  termhash_init(h);
+
+  ASSERT(h->n_occupied == 0);
+  //ASSERT(!termhash_getting_full(h));
+
+  free(h);
+  return NO_ERROR;
+}
+
+TEST(termhash_lookups_on_empty) {
+  termhash* h = malloc(termhash_initial_size());
+  termhash_init(h);
+
+  term t1 = {0, 0};
+  term t2 = {10, 20};
+  term t3 = {123, 345};
+
+  ASSERT(termhash_get_val(h, t1) == (uint32_t)-1);
+  ASSERT(termhash_get_val(h, t2) == (uint32_t)-1);
+  ASSERT(termhash_get_val(h, t3) == (uint32_t)-1);
+
+  free(h);
+  return NO_ERROR;
+}
+
+TEST(termhash_overwriting) {
+  termhash* h = malloc(termhash_initial_size());
+  termhash_init(h);
+
+  term t1 = {5, 11};
+
+  ASSERT(termhash_get_val(h, t1) == (uint32_t)-1);
+  RELAY_ERROR(termhash_put_val(h, t1, 1234));
+  ASSERT(termhash_get_val(h, t1) == 1234);
+
+  RELAY_ERROR(termhash_put_val(h, t1, 2345));
+  ASSERT(termhash_get_val(h, t1) == 2345);
+
+  RELAY_ERROR(termhash_put_val(h, t1, 1));
+  ASSERT(termhash_get_val(h, t1) == 1);
+
+  free(h);
+  return NO_ERROR;
+}
+
+TEST(termhash_many_puts) { // try and force a resize
+  termhash* h = malloc(termhash_initial_size());
+  termhash_init(h);
+
+  term t1 = {1, 0};
+
+  for(int i = 1; i < 100; i++) {
+    t1.word_s = i;
+    RELAY_ERROR(termhash_put_val(h, t1, 1000 + i));
+    if(termhash_needs_bump(h)) {
+      h = realloc(h, termhash_next_size(h));
+      if(h == NULL) RAISE_SYSERROR("realloc");
+      termhash_setup(h);
+      RELAY_ERROR(termhash_bump_size(h));
+    }
+  }
+
+  t1.word_s = 55;
+  uint32_t v = termhash_get_val(h, t1);
+  ASSERT(v == 1055);
+
+  free(h);
+  return NO_ERROR;
+}
+
+TEST(termhash_detects_out_of_room) {
+  termhash* h = malloc(termhash_initial_size());
+  termhash_init(h);
+
+  term t = {1, 0};
+
+  for(int i = 0; i < 3; i++) {
+    t.word_s = i;
+    RELAY_ERROR(termhash_put_val(h, t, 100 + i));
+  }
+
+  t.word_s = 999;
+  wp_error* e = termhash_put_val(h, t, 999);
+  ASSERT(e != NULL);
+  wp_error_free(e);
+
+  free(h);
+  return NO_ERROR;
+}

data/ext/whistlepig/test-tokenizer.c

@@ -0,0 +1,55 @@
+#include "test.h"
+#include "tokenizer.lex.h"
+
+#define ASSERT_NEXT_WORD(word) { \
+  int token_type = yylex(scanner); \
+  ASSERT(token_type == TOK_WORD); \
+  ASSERT(!strcmp(word, yyget_text(scanner))); \
+}
+
+#define ASSERT_DONE { \
+  int token_type = yylex(scanner); \
+  ASSERT(token_type == TOK_DONE); \
+}
+
+TEST(tokenizes_easy_words) {
+  yyscan_t scanner;
+  lexinfo charpos = {0, 0};
+
+  yylex_init_extra(&charpos, &scanner);
+
+  const char* string = "i love mice";
+  YY_BUFFER_STATE state = yy_scan_string(string, scanner);
+
+  ASSERT_NEXT_WORD("i");
+  ASSERT_NEXT_WORD("love");
+  ASSERT_NEXT_WORD("mice");
+  ASSERT_DONE;
+
+  yy_delete_buffer(state, scanner);
+  yylex_destroy(scanner);
+
+  return NO_ERROR;
+}
+
+TEST(strips_trailing_punctuation) {
+  yyscan_t scanner;
+  lexinfo charpos = {0, 0};
+
+  yylex_init_extra(&charpos, &scanner);
+
+  const char* string = "hey! this: you're <cool>";
+  YY_BUFFER_STATE state = yy_scan_string(string, scanner);
+
+  ASSERT_NEXT_WORD("hey");
+  ASSERT_NEXT_WORD("this");
+  ASSERT_NEXT_WORD("you're");
+  ASSERT_NEXT_WORD("cool");
+  ASSERT_DONE;
+
+  yy_delete_buffer(state, scanner);
+  yylex_destroy(scanner);
+
+  return NO_ERROR;
+}
+

data/ext/whistlepig/test.h

@@ -0,0 +1,38 @@
+#ifndef WP_TEST_H_
+#define WP_TEST_H_
+
+// whistlepig test header file
+// (c) 2011 William Morgan. See COPYING for license terms.
+//
+// macros for the c unit tests
+
+#define ASSERT(x) do { \
+  (*asserts)++; \
+  if(!(x)) { \
+    printf("-- test failure: (" #x ") is FALSE in %s (%s:%d)\n\n", __PRETTY_FUNCTION__, __FILE__, __LINE__); \
+    *fail = 1; \
+    return NO_ERROR; \
+  } \
+} while(0)
+
+#define TEST(x) wp_error* test_##x(int* fail, int* asserts)
+
+#define RUNTEST(x) do { \
+  int fail = 0; \
+  int this_asserts = 0; \
+  tests++; \
+  wp_error* err = test_##x(&fail, &this_asserts); \
+  asserts += this_asserts; \
+  if(fail) { \
+    printf("FAIL " #x "\n"); \
+    failures++; \
+  } \
+  else if(err) { \
+    errors++; \
+    printf(" ERR " #x "\n"); \
+    PRINT_ERROR(err, stdout); \
+  } \
+  else printf("PASS %d/%d " #x "\n", this_asserts, this_asserts); \
+} while(0)
+
+#endif

data/ext/whistlepig/timer.h

@@ -0,0 +1,28 @@
+#ifndef WP_TIMER_H_
+#define WP_TIMER_H_
+
+// whistlepig main header file
+// (c) 2011 William Morgan. See COPYING for license terms.
+//
+// just some timer macros
+
+#include <sys/time.h>
+
+#define TIMER(name) \
+  struct timeval name##_startt, name##_endt; \
+  long name##_elapsed;
+
+#define START_TIMER(name) \
+  TIMER(name) \
+  gettimeofday(&name##_startt, NULL);
+
+#define RESET_TIMER(name) gettimeofday(&name##_startt, NULL);
+
+#define MARK_TIMER(name) \
+  gettimeofday(&name##_endt, NULL); \
+  name##_elapsed = ((name##_endt.tv_sec - name##_startt.tv_sec) * 1000) + ((name##_endt.tv_usec - name##_startt.tv_usec) / 1000);
+
+#define TIMER_MS(name) name##_elapsed
+#define TIMER_MS(name) name##_elapsed
+
+#endif

data/ext/whistlepig/{whistlepigc.c → whistlepig.c}

@@ -367,6 +367,28 @@ static VALUE query_clone(VALUE self) {
   return o_query;
 }
 
+static const char* yielding_substituter(const char* field, const char* term) {
+  VALUE result = rb_yield_values(2, rb_str_new2(field), rb_str_new2(term));
+  if(NIL_P(result)) return strdup(term);
+  else return strdup(RSTRING_PTR(result));
+}
+
+/*
+ * Returns a new query that's the result of applying the block to each
+ * word in the query. Useful for transforming queries programmatically
+ * after they've been parsed.
+ *
+ */
+static VALUE query_map_terms(VALUE self) {
+  char buf[1024];
+
+  wp_query* query; Data_Get_Struct(self, wp_query, query);
+  wp_query* result = wp_query_substitute(query, yielding_substituter);
+
+  VALUE o_query = Data_Wrap_Struct(c_query, NULL, wp_query_free, result);
+  return o_query;
+}
+
 /*
  * call-seq: and(other)
  *
@@ -504,7 +526,7 @@ static VALUE index_run_query(VALUE self, VALUE v_query, VALUE v_max_num_results)
   return array;
 }
 
-void Init_whistlepigc() {
+void Init_whistlepig() {
   VALUE m_whistlepig;
 
   m_whistlepig = rb_define_module("Whistlepig");
@@ -541,6 +563,7 @@ void Init_whistlepigc() {
   rb_define_method(c_query, "or", query_or, 1);
   rb_define_method(c_query, "to_s", query_to_s, 0);
   rb_define_method(c_query, "clone", query_clone, 0);
+  rb_define_method(c_query, "term_map", query_map_terms, 0);
   rb_define_attr(c_query, "query", 1, 0);
 
   c_error = rb_define_class_under(m_whistlepig, "Error", rb_eStandardError);

data/lib/whistlepig.rb

@@ -1,4 +1,4 @@
-require "whistlepigc"
+require "whistlepig/whistlepig"
 
 module Whistlepig
   ## A full-text index. You can add entries to it, and you can run queries

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: whistlepig
 version: !ruby/object:Gem::Version
-  version: '0.7'
+  version: '0.8'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-06-19 13:10:35.000000000 -07:00
+date: 2012-03-13 14:35:03.000000000 -07:00
 default_executable: 
 dependencies: []
 description: Whistlepig is a minimalist realtime full-text search index. Its goal
@@ -22,43 +22,51 @@ extensions:
 - ext/whistlepig/extconf.rb
 extra_rdoc_files:
 - README
-- ext/whistlepig/whistlepigc.c
+- ext/whistlepig/whistlepig.c
 files:
 - README
 - ext/whistlepig/extconf.rb
 - lib/whistlepig.rb
 - ext/whistlepig/query-parser.lex.h
 - ext/whistlepig/entry.h
-- ext/whistlepig/whistlepigc.c
 - ext/whistlepig/stringmap.c
 - ext/whistlepig/tokenizer.lex.h
 - ext/whistlepig/whistlepig.h
 - ext/whistlepig/error.c
 - ext/whistlepig/extconf.h
 - ext/whistlepig/stringmap.h
+- ext/whistlepig/timer.h
 - ext/whistlepig/query-parser.lex.c
 - ext/whistlepig/defaults.h
 - ext/whistlepig/tokenizer.lex.c
+- ext/whistlepig/test-stringpool.c
 - ext/whistlepig/termhash.c
 - ext/whistlepig/query-parser.h
 - ext/whistlepig/index.c
 - ext/whistlepig/stringpool.c
+- ext/whistlepig/test-termhash.c
 - ext/whistlepig/query.h
 - ext/whistlepig/query-parser.c
 - ext/whistlepig/stringpool.h
 - ext/whistlepig/mmap-obj.c
+- ext/whistlepig/whistlepig.c
 - ext/whistlepig/search.c
 - ext/whistlepig/termhash.h
 - ext/whistlepig/query.c
+- ext/whistlepig/dump.c
 - ext/whistlepig/query-parser.tab.h
+- ext/whistlepig/test.h
 - ext/whistlepig/khash.h
 - ext/whistlepig/query-parser.tab.c
 - ext/whistlepig/entry.c
 - ext/whistlepig/index.h
 - ext/whistlepig/segment.h
+- ext/whistlepig/test-stringmap.c
 - ext/whistlepig/mmap-obj.h
 - ext/whistlepig/segment.c
+- ext/whistlepig/test-segment.c
 - ext/whistlepig/search.h
+- ext/whistlepig/test-tokenizer.c
 - ext/whistlepig/error.h
 has_rdoc: true
 homepage: http://masanjin.net/whistlepig