isomorfeus-ferret 0.12.0

data/ext/isomorfeus_ferret_ext/test_term_vectors.c

@@ -0,0 +1,373 @@
+#include "frt_index.h"
+#include "test.h"
+
+#define NUM_TERMS 100
+#define TERM_LEN 10
+
+
+static void test_posting(TestCase *tc, void *data)
+{
+    FrtMemoryPool *mp = (FrtMemoryPool *)data;
+    FrtPostingList *pl;
+    FrtPosting *p = frt_p_new(mp, 0, 10);
+    Aiequal(0, p->doc_num);
+    Aiequal(1, p->freq);
+    Aiequal(10, p->first_occ->pos);
+    Apnull(p->first_occ->next);
+
+    pl = frt_pl_new(mp, "seven", 5, p);
+    Aiequal(5, pl->term_len);
+    Asequal("seven", pl->term);
+    Apequal(p->first_occ, pl->last_occ);
+
+    frt_pl_add_occ(mp, pl, 50);
+    Apequal(pl->last_occ, p->first_occ->next);
+    Aiequal(2, p->freq);
+    Aiequal(50,  pl->last_occ->pos);
+    Apnull(pl->last_occ->next);
+
+    frt_pl_add_occ(mp, pl, 345);
+    Apequal(pl->last_occ, p->first_occ->next->next);
+    Aiequal(3, p->freq);
+    Aiequal(345, pl->last_occ->pos);
+    Apnull(pl->last_occ->next);
+}
+
+static FrtFieldInfos *create_tv_fis()
+{
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO);
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv2"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv_with_positions"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv_with_offsets"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_OFFSETS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv_with_positions_offsets"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
+    return fis;
+}
+
+static char **create_tv_terms(FrtMemoryPool *mp)
+{
+    int i;
+    char term_buf[10];
+    char **terms = FRT_MP_ALLOC_N(mp, char *, NUM_TERMS);
+    for (i = 0; i < NUM_TERMS; i++) {
+        sprintf(term_buf, "%09d", i);
+        terms[i] = frt_mp_strdup(mp, term_buf);
+    }
+    return terms;
+}
+
+static FrtPostingList **create_tv_plists(FrtMemoryPool *mp, char **terms)
+{
+    int i, j;
+    FrtPostingList **plists, *pl;
+    plists = FRT_MP_ALLOC_N(mp, FrtPostingList *, NUM_TERMS);
+    for (i = 0; i < NUM_TERMS; i++) {
+        pl = plists[i] =
+            frt_pl_new(mp, terms[i], 9, frt_p_new(mp, 0, 0));
+        for (j = 1; j <= i; j++) {
+            frt_pl_add_occ(mp, pl, j);
+        }
+    }
+    return plists;
+}
+
+static FrtOffset *create_tv_offsets(FrtMemoryPool *mp)
+{
+    int i;
+    FrtOffset *offsets = FRT_MP_ALLOC_N(mp, FrtOffset, NUM_TERMS);
+    for (i = 0; i < NUM_TERMS; i++) {
+        offsets[i].start = 5 * i;
+        offsets[i].end = 5 * i + 4;
+    }
+    return offsets;
+}
+
+static void test_tv_single_doc(TestCase *tc, void *data)
+{
+    int i, j;
+    FrtStore *store = frt_open_ram_store();
+    FrtMemoryPool *mp = (FrtMemoryPool *)data;
+    FrtFieldsReader *fr;
+    FrtFieldsWriter *fw;
+    FrtTermVector *tv;
+    FrtHash *tvs;
+    FrtFieldInfos *fis = create_tv_fis();
+    char **terms = create_tv_terms(mp);
+    FrtPostingList **plists = create_tv_plists(mp, terms);
+    FrtOffset *offsets = create_tv_offsets(mp);
+    FrtDocument *doc = frt_doc_new();
+
+    fw = frt_fw_open(store, "_0", fis);
+    frt_fw_close(fw);
+
+    fr = frt_fr_open(store, "_0", fis);
+    Aiequal(0, fr->size);
+    frt_fr_close(fr);
+
+
+    fw = frt_fw_open(store, "_0", fis);
+    frt_fw_add_doc(fw, doc);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv_with_positions"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv_with_offsets"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv_with_positions_offsets"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+    frt_fw_write_tv_index(fw);
+    frt_fw_close(fw);
+   frt_doc_destroy(doc);
+
+    fr = frt_fr_open(store, "_0", fis);
+    Aiequal(1, fr->size);
+
+    /* test individual field's term vectors */
+    tv = frt_fr_get_field_tv(fr, 0, frt_fis_get_field(fis, rb_intern("tv"))->number);
+    if (Apnotnull(tv)) {
+        Aiequal(frt_fis_get_field(fis, rb_intern("tv"))->number, tv->field_num);
+        Aiequal(NUM_TERMS, tv->term_cnt);
+        Aiequal(0, tv->offset_cnt);
+        for (i = 0; i < NUM_TERMS; i++) {
+            Asequal(terms[i], tv->terms[i].text);
+            Aiequal(i + 1, tv->terms[i].freq);
+            Apnull(tv->terms[i].positions);
+        }
+        Apnull(tv->offsets);
+    }
+    if (tv) frt_tv_destroy(tv);
+
+    tv = frt_fr_get_field_tv(fr, 0,
+                         frt_fis_get_field(fis, rb_intern("tv_with_positions"))->number);
+    if (Apnotnull(tv)) {
+        Aiequal(frt_fis_get_field(fis, rb_intern("tv_with_positions"))->number,
+                tv->field_num);
+        Aiequal(NUM_TERMS, tv->term_cnt);
+        Aiequal(0, tv->offset_cnt);
+        for (i = 0; i < NUM_TERMS; i++) {
+            Asequal(terms[i], tv->terms[i].text);
+            Aiequal(i + 1, tv->terms[i].freq);
+            for (j = 0; j <= i; j++) {
+                Aiequal(j, tv->terms[i].positions[j]);
+            }
+        }
+        Apnull(tv->offsets);
+    }
+    if (tv) frt_tv_destroy(tv);
+
+    tv = frt_fr_get_field_tv(fr, 0, frt_fis_get_field(fis, rb_intern("tv_with_offsets"))->number);
+    if (Apnotnull(tv)) {
+        Aiequal(frt_fis_get_field(fis, rb_intern("tv_with_offsets"))->number, tv->field_num);
+        Aiequal(NUM_TERMS, tv->term_cnt);
+        Aiequal(NUM_TERMS, tv->offset_cnt);
+        for (i = 0; i < NUM_TERMS; i++) {
+            Asequal(terms[i], tv->terms[i].text);
+            Aiequal(i + 1, tv->terms[i].freq);
+            Apnull(tv->terms[i].positions);
+        }
+        for (i = 0; i < NUM_TERMS; i++) {
+            Aiequal(i * 5, tv->offsets[i].start);
+            Aiequal(i * 5 + 4, tv->offsets[i].end);
+        }
+    }
+    if (tv) frt_tv_destroy(tv);
+
+    tv = frt_fr_get_field_tv(fr, 0, frt_fis_get_field(fis, rb_intern("tv_with_positions_offsets"))->number);
+    if (Apnotnull(tv)) {
+        Aiequal(frt_fis_get_field(fis, rb_intern("tv_with_positions_offsets"))->number, tv->field_num);
+        Aiequal(NUM_TERMS, tv->term_cnt);
+        Aiequal(NUM_TERMS, tv->offset_cnt);
+        for (i = 0; i < NUM_TERMS; i++) {
+            Asequal(terms[i], tv->terms[i].text);
+            Aiequal(i + 1, tv->terms[i].freq);
+            for (j = 1; j <= i; j++) {
+                Aiequal(j, tv->terms[i].positions[j]);
+            }
+        }
+        for (i = 0; i < NUM_TERMS; i++) {
+            Aiequal(i * 5, tv->offsets[i].start);
+            Aiequal(i * 5 + 4, tv->offsets[i].end);
+        }
+    }
+    if (tv) frt_tv_destroy(tv);
+
+    tv = frt_fr_get_field_tv(fr, 0, frt_fis_get_or_add_field(fis, rb_intern("tv2"))->number);
+    Apnull(tv);
+    tv = frt_fr_get_field_tv(fr, 0, frt_fis_get_or_add_field(fis, rb_intern("new"))->number);
+    Apnull(tv);
+
+    /* test document's term vectors */
+    tvs = frt_fr_get_tv(fr, 0);
+    Aiequal(4, tvs->size);
+    tv = (FrtTermVector*)frt_h_get(tvs, (void *)rb_intern("tv2"));
+    Apnull(tv);
+    tv = (FrtTermVector*)frt_h_get(tvs, (void *)rb_intern("other"));
+    Apnull(tv);
+
+    tv = (FrtTermVector*)frt_h_get(tvs, (void *)rb_intern("tv_with_positions_offsets"));
+    if (Apnotnull(tv)) {
+        Aiequal(frt_fis_get_field(fis, rb_intern("tv_with_positions_offsets"))->number, tv->field_num);
+        Aiequal(NUM_TERMS, tv->term_cnt);
+        Aiequal(NUM_TERMS, tv->offset_cnt);
+        for (i = 0; i < NUM_TERMS; i++) {
+            Asequal(terms[i], tv->terms[i].text);
+            Aiequal(i + 1, tv->terms[i].freq);
+            for (j = 1; j <= i; j++) {
+                Aiequal(j, tv->terms[i].positions[j]);
+            }
+        }
+        for (i = 0; i < NUM_TERMS; i++) {
+            Aiequal(i * 5, tv->offsets[i].start);
+            Aiequal(i * 5 + 4, tv->offsets[i].end);
+        }
+    }
+    frt_h_destroy(tvs);
+
+    frt_fr_close(fr);
+    frt_fis_deref(fis);
+    frt_store_deref(store);
+}
+
+static void test_tv_multi_doc(TestCase *tc, void *data)
+{
+    int i, j;
+    FrtStore *store = frt_open_ram_store();
+    FrtMemoryPool *mp = (FrtMemoryPool *)data;
+    FrtFieldsReader *fr;
+    FrtFieldsWriter *fw;
+    FrtTermVector *tv;
+    FrtHash *tvs;
+    FrtFieldInfos *fis = create_tv_fis();
+    char **terms = create_tv_terms(mp);
+    FrtPostingList **plists = create_tv_plists(mp, terms);
+    FrtOffset *offsets = create_tv_offsets(mp);
+    FrtDocument *doc = frt_doc_new();
+
+    fw = frt_fw_open(store, "_0", fis);
+    frt_fw_add_doc(fw, doc);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+    frt_fw_write_tv_index(fw); frt_fw_add_doc(fw, doc);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv_with_positions"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+    frt_fw_write_tv_index(fw); frt_fw_add_doc(fw, doc);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv_with_offsets"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+    frt_fw_write_tv_index(fw); frt_fw_add_doc(fw, doc);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv_with_positions_offsets"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+    frt_fw_write_tv_index(fw); frt_fw_add_doc(fw, doc);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv_with_positions"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv_with_offsets"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+    frt_fw_add_postings(fw, frt_fis_get_field(fis, rb_intern("tv_with_positions_offsets"))->number, plists, NUM_TERMS, offsets, NUM_TERMS);
+
+    frt_fw_write_tv_index(fw);
+    frt_fw_close(fw);
+    frt_doc_destroy(doc);
+
+    fr = frt_fr_open(store, "_0", fis);
+    Aiequal(5, fr->size);
+
+    tv = frt_fr_get_field_tv(fr, 0, frt_fis_get_field(fis, rb_intern("tv"))->number);
+    if (Apnotnull(tv)) {
+        Aiequal(frt_fis_get_field(fis, rb_intern("tv"))->number, tv->field_num);
+        Aiequal(NUM_TERMS, tv->term_cnt);
+        Aiequal(0, tv->offset_cnt);
+        for (i = 0; i < NUM_TERMS; i++) {
+            Asequal(terms[i], tv->terms[i].text);
+            Aiequal(i + 1, tv->terms[i].freq);
+            Apnull(tv->terms[i].positions);
+        }
+        Apnull(tv->offsets);
+    }
+    Apnull(frt_fr_get_field_tv(fr, 0, frt_fis_get_field(fis, rb_intern("tv_with_positions"))->number));
+    Apnull(frt_fr_get_field_tv(fr, 0, frt_fis_get_field(fis, rb_intern("tv_with_offsets"))->number));
+    Apnull(frt_fr_get_field_tv(fr, 0, frt_fis_get_field(fis, rb_intern("tv_with_positions_offsets"))->number));
+    frt_tv_destroy(tv);
+
+    tv = frt_fr_get_field_tv(fr, 3, frt_fis_get_field(fis, rb_intern("tv_with_positions_offsets"))->number);
+    if (Apnotnull(tv)) {
+        Aiequal(frt_fis_get_field(fis, rb_intern("tv_with_positions_offsets"))->number,
+                tv->field_num);
+        Aiequal(NUM_TERMS, tv->term_cnt);
+        Aiequal(NUM_TERMS, tv->offset_cnt);
+        for (i = 0; i < NUM_TERMS; i++) {
+            Asequal(terms[i], tv->terms[i].text);
+            Aiequal(i + 1, tv->terms[i].freq);
+            for (j = 1; j <= i; j++) {
+                Aiequal(j, tv->terms[i].positions[j]);
+            }
+        }
+        for (i = 0; i < NUM_TERMS; i++) {
+            Aiequal(i * 5, tv->offsets[i].start);
+            Aiequal(i * 5 + 4, tv->offsets[i].end);
+        }
+    }
+    frt_tv_destroy(tv);
+
+    /* test document's term vector */
+    tvs = frt_fr_get_tv(fr, 0);
+    Aiequal(1, tvs->size);
+    frt_h_destroy(tvs);
+
+    tvs = frt_fr_get_tv(fr, 4);
+    Aiequal(4, tvs->size);
+    tv = (FrtTermVector*)frt_h_get(tvs, (void *)rb_intern("tv2"));
+    Apnull(tv);
+    tv = (FrtTermVector*)frt_h_get(tvs, (void *)rb_intern("other"));
+    Apnull(tv);
+
+    tv = (FrtTermVector*)frt_h_get(tvs, (void *)rb_intern("tv_with_positions_offsets"));
+    if (Apnotnull(tv)) {
+        Aiequal(frt_fis_get_field(fis, rb_intern("tv_with_positions_offsets"))->number, tv->field_num);
+        Aiequal(NUM_TERMS, tv->term_cnt);
+        Aiequal(NUM_TERMS, tv->offset_cnt);
+        for (i = 0; i < NUM_TERMS; i++) {
+            Asequal(terms[i], tv->terms[i].text);
+            Aiequal(i + 1, tv->terms[i].freq);
+            for (j = 1; j <= i; j++) {
+                Aiequal(j, tv->terms[i].positions[j]);
+            }
+        }
+        for (i = 0; i < NUM_TERMS; i++) {
+            Aiequal(i * 5, tv->offsets[i].start);
+            Aiequal(i * 5 + 4, tv->offsets[i].end);
+        }
+    }
+
+    for (i = 0; i < NUM_TERMS; i++) {
+        char buf[100];
+        int len = sprintf(buf, "%s", tv->terms[i].text);
+        assert(strlen(tv->terms[i].text) < 100);
+        Aiequal(i, frt_tv_get_term_index(tv, buf));
+
+        /* make the word lexically less than it was but greater than any other
+         * word in the index that originally came before it. */
+        buf[len - 1]--;
+        buf[len    ] = '~';
+        buf[len + 1] = '\0';
+        Aiequal(-1, frt_tv_get_term_index(tv, buf));
+        Aiequal(i, frt_tv_scan_to_term_index(tv, buf));
+
+        /* make the word lexically more than it was by less than any other
+         * word in the index that originally came after it. */
+        buf[len - 1]++;
+        buf[len    ] = '.';
+        Aiequal(-1, frt_tv_get_term_index(tv, buf));
+        Aiequal(i + 1, frt_tv_scan_to_term_index(tv, buf));
+    }
+    Aiequal(-1, frt_tv_get_term_index(tv, "UnKnOwN TeRm"));
+    frt_h_destroy(tvs);
+    frt_fr_close(fr);
+    frt_fis_deref(fis);
+    frt_store_deref(store);
+}
+
+
+TestSuite *ts_term_vectors(TestSuite *suite)
+{
+    FrtMemoryPool *mp = frt_mp_new();
+
+    suite = ADD_SUITE(suite);
+
+    tst_run_test(suite, test_posting, mp);
+    frt_mp_reset(mp);
+    tst_run_test(suite, test_tv_single_doc, mp);
+    frt_mp_reset(mp);
+    tst_run_test(suite, test_tv_multi_doc, mp);
+    frt_mp_destroy(mp);
+    return suite;
+}

data/ext/isomorfeus_ferret_ext/test_test.c

@@ -0,0 +1,83 @@
+#include "frt_except.h"
+#include "test.h"
+
+static void raise_eof()
+{
+    FRT_RAISE(FRT_EOF_ERROR, "Error, end of file");
+}
+
+/**
+ * Test the basic test functions
+ */
+static void test_asserts(TestCase *tc, void *data)
+{
+    char *p[10];
+
+    int ia1[3] = { 1, 2, 3 };
+    int ia2[3] = { 1, 2, 3 };
+
+    static const char *sa1[10] = { "one", "two", "three" };
+    static const char *sa2[10] = { "one", "two", "three" };
+    (void)data; /* suppress unused argument warning */
+
+    Aaiequal(ia1, ia2, 3);
+    Aasequal(sa1, sa2, 3);
+    Aiequal(1, 1);
+    Asequal("String One", "String One");
+    Asnequal("String One", "String Two", 7);
+    Apnotnull(p);
+    Apequal(p, p);
+    Atrue(p != NULL);
+    Atrue(1);
+    Atrue(!0);
+    Assert(1 == 1, "%d != %d", 1, 1);
+    Araise(FRT_EOF_ERROR, &raise_eof, NULL);
+}
+
+static void raise_nothing(void *nothing)
+{
+    (void)nothing;
+}
+/**
+ * Test test failures. This method isn't called because we want 100% tests
+ * passing but if you want to check the tests work in case of failure, run
+ * this test.
+ */
+static void test_failures(TestCase *tc, void *data)
+{
+    void *q = NULL;
+    void *p = frt_emalloc(10);
+
+    int ia1[3] = { 1, 2, 3 };
+    int ia2[3] = { 1, 2, 4 };
+
+    static const char *sa1[10] = { "one", "two", "three" };
+    static const char *sa2[10] = { "one", "two", "there" };
+    (void)data; /* suppress unused argument warning */
+
+    Aaiequal(ia1, ia2, 3);
+    Aasequal(sa1, sa2, 3);
+    Aiequal(1, 2);
+    Asequal("String One", "String Two");
+    Asnequal("String One", "String Two", 8);
+    Apnotnull(q);
+    Apequal(p, q);
+    Atrue(1 == 2);
+    Assert(1 == 2, "%d != %d", 1, 2);
+    Atrue(0);
+    free(p);
+    Araise(EXCEPTION, &raise_eof, NULL);
+    Araise(EXCEPTION, &raise_nothing, NULL);
+}
+
+TestSuite *ts_test(TestSuite *suite)
+{
+    suite = ADD_SUITE(suite);
+
+    tst_run_test(suite, test_asserts, NULL);
+    if (false) {
+        tst_run_test(suite, test_failures, NULL);
+    }
+
+    return suite;
+}

data/ext/isomorfeus_ferret_ext/test_threading.c

@@ -0,0 +1,188 @@
+#include "frt_global.h"
+#include "frt_search.h"
+#include "frt_ind.h"
+#include "testhelper.h"
+#include "test.h"
+#include "pthread.h"
+
+extern char *num_to_str(int num);
+
+#define test_num(n, expected) num = num_to_str(n); Asequal(expected, num); free(num)
+
+static void test_number_to_str(TestCase *tc, void *data)
+{
+    char *num;
+    (void)data;
+    test_num(0, "zero");
+    test_num(9, "nine");
+    test_num(10, "ten");
+    test_num(13, "thirteen");
+    test_num(19, "nineteen");
+    test_num(20, "twenty");
+    test_num(21, "twenty one");
+    test_num(99, "ninety nine");
+    test_num(100, "one hundred");
+    test_num(101, "one hundred and one");
+    test_num(111, "one hundred and eleven");
+    test_num(1111, "one thousand one hundred and eleven");
+    test_num(22222, "twenty two thousand two hundred and twenty two");
+    test_num(333333, "three hundred and thirty three thousand three hundred and thirty three");
+    test_num(8712387, "eight million seven hundred and twelve thousand three hundred and eighty seven");
+    test_num(1000000000, "one billion");
+    test_num(-8712387, "negative eight million seven hundred and twelve thousand three hundred and eighty seven");
+
+}
+
+void dummy_log(const void *fmt, ...) {(void)fmt;}
+#define ITERATIONS 10
+#define NTHREADS 10
+#ifdef FRT_HAS_VARARGS
+#define tlog(...)
+#else
+#define tlog dummy_log
+#endif
+/*#define tlog printf */
+
+static void do_optimize(FrtIndex *index)
+{
+    tlog("Optimizing the index\n");
+    frt_index_optimize(index);
+}
+
+static void do_delete_doc(FrtIndex *index)
+{
+    int size;
+    if ((size = frt_index_size(index)) > 0) {
+        int doc_num = rand() % size;
+        tlog("Deleting %d from index which has%s deletions\n",
+             doc_num, (frt_index_has_del(index) ? "" : " no"));
+        if (frt_index_is_deleted(index, doc_num)) {
+            tlog("document was already deleted\n");
+        } else {
+            frt_index_delete(index, doc_num);
+        }
+    }
+}
+
+ID id;
+ID contents;
+
+static void do_add_doc(FrtIndex *index)
+{
+    FrtDocument *doc = frt_doc_new();
+    int n = rand();
+
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(id), frt_strfmt("%d", n)))->destroy_data = true;
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(contents), num_to_str(n)))->destroy_data = true;
+    tlog("Adding %d\n", n);
+    frt_index_add_doc(index, doc);
+   frt_doc_destroy(doc);
+}
+
+static void do_search(FrtIndex *index)
+{
+    int n = rand(), i;
+    char *query = num_to_str(n);
+    FrtTopDocs *td;
+
+    tlog("Searching for %d\n", n);
+
+    frt_mutex_lock(&index->mutex);
+    td = frt_index_search_str(index, query, 0, 3, NULL, NULL, NULL);
+    free(query);
+    for (i = 0; i < td->size; i++) {
+        FrtHit *hit = td->hits[i];
+        FrtDocument *doc = frt_index_get_doc(index, hit->doc);
+        tlog("Hit for %d: %s - %f\n", hit->doc, frt_doc_get_field(doc, id)->data[0], hit->score);
+       frt_doc_destroy(doc);
+    }
+    tlog("Searched for %d: total = %d\n", n, td->total_hits);
+    frt_mutex_unlock(&index->mutex);
+
+    frt_td_destroy(td);
+}
+
+static void *indexing_thread(void *p)
+{
+    int i, choice;
+    FrtIndex *index = (FrtIndex *)p;
+
+    for (i = 0; i < ITERATIONS; i++) {
+        choice = rand() % 1000;
+
+        if (choice > 999) {
+            do_optimize(index);
+        } else if (choice > 900) {
+            do_delete_doc(index);
+        } else if (choice > 700) {
+            do_search(index);
+        } else {
+            do_add_doc(index);
+        }
+    }
+    return NULL;
+}
+
+static void test_threading_test(TestCase *tc, void *data)
+{
+    FrtIndex *index = (FrtIndex *)data;
+    (void)data;
+    (void)tc;
+    indexing_thread(index);
+}
+
+static void test_threading(TestCase *tc, void *data)
+{
+    int i;
+    pthread_t thread_id[NTHREADS];
+    FrtIndex *index = (FrtIndex *)data;
+    (void)data;
+    (void)tc;
+
+    for(i=0; i < NTHREADS; i++) {
+        pthread_create(&thread_id[i], NULL, &indexing_thread, index );
+    }
+
+    for(i=0; i < NTHREADS; i++) {
+        pthread_join(thread_id[i], NULL);
+    }
+}
+
+TestSuite *ts_threading(TestSuite *suite)
+{
+    id = rb_intern("id");
+    contents = rb_intern("contents");
+
+    FrtAnalyzer *a = frt_letter_analyzer_new(true);
+    FrtStore *store = frt_open_fs_store("./test/testdir/store");
+    FrtIndex *index;
+    FrtHashSet *def_fields = frt_hs_new_ptr(NULL);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
+                              FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    frt_fis_add_field(fis, frt_fi_new(id, FRT_STORE_YES, FRT_INDEX_UNTOKENIZED,
+                              FRT_TERM_VECTOR_YES));
+    frt_index_create(store, fis);
+    frt_fis_deref(fis);
+
+    frt_hs_add(def_fields, (void *)contents);
+    store->clear_all(store);
+    index = frt_index_new(store, a, def_fields, true);
+    frt_hs_destroy(def_fields);
+
+    suite = ADD_SUITE(suite);
+
+    frt_store_deref(store);
+    frt_a_deref(a);
+
+    tst_run_test(suite, test_number_to_str, NULL);
+    tst_run_test(suite, test_threading_test, index);
+    // tst_run_test(suite, test_threading, index);
+
+    frt_index_destroy(index);
+
+    store = frt_open_fs_store("./test/testdir/store");
+    store->clear_all(store);
+    frt_store_deref(store);
+
+    return suite;
+}