isomorfeus-ferret 0.12.0

data/ext/isomorfeus_ferret_ext/test_lang.c

@@ -0,0 +1,74 @@
+#include "test.h"
+#include "frt_lang.h"
+
+typedef void *(*alloc_ft)(size_t);
+
+static void huge_emalloc(void *data)
+{
+    (void)data;
+    frt_emalloc((size_t)-1);
+}
+
+static void huge_ecalloc(void *data)
+{
+    (void)data;
+    frt_ecalloc((size_t)-1);
+}
+
+static void huge_erealloc(void *data)
+{
+    char *p = NULL;
+    (void)data;
+    frt_erealloc(p, (size_t)-1);
+}
+
+static void test_emalloc(TestCase *tc, void *data)
+{
+    char *p;
+    (void)data; /* suppress warning */
+
+    p = (char *)frt_emalloc(100);
+    Apnotnull(p);
+    free(p);
+
+    Araise(FRT_MEM_ERROR, huge_emalloc, NULL);
+}
+
+static void test_ecalloc(TestCase *tc, void *data)
+{
+    int i;
+    char *p;
+    (void)data; /* suppress warning */
+
+    p = (char *)frt_ecalloc(100);
+    Apnotnull(p);
+    for (i = 0; i < 100; ++i) {
+        Aiequal(p[i], 0);
+    }
+    free(p);
+
+    Araise(FRT_MEM_ERROR, huge_ecalloc, NULL);
+}
+
+static void test_erealloc(TestCase *tc, void *data)
+{
+    char *p = NULL;
+    (void)data; /* suppress warning */
+
+    p = (char *)frt_erealloc(p, 100);
+    Apnotnull(p);
+    free(p);
+
+    Araise(FRT_MEM_ERROR, huge_erealloc, NULL);
+}
+
+TestSuite *ts_lang(TestSuite *suite)
+{
+    suite = ADD_SUITE(suite);
+
+    tst_run_test(suite, test_emalloc, NULL);
+    tst_run_test(suite, test_ecalloc, NULL);
+    tst_run_test(suite, test_erealloc, NULL);
+
+    return suite;
+}

data/ext/isomorfeus_ferret_ext/test_mempool.c

@@ -0,0 +1,102 @@
+#include <stdlib.h>
+#include "test.h"
+#include "frt_mempool.h"
+
+
+static void test_mp_default_capa(TestCase *tc, void *data)
+{
+    FrtMemoryPool *mp = frt_mp_new();
+    (void)data;
+    Aiequal(FRT_MP_INIT_CAPA, mp->buf_capa);
+    frt_mp_destroy(mp);
+}
+
+struct MemChecker {
+    int size;
+    char vals[1];
+};
+
+#define NUM_ALLOCS 10000
+#define MAX_SIZE 100
+
+static void do_mp_test(TestCase *tc, FrtMemoryPool *mp)
+{
+    int i, j;
+    int max_necessary;
+    int total_bytes = 0;
+    struct MemChecker *mem_checkers[NUM_ALLOCS];
+
+    for (i = 0; i < NUM_ALLOCS; i++) {
+        int size = rand() % MAX_SIZE;
+        total_bytes += size + sizeof(int);
+        mem_checkers[i] = (struct MemChecker *)frt_mp_alloc(mp, size + sizeof(int));
+        mem_checkers[i]->size =  size;
+    }
+    for (i = 0; i < NUM_ALLOCS; i++) {
+        for (j = 0; j < mem_checkers[i]->size; j++) {
+            mem_checkers[i]->vals[j] = (char)(i & 0xFF);
+        }
+    }
+    for (i = 0; i < NUM_ALLOCS; i++) {
+        for (j = 0; j < mem_checkers[i]->size; j++) {
+            if (mem_checkers[i]->vals[j] != (char)(i & 0xFF)) {
+                Aiequal(i & 0xFF, mem_checkers[i]->vals[j]);
+            }
+        }
+    }
+    if (!Atrue(total_bytes < (mp->buf_alloc * mp->chunk_size))) {
+        Tmsg("total bytes allocated <%d> > memory used <%d>",
+             total_bytes, mp->buf_alloc * mp->chunk_size);
+    }
+
+    max_necessary =
+        (mp->buf_alloc - 1) * (mp->chunk_size - (MAX_SIZE+sizeof(int)));
+    if (!Atrue(total_bytes > max_necessary)) {
+        Tmsg("total bytes allocated <%d> < max memory needed <%d>",
+             total_bytes, max_necessary);
+    }
+}
+
+static void test_mp_dup(TestCase *tc, void *data)
+{
+    FrtMemoryPool *mp = frt_mp_new_capa(2000, 16);
+    (void)data;
+
+    do_mp_test(tc, mp);
+    frt_mp_reset(mp);
+    do_mp_test(tc, mp);
+    frt_mp_destroy(mp);
+}
+
+static void test_mp_alloc(TestCase *tc, void *data)
+{
+    FrtMemoryPool *mp = frt_mp_new_capa(20, 16);
+    char *t;
+    (void)data;
+
+    t = frt_mp_strdup(mp, "012345678901234");
+
+    Asequal("012345678901234", t);
+    Aiequal(strlen(t) + 1, frt_mp_used(mp));
+
+    t = (char *)frt_mp_memdup(mp, "012345678901234", 10);
+    Asnequal("012345678901234", t, 10);
+    Aiequal(30, frt_mp_used(mp));
+
+    t = (char *)frt_mp_strndup(mp, "012345678", 9);
+    Asequal("012345678", t);
+    Aiequal(40, frt_mp_used(mp)); /* Stays in the same chunk */
+
+    frt_mp_destroy(mp);
+}
+
+TestSuite *ts_mem_pool(TestSuite *suite)
+{
+    suite = ADD_SUITE(suite);
+
+    tst_run_test(suite, test_mp_default_capa, NULL);
+    tst_run_test(suite, test_mp_alloc, NULL);
+    tst_run_test(suite, test_mp_dup, NULL);
+
+    return suite;
+}

data/ext/isomorfeus_ferret_ext/test_multimapper.c

@@ -0,0 +1,64 @@
+#include "testhelper.h"
+#include "frt_multimapper.h"
+#include "test.h"
+
+static void test_multimapper(TestCase *tc, void *data)
+{
+    char text[] = "abc cabc abd cabcd";
+    char dest[1000];
+    FrtMultiMapper *mapper = frt_mulmap_new();
+    (void)data;
+
+    frt_mulmap_add_mapping(mapper, "abc", "hello");
+
+    frt_mulmap_compile(mapper);
+    Aiequal(24, frt_mulmap_map_len(mapper, dest, text, 1000));
+    Asequal("hello chello abd chellod", frt_mulmap_map(mapper, dest, text, 1000));
+    Asequal("hello chello abd chel", frt_mulmap_map(mapper, dest, text, 22));
+    Aiequal(21, frt_mulmap_map_len(mapper, dest, text, 22));
+    Asequal("hello chello a", frt_mulmap_map(mapper, dest, text, 15));
+
+    frt_mulmap_add_mapping(mapper, "abcd", "hello");
+    frt_mulmap_compile(mapper);
+    Asequal("hello chello abd chellod", frt_mulmap_map(mapper, dest, text, 1000));
+
+    frt_mulmap_add_mapping(mapper, "cab", "taxi");
+    frt_mulmap_compile(mapper);
+    Asequal("hello taxic abd taxicd", frt_mulmap_map(mapper, dest, text, 1000));
+
+    frt_mulmap_destroy(mapper);
+}
+
+static void test_multimapper_utf8(TestCase *tc, void *data)
+{
+    char text[] = "zàáâãäåāăz";
+    char dest[1000];
+    char *dest_dynamic;
+    FrtMultiMapper *mapper = frt_mulmap_new();
+    (void)data;
+
+    frt_mulmap_add_mapping(mapper, "à", "a");
+    frt_mulmap_add_mapping(mapper, "á", "a");
+    frt_mulmap_add_mapping(mapper, "â", "a");
+    frt_mulmap_add_mapping(mapper, "ã", "a");
+    frt_mulmap_add_mapping(mapper, "ä", "a");
+    frt_mulmap_add_mapping(mapper, "å", "a");
+    frt_mulmap_add_mapping(mapper, "ā", "a");
+    frt_mulmap_add_mapping(mapper, "ă", "a");
+    frt_mulmap_compile(mapper);
+    Asequal("zaaaaaaaaz", frt_mulmap_map(mapper, dest, text, 1000));
+    dest_dynamic = frt_mulmap_dynamic_map(mapper, text);
+    Asequal("zaaaaaaaaz", dest_dynamic);
+    free(dest_dynamic);
+    frt_mulmap_destroy(mapper);
+}
+
+TestSuite *ts_multimapper(TestSuite *suite)
+{
+    suite = ADD_SUITE(suite);
+
+    tst_run_test(suite, test_multimapper, NULL);
+    tst_run_test(suite, test_multimapper_utf8, NULL);
+
+    return suite;
+}

data/ext/isomorfeus_ferret_ext/test_priorityqueue.c

@@ -0,0 +1,213 @@
+#include "frt_priorityqueue.h"
+#include "frt_global.h"
+#include <string.h>
+#include "test.h"
+
+/**
+ * Simple string less than function used for testing adding of strings to the
+ * priority queue.
+ */
+static bool str_lt(void *p1, void *p2)
+{
+    return (strcmp((char *) p1, (char *) p2) < 0);
+}
+
+static bool str_lt_rev(void *p1, void *p2)
+{
+    return (strcmp((char *) p1, (char *) p2) > 0);
+}
+
+/**
+ * Test basic FrtPriorityQueue functions.
+ */
+static void test_pq(TestCase *tc, void *data)
+{
+    char *tmp;
+    FrtPriorityQueue *pq = frt_pq_new(4, (frt_lt_ft)&str_lt, &free);
+    (void)data; /* suppress unused argument warning */
+
+    Aiequal(0, pq->size);
+    Aiequal(4, pq->capa);
+    frt_pq_push(pq, frt_estrdup("bword"));
+    Aiequal(1, pq->size);
+    Asequal("bword", (char *) frt_pq_top(pq));
+    frt_pq_push(pq, frt_estrdup("cword"));
+    Aiequal(2, pq->size);
+    Asequal("bword", (char *) frt_pq_top(pq));
+    frt_pq_push(pq, frt_estrdup("aword"));
+    Aiequal(3, pq->size);
+    Asequal("aword", (char *) frt_pq_top(pq));
+    frt_pq_push(pq, frt_estrdup("dword"));
+    Aiequal(4, pq->size);
+    Asequal("aword", (char *) frt_pq_top(pq));
+    Asequal("aword", tmp = (char *) frt_pq_pop(pq));
+    Aiequal(3, pq->size);
+    free(tmp);
+    Asequal("bword", tmp = (char *) frt_pq_pop(pq));
+    Aiequal(2, pq->size);
+    free(tmp);
+    Asequal("cword", tmp = (char *) frt_pq_pop(pq));
+    Aiequal(1, pq->size);
+    free(tmp);
+    Asequal("dword", tmp = (char *) frt_pq_pop(pq));
+    Aiequal(0, pq->size);
+    free(tmp);
+    frt_pq_destroy(pq);
+}
+
+/**
+ * Test basic FrtPriorityQueue functions.
+ */
+static void test_pq_reverse(TestCase *tc, void *data)
+{
+    char *tmp;
+    FrtPriorityQueue *pq = frt_pq_new(4, (frt_lt_ft)&str_lt_rev, &free);
+    (void)data; /* suppress unused argument warning */
+
+    Aiequal(0, pq->size);
+    Aiequal(4, pq->capa);
+    frt_pq_push(pq, frt_estrdup("bword"));
+    Aiequal(1, pq->size);
+    Asequal("bword", (char *) frt_pq_top(pq));
+    frt_pq_push(pq, frt_estrdup("cword"));
+    Aiequal(2, pq->size);
+    Asequal("cword", (char *) frt_pq_top(pq));
+    frt_pq_push(pq, frt_estrdup("aword"));
+    Aiequal(3, pq->size);
+    Asequal("cword", (char *) frt_pq_top(pq));
+    frt_pq_push(pq, frt_estrdup("dword"));
+    Aiequal(4, pq->size);
+    Asequal("dword", (char *) frt_pq_top(pq));
+    Asequal("dword", tmp = (char *) frt_pq_pop(pq));
+    Aiequal(3, pq->size);
+    free(tmp);
+    Asequal("cword", tmp = (char *) frt_pq_pop(pq));
+    Aiequal(2, pq->size);
+    free(tmp);
+    Asequal("bword", tmp = (char *) frt_pq_pop(pq));
+    Aiequal(1, pq->size);
+    free(tmp);
+    Asequal("aword", tmp = (char *) frt_pq_pop(pq));
+    Aiequal(0, pq->size);
+    free(tmp);
+    frt_pq_destroy(pq);
+}
+
+/**
+ * Free mock used to test that the FrtPriorityQueue is being destroyed correctly
+ * including it's elements
+ */
+static void pq_free_mock(void *p)
+{
+    char *str = (char *) p;
+    strcpy(str, "free");
+}
+
+/**
+ * Test pq_clear function
+ */
+static void test_pq_clear(TestCase *tc, void *data)
+{
+    char word1[10] = "word1";
+    char word2[10] = "word2";
+    char word3[10] = "word3";
+    FrtPriorityQueue *pq = frt_pq_new(3, (frt_lt_ft)&str_lt, &pq_free_mock);
+    (void)data; /* suppress unused argument warning */
+
+    frt_pq_push(pq, word1);
+    frt_pq_push(pq, word2);
+    frt_pq_push(pq, word3);
+    Aiequal(3, pq->size);
+    frt_pq_clear(pq);
+    Aiequal(0, pq->size);
+    Asequal("free", word1);
+    Asequal("free", word2);
+    Asequal("free", word3);
+    frt_pq_destroy(pq);
+}
+
+/**
+ * Test that FrtPriorityQueue will handle insert overflow. That is, when you
+ * insert more than the PriorityQueue's capacity, the extra elements that drop
+ * off the bottom are destroyed.
+ */
+static void test_pq_insert_overflow(TestCase *tc, void *data)
+{
+    char word1[10] = "word1";
+    char word2[10] = "word2";
+    char word3[10] = "word3";
+    char word4[10] = "word4";
+    char word5[10] = "word5";
+    char word6[10] = "word6";
+    FrtPriorityQueue *pq = frt_pq_new(3, (frt_lt_ft)&str_lt, &pq_free_mock);
+    (void)data; /* suppress unused argument warning */
+
+    Aiequal(FRT_PQ_ADDED, frt_pq_insert(pq, word2));
+    Aiequal(FRT_PQ_ADDED, frt_pq_insert(pq, word3));
+    Aiequal(FRT_PQ_ADDED, frt_pq_insert(pq, word4));
+    Aiequal(FRT_PQ_INSERTED, frt_pq_insert(pq, word5));
+    Aiequal(FRT_PQ_INSERTED, frt_pq_insert(pq, word6));
+    Aiequal(FRT_PQ_DROPPED, frt_pq_insert(pq, word1));
+    Aiequal(3, pq->size);
+    Asequal("free", word1);
+    Asequal("free", word2);
+    Asequal("free", word3);
+    Asequal("word4", word4);
+    Asequal("word5", word5);
+    Asequal("word6", word6);
+    frt_pq_clear(pq);
+    Aiequal(0, pq->size);
+    Asequal("free", word4);
+    Asequal("free", word5);
+    Asequal("free", word6);
+    frt_pq_destroy(pq);
+}
+
+/**
+ * Stress test the PriorityQueue. Make PQ_STRESS_SIZE much larger if you want
+ * to really stress test PriorityQueue.
+ */
+#define PQ_STRESS_SIZE 1000
+static void stress_pq(TestCase *tc, void *data)
+{
+    int i;
+    char buf[100], *prev, *curr;
+    FrtPriorityQueue *pq = frt_pq_new(PQ_STRESS_SIZE, (frt_lt_ft)&str_lt, &free);
+    (void)data; /* suppress unused argument warning */
+
+    for (i = 0; i < PQ_STRESS_SIZE; i++) {
+        sprintf(buf, "<%d>", rand());
+        frt_pq_push(pq, frt_estrdup(buf));
+    }
+    Aiequal(PQ_STRESS_SIZE, pq->size);
+
+    prev = (char *) frt_pq_pop(pq);
+    for (i = 0; i < PQ_STRESS_SIZE - 1; i++) {
+        curr = (char *) frt_pq_pop(pq);
+        if (str_lt(curr, prev) == true) {
+            Assert(false, "previous should be less than or equal to current");
+            Tmsg("%d: %s, %s\n", i, prev, curr);
+        }
+        free(prev);
+        prev = curr;
+    }
+    free(prev);
+    frt_pq_clear(pq);
+    frt_pq_destroy(pq);
+}
+
+/**
+ * PriorityQueue's test suite
+ */
+TestSuite *ts_priorityqueue(TestSuite *suite)
+{
+    suite = ADD_SUITE(suite);
+
+    tst_run_test(suite, test_pq, NULL);
+    tst_run_test(suite, test_pq_reverse, NULL);
+    tst_run_test(suite, test_pq_clear, NULL);
+    tst_run_test(suite, test_pq_insert_overflow, NULL);
+    tst_run_test(suite, stress_pq, NULL);
+
+    return suite;
+}

data/ext/isomorfeus_ferret_ext/test_q_const_score.c

@@ -0,0 +1,84 @@
+#include "frt_search.h"
+#include "test.h"
+
+static FrtSymbol num;
+
+extern void prepare_filter_index(FrtStore *store);
+
+static void test_const_score_query(TestCase *tc, void *data)
+{
+    FrtSearcher *searcher = (FrtSearcher *)data;
+    FrtQuery *q;
+    q = frt_csq_new_nr(frt_rfilt_new(num, "2", "6", true, true));
+    tst_check_hits(tc, searcher, q, "2,3,4,5,6", -1);
+    frt_q_deref(q);
+    q = frt_csq_new_nr(frt_rfilt_new(num, "2", "6", true, false));
+    tst_check_hits(tc, searcher, q, "2,3,4,5", -1);
+    frt_q_deref(q);
+    q = frt_csq_new_nr(frt_rfilt_new(num, "2", "6", false, true));
+    tst_check_hits(tc, searcher, q, "3,4,5,6", -1);
+    frt_q_deref(q);
+    q = frt_csq_new_nr(frt_rfilt_new(num, "2", "6", false, false));
+    tst_check_hits(tc, searcher, q, "3,4,5", -1);
+    frt_q_deref(q);
+    q = frt_csq_new_nr(frt_rfilt_new(num, "6", NULL, true, false));
+    tst_check_hits(tc, searcher, q, "6,7,8,9", -1);
+    frt_q_deref(q);
+    q = frt_csq_new_nr(frt_rfilt_new(num, "6", NULL, false, false));
+    tst_check_hits(tc, searcher, q, "7,8,9", -1);
+    frt_q_deref(q);
+    q = frt_csq_new_nr(frt_rfilt_new(num, NULL, "2", false, true));
+    tst_check_hits(tc, searcher, q, "0,1,2", -1);
+    frt_q_deref(q);
+    q = frt_csq_new_nr(frt_rfilt_new(num, NULL, "2", false, false));
+    tst_check_hits(tc, searcher, q, "0,1", -1);
+    frt_q_deref(q);
+}
+
+static void test_const_score_query_hash(TestCase *tc, void *data)
+{
+    FrtFilter *f;
+    FrtQuery *q1, *q2;
+    (void)data;
+    f = frt_rfilt_new(num, "2", "6", true, true);
+    q1 = frt_csq_new_nr(f);
+    q2 = frt_csq_new(f);
+
+    Assert(frt_q_eq(q1, q1), "Test same queries are equal");
+    Aiequal(frt_q_hash(q1), frt_q_hash(q2));
+    Assert(frt_q_eq(q1, q2), "Queries are equal");
+    frt_q_deref(q2);
+
+    q2 = frt_csq_new_nr(frt_rfilt_new(num, "2", "6", true, true));
+    Aiequal(frt_q_hash(q1), frt_q_hash(q2));
+    Assert(frt_q_eq(q1, q2), "Queries are equal");
+    frt_q_deref(q2);
+
+    q2 = frt_csq_new_nr(frt_rfilt_new(num, "3", "6", true, true));
+    Assert(frt_q_hash(q1) != frt_q_hash(q2), "Queries are not equal");
+    Assert(!frt_q_eq(q1, q2), "Queries are not equal");
+    frt_q_deref(q2);
+    frt_q_deref(q1);
+}
+
+TestSuite *ts_q_const_score(TestSuite *suite)
+{
+    FrtStore *store = frt_open_ram_store();
+    FrtIndexReader *ir;
+    FrtSearcher *searcher;
+
+    num = rb_intern("num");
+
+    suite = ADD_SUITE(suite);
+
+    prepare_filter_index(store);
+    ir = frt_ir_open(store);
+    searcher = frt_isea_new(ir);
+
+    tst_run_test(suite, test_const_score_query, (void *)searcher);
+    tst_run_test(suite, test_const_score_query_hash, NULL);
+
+    frt_store_deref(store);
+    frt_searcher_close(searcher);
+    return suite;
+}

data/ext/isomorfeus_ferret_ext/test_q_filtered.c

@@ -0,0 +1,61 @@
+#include "frt_search.h"
+#include "test.h"
+
+static FrtSymbol num, flipflop;
+
+extern void prepare_filter_index(FrtStore *store);
+
+static void test_filtered_query(TestCase *tc, void *data)
+{
+    FrtSearcher *searcher = (FrtSearcher *)data;
+    FrtQuery *q;
+    q = frt_fq_new(frt_maq_new(), frt_rfilt_new(num, "2", "6", true, true));
+    tst_check_hits(tc, searcher, q, "2,3,4,5,6", -1);
+    frt_q_deref(q);
+    q = frt_fq_new(frt_tq_new(flipflop, "on"), frt_rfilt_new(num, "2", "6", true, true));
+    tst_check_hits(tc, searcher, q, "2,4,6", -1);
+    frt_q_deref(q);
+    q = frt_fq_new(frt_maq_new(), frt_rfilt_new(num, "2", "6", true, false));
+    tst_check_hits(tc, searcher, q, "2,3,4,5", -1);
+    frt_q_deref(q);
+    q = frt_fq_new(frt_maq_new(), frt_rfilt_new(num, "2", "6", false, true));
+    tst_check_hits(tc, searcher, q, "3,4,5,6", -1);
+    frt_q_deref(q);
+    q = frt_fq_new(frt_maq_new(), frt_rfilt_new(num, "2", "6", false, false));
+    tst_check_hits(tc, searcher, q, "3,4,5", -1);
+    frt_q_deref(q);
+    q = frt_fq_new(frt_maq_new(), frt_rfilt_new(num, "6", NULL, true, false));
+    tst_check_hits(tc, searcher, q, "6,7,8,9", -1);
+    frt_q_deref(q);
+    q = frt_fq_new(frt_maq_new(), frt_rfilt_new(num, "6", NULL, false, false));
+    tst_check_hits(tc, searcher, q, "7,8,9", -1);
+    frt_q_deref(q);
+    q = frt_fq_new(frt_maq_new(), frt_rfilt_new(num, NULL, "2", false, true));
+    tst_check_hits(tc, searcher, q, "0,1,2", -1);
+    frt_q_deref(q);
+    q = frt_fq_new(frt_maq_new(), frt_rfilt_new(num, NULL, "2", false, false));
+    tst_check_hits(tc, searcher, q, "0,1", -1);
+    frt_q_deref(q);
+}
+
+TestSuite *ts_q_filtered(TestSuite *suite)
+{
+    FrtStore *store = frt_open_ram_store();
+    FrtIndexReader *ir;
+    FrtSearcher *searcher;
+
+    num      = rb_intern("num");
+    flipflop = rb_intern("flipflop");
+
+    suite = ADD_SUITE(suite);
+
+    prepare_filter_index(store);
+    ir = frt_ir_open(store);
+    searcher = frt_isea_new(ir);
+
+    tst_run_test(suite, test_filtered_query, (void *)searcher);
+
+    frt_store_deref(store);
+    searcher->close(searcher);
+    return suite;
+}