isomorfeus-ferret 0.12.0

data/ext/isomorfeus_ferret_ext/benchmark.c

@@ -0,0 +1,223 @@
+#include <unistd.h>
+#include <string.h>
+#include "ruby.h"
+#include "frt_global.h"
+#include "benchmark.h"
+#include "benchmarks_all.h"
+#include "word_list.h"
+
+extern VALUE mFerret;
+static VALUE mBenchmark;
+
+#if (defined POSH_OS_WIN32 || defined POSH_OS_WIN64)
+
+#define RUSAGE_SELF		0
+
+struct rusage
+{
+	struct timeval ru_utime;	/* user time used */
+	struct timeval ru_stime;	/* system time used */
+};
+
+int getrusage(int who, struct rusage *rusage)
+{
+	FILETIME	starttime;
+	FILETIME	exittime;
+	FILETIME	kerneltime;
+	FILETIME	usertime;
+	ULARGE_INTEGER li;
+
+	if (who != RUSAGE_SELF) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	if (rusage == (struct rusage *) NULL) {
+		errno = EFAULT;
+		return -1;
+	}
+	memset(rusage, 0, sizeof(struct rusage));
+	if (GetProcessTimes(GetCurrentProcess(), &starttime, &exittime, &kerneltime, &usertime) == 0) {
+		fprintf(stderr, "getrusage: error in GetProcessTimes %lu\n", GetLastError());
+		return -1;
+	}
+
+	memcpy(&li, &kerneltime, sizeof(FILETIME));
+	li.QuadPart /= 10L;
+	rusage->ru_stime.tv_sec = li.QuadPart / 1000000L;
+	rusage->ru_stime.tv_usec = li.QuadPart % 1000000L;
+
+	memcpy(&li, &usertime, sizeof(FILETIME));
+	li.QuadPart /= 10L;
+	rusage->ru_utime.tv_sec = li.QuadPart / 1000000L;
+	rusage->ru_utime.tv_usec = li.QuadPart % 1000000L;
+    return 0;
+}
+#else
+#include <sys/time.h>
+#include <sys/resource.h>
+#endif
+
+static int bmtcmp(const void *p1, const void *p2)
+{
+    BenchMarkTimes *bmt1 = *(BenchMarkTimes **)p1;
+    BenchMarkTimes *bmt2 = *(BenchMarkTimes **)p2;
+
+    if (bmt1->rtime > bmt2->rtime) return 1;
+    else if (bmt1->rtime < bmt2->rtime) return -1;
+    else return 0;
+}
+
+void bm_add(BenchMark *benchmark, bm_run_ft run, const char *name)
+{
+    BenchMarkUnit *unit =
+        (BenchMarkUnit *)frt_emalloc(sizeof(BenchMarkUnit) +
+                                 benchmark->count * sizeof(BenchMarkTimes *));
+    int i;
+    unit->name = frt_estrdup(name);
+    unit->run = run;
+    unit->next = NULL;
+    if (benchmark->count > 1) {
+        for (i = 0; i < benchmark->count; i++) {
+            unit->times[i] = FRT_ALLOC(BenchMarkTimes);
+        }
+    }
+    if (benchmark->tail) {
+        benchmark->tail = benchmark->tail->next = unit;
+    }
+    else {
+        benchmark->tail = benchmark->head = unit;
+    }
+}
+
+static void bm_clear(BenchMark *benchmark)
+{
+    BenchMarkUnit *unit, *next = benchmark->head;
+    while (NULL != (unit = next)) {
+        next = unit->next;
+        if (benchmark->count > 1) {
+            int i;
+            for (i = 0; i < benchmark->count; i++) {
+                free(unit->times[i]);
+            }
+        }
+        free(unit->name);
+        free(unit);
+    }
+    benchmark->head = benchmark->tail = NULL;
+}
+
+#define TVAL_TO_SEC(before, after) \
+  ((double)after.tv_sec  + ((double)after.tv_usec/1000000)) - \
+  ((double)before.tv_sec + ((double)before.tv_usec/1000000))
+
+static void bm_single_run(BenchMarkUnit *unit, BenchMarkTimes *bm_times)
+{
+    struct timeval tv_before, tv_after;
+    struct rusage ru_before, ru_after;
+
+    if (gettimeofday(&tv_before, NULL) == -1)
+        FRT_RAISE(FRT_UNSUPPORTED_ERROR, "gettimeofday failed\n");
+    getrusage(RUSAGE_SELF, &ru_before);
+
+    unit->run();
+
+    if (gettimeofday(&tv_after, NULL) == -1)
+        FRT_RAISE(FRT_UNSUPPORTED_ERROR, "gettimeofday failed\n");
+    getrusage(RUSAGE_SELF, &ru_after);
+
+    bm_times->rtime = TVAL_TO_SEC(tv_before, tv_after);
+    bm_times->utime = TVAL_TO_SEC(ru_before.ru_utime, ru_after.ru_utime);
+    bm_times->stime = TVAL_TO_SEC(ru_before.ru_stime, ru_after.ru_stime);
+}
+
+#define DO_SETUP(bm) if (bm->setup) bm->setup();
+#define DO_TEARDOWN(bm) if (bm->teardown) bm->teardown();
+
+static void bm_run(BenchMark *benchmark)
+{
+    int i;
+    BenchMarkUnit *unit;
+    int max_name_len = 0;
+    char fmt[40];
+    int start = 0, end = benchmark->count;
+    if (benchmark->discard) {
+        start += benchmark->discard;
+        end   -= benchmark->discard;
+    }
+    if (benchmark->count > 1) {
+        for (i = 0; i < benchmark->count; i++) {
+            DO_SETUP(benchmark);
+            for (unit = benchmark->head; unit; unit = unit->next) {
+                bm_single_run(unit, unit->times[i]);
+            }
+            DO_TEARDOWN(benchmark);
+        }
+        for (unit = benchmark->head; unit; unit = unit->next) {
+            double rtime = 0.0, utime = 0.0, stime = 0.0;
+            int result_count = end - start;
+            /* we only need to sort if we are discarding outliers */
+            if (benchmark->discard > 0) {
+                qsort(unit->times, benchmark->count,
+                      sizeof(BenchMarkTimes*), &bmtcmp);
+            }
+
+            for (i = start; i < end; i++) {
+                rtime += unit->times[i]->rtime;
+                utime += unit->times[i]->utime;
+                stime += unit->times[i]->stime;
+            }
+            unit->final_times.utime = utime/result_count;
+            unit->final_times.stime = stime/result_count;
+            unit->final_times.rtime = rtime/result_count;
+        }
+    }
+    else {
+        DO_SETUP(benchmark);
+        for (unit = benchmark->head; unit; unit = unit->next) {
+            bm_single_run(unit, &(unit->final_times));
+        }
+        DO_TEARDOWN(benchmark);
+    }
+
+    /* get maximum unit name length for print out */
+    for (unit = benchmark->head; unit; unit = unit->next) {
+        int name_len = (int)strlen(unit->name);
+        if (name_len > max_name_len) {
+            max_name_len = name_len;
+        }
+    }
+
+    for (i = 0; i < max_name_len; i++) putchar(' ');
+    puts("\t         user     system       real");
+    sprintf(fmt, "\t%%%ds %%10.6lf %%10.6lf %%10.6lf\n", max_name_len);
+    for (unit = benchmark->head; unit; unit = unit->next) {
+        printf(fmt,
+               unit->name,
+               unit->final_times.utime,
+               unit->final_times.stime,
+               unit->final_times.rtime);
+    }
+}
+
+static VALUE frb_bm_run_all(VALUE v) {
+    int i;
+    BenchMark benchmark;
+    benchmark.head = benchmark.tail = NULL;
+
+    for (i = 0; i < FRT_NELEMS(all_benchmarks); i++) {
+        printf("\nBenching [%s]...\n", all_benchmarks[i].name);
+        benchmark.count = 1;
+        benchmark.discard = 0;
+        benchmark.setup = benchmark.teardown = NULL;
+        all_benchmarks[i].initialize(&benchmark);
+        bm_run(&benchmark);
+        bm_clear(&benchmark);
+    }
+    return INT2FIX(0);
+}
+
+void Init_Benchmark(void) {
+    mBenchmark = rb_define_module_under(mFerret, "Benchmark");
+    rb_define_singleton_method(mBenchmark, "run_all",                     frb_bm_run_all, 0);
+}

data/ext/isomorfeus_ferret_ext/benchmark.h

@@ -0,0 +1,45 @@
+#ifndef BENCHMARK_H
+#define BENCHMARK_H
+
+#include "frt_global.h"
+#include <time.h>
+
+#define BM_RUN_COUNT 6
+#define BENCH(name) extern void bm_##name(BenchMark *bm)
+
+extern const char *WORD_LIST[];
+
+typedef void (*bm_run_ft)();
+
+typedef struct BenchMarkTimes {
+    double utime;
+    double stime;
+    double rtime;
+} BenchMarkTimes;
+
+typedef struct BenchMarkUnit {
+    char *name;
+    bm_run_ft  run;
+    struct BenchMarkUnit *next;
+    BenchMarkTimes final_times;
+    BenchMarkTimes *times[1];
+} BenchMarkUnit;
+
+typedef struct BenchMark {
+    int            count;   /* the number of bench runs to complete */
+    int            discard; /* the number of outliers to discard */
+    void           (*setup)();
+    void           (*teardown)();
+    BenchMarkUnit *head;
+    BenchMarkUnit *tail;
+} BenchMark;
+
+void bm_add(BenchMark *benchmark, bm_run_ft call, const char *name);
+
+#define BM_SETUP(func) bm->setup = &func;
+#define BM_TEARDOWN(func) bm->teardown = &func;
+#define BM_ADD(call) bm_add(bm, &call, #call)
+#define BM_COUNT(num) bm->count = num;
+#define BM_DISCARD(num) bm->discard = num;
+
+#endif

data/ext/isomorfeus_ferret_ext/benchmarks_all.h

@@ -0,0 +1,25 @@
+#ifndef ALL_BENCHMARKS_H
+#define ALL_BENCHMARKS_H
+
+#include "benchmark.h"
+
+void bm_vint_io(BenchMark *bm);
+void bm_strcmp_when_length_is_known(BenchMark *bm);
+void bm_snprintf_vs_strncat(BenchMark *bm);
+void bm_hash_implementations(BenchMark *bm);
+void bm_specialized_string_hash(BenchMark *bm);
+void bm_bitvector_implementations(BenchMark *bm);
+
+const struct BenchMarkList
+{
+    void (*initialize)(BenchMark *benchmark);
+    const char *name;
+} all_benchmarks[] = {
+    {bm_vint_io, "vint_io"},
+    {bm_strcmp_when_length_is_known, "strcmp_when_length_is_known"},
+    {bm_hash_implementations, "hash_implementations"},
+    {bm_specialized_string_hash, "specialized_string_hash"},
+    {bm_bitvector_implementations, "bitvector_implementations"}
+};
+
+#endif

data/ext/isomorfeus_ferret_ext/bm_bitvector.c

@@ -0,0 +1,123 @@
+#include <assert.h>
+#include "frt_bitvector.h"
+#include "benchmark.h"
+
+#define N 10
+#define DENSE_SCAN_SIZE 20000000
+#define SCAN_INC 97
+#define SCAN_SIZE DENSE_SCAN_SIZE * SCAN_INC
+
+static FrtBitVector *bv;
+
+static void setup()
+{
+    bv = frt_bv_new_capa(SCAN_SIZE);
+}
+
+static void teardown()
+{
+    frt_bv_destroy(bv);
+}
+
+static void ferret_bv_and_sparse()
+{
+    FrtBitVector * _bv = frt_bv_and(bv, bv);
+    free(_bv);
+}
+static void ferret_bv_or_sparse()
+{
+    FrtBitVector * _bv = frt_bv_or(bv, bv);
+    free(_bv);
+}
+static void ferret_bv_xor_sparse()
+{
+    FrtBitVector * _bv = frt_bv_xor(bv, bv);
+    free(_bv);
+}
+static void ferret_bv_not_sparse()
+{
+    FrtBitVector * _bv = frt_bv_not(bv);
+    free(_bv);
+}
+static void ferret_bv_and_dense()
+{
+    ferret_bv_and_sparse();
+}
+static void ferret_bv_or_dense()
+{
+    ferret_bv_or_sparse();
+}
+static void ferret_bv_xor_dense()
+{
+    ferret_bv_xor_sparse();
+}
+static void ferret_bv_not_dense()
+{
+    ferret_bv_not_sparse();
+}
+
+static void ferret_bv_set_sparse()
+{
+    int i;
+
+    for (i = SCAN_INC; i < SCAN_SIZE; i += SCAN_INC) {
+        frt_bv_set_fast(bv, i);
+        assert(frt_bv_get(bv, i) == 1);
+        assert(frt_bv_get(bv, i+1) == 0);
+    }
+}
+
+static void ferret_bv_scan_sparse()
+{
+    int i, j;
+
+    for (i = 0; i < N; i++) {
+        frt_bv_scan_reset(bv);
+        for (j = SCAN_INC; j < SCAN_SIZE; j += SCAN_INC) {
+            assert(j == frt_bv_scan_next(bv));
+        }
+        assert(-1 == frt_bv_scan_next(bv));
+    }
+}
+
+static void ferret_bv_set_dense()
+{
+    int i;
+    frt_bv_clear(bv);
+    for (i = 0; i < DENSE_SCAN_SIZE; i++) {
+        frt_bv_set(bv, i);
+    }
+}
+
+static void ferret_bv_scan_dense()
+{
+    int i, j;
+
+    for (i = 0; i < N; i++) {
+        frt_bv_scan_reset(bv);
+        for (j = 0; j < DENSE_SCAN_SIZE; j++) {
+            assert(j == frt_bv_scan_next(bv));
+        }
+        assert(-1 == frt_bv_scan_next(bv));
+    }
+}
+
+BENCH(bitvector_implementations)
+{
+    BM_SETUP(setup);
+
+    BM_ADD(ferret_bv_set_sparse);
+    BM_ADD(ferret_bv_scan_sparse);
+    BM_ADD(ferret_bv_and_sparse);
+    BM_ADD(ferret_bv_or_sparse);
+    BM_ADD(ferret_bv_not_sparse);
+    BM_ADD(ferret_bv_xor_sparse);
+
+    BM_ADD(ferret_bv_set_dense);
+    BM_ADD(ferret_bv_scan_dense);
+    BM_ADD(ferret_bv_and_dense);
+    BM_ADD(ferret_bv_or_dense);
+    BM_ADD(ferret_bv_not_dense);
+    BM_ADD(ferret_bv_xor_dense);
+    BM_TEARDOWN(teardown);
+}

data/ext/isomorfeus_ferret_ext/bm_hash.c

@@ -0,0 +1,118 @@
+#include <string.h>
+#include "frt_hash.h"
+#include "benchmark.h"
+
+#define N 20
+
+static void ferret_hash()
+{
+    int i;
+    for (i = 0; i < N; i++) {
+        FrtHash *h = frt_h_new_str(NULL, NULL);
+        const char **word;
+        char buf[100];
+        long res;
+        for (word = WORD_LIST; *word; word++) {
+            frt_h_set(h, *word, (void *)1);
+        }
+        for (word = WORD_LIST; *word; word++) {
+            strcpy(buf, *word);
+            res = (long)frt_h_get(h, buf);
+        }
+        frt_h_destroy(h);
+    }
+}
+
+BENCH(hash_implementations)
+{
+    BM_ADD(ferret_hash);
+}
+
+static void standard_hash()
+{
+    int i;
+    for (i = 0; i < N; i++) {
+        FrtHash *h = frt_h_new_str(NULL, NULL);
+        const char **word;
+        char buf[100];
+        long res;
+        for (word = WORD_LIST; *word; word++) {
+            frt_h_set(h, *word, (void *)1);
+            strcpy(buf, *word);
+            res = (long)frt_h_get(h, buf);
+        }
+        frt_h_destroy(h);
+    }
+}
+
+#define PERTURB_SHIFT 5
+static const char *dummy_key = "";
+static FrtHashEntry *h_lookup_str(FrtHash *ht, register const void *key)
+{
+    register const unsigned long hash = frt_str_hash((const char *)key);
+    register unsigned int perturb;
+    register int mask = ht->mask;
+    register FrtHashEntry *he0 = ht->table;
+    register int i = hash & mask;
+    register FrtHashEntry *he = &he0[i];
+    register FrtHashEntry *freeslot = NULL;
+
+    if (he->key == NULL || he->key == key) {
+        he->hash = hash;
+        return he;
+    }
+    if (he->key == dummy_key) {
+        freeslot = he;
+    }
+    else {
+        if ((he->hash == hash)
+            && 0 == strcmp((const char *)he->key, (const char *)key)) {
+            return he;
+        }
+    }
+
+    for (perturb = hash;; perturb >>= PERTURB_SHIFT) {
+        i = (i << 2) + i + perturb + 1;
+        he = &he0[i & mask];
+        if (he->key == NULL) {
+            if (freeslot != NULL) {
+                he = freeslot;
+            }
+            he->hash = hash;
+            return he;
+        }
+        if (he->key == key
+            || (he->hash == hash
+                && he->key != dummy_key
+                && 0 == strcmp((const char *)he->key, (const char *)key))) {
+            return he;
+        }
+        if (he->key == dummy_key && freeslot == NULL) {
+            freeslot = he;
+        }
+    }
+}
+
+static void string_hash()
+{
+    int i;
+    for (i = 0; i < N; i++) {
+        FrtHash *h = frt_h_new_str(NULL, NULL);
+        const char **word;
+        char buf[100];
+        long res;
+        h->lookup_i = &h_lookup_str;
+        for (word = WORD_LIST; *word; word++) {
+            frt_h_set(h, *word, (void *)1);
+            strcpy(buf, *word);
+            res = (long)frt_h_get(h, buf);
+        }
+        frt_h_destroy(h);
+    }
+}
+
+BENCH(specialized_string_hash)
+{
+    BM_ADD(standard_hash);
+    BM_ADD(string_hash);
+}

data/ext/isomorfeus_ferret_ext/bm_micro_string.c

@@ -0,0 +1,40 @@
+#include <string.h>
+#include "benchmark.h"
+
+#define N 10
+
+static void do_strcmp()
+{
+    char **word;
+    char buf[100];
+    int res, i;
+
+    for (i = 0; i < N; i++)
+        for (word = WORD_LIST; *word; word++) {
+            size_t len = strlen(*word);
+            memcpy(buf, *word, len+1);
+            res = strcmp(buf, *word);
+        }
+}
+
+static void do_strncmp()
+{
+    char **word;
+    char buf[100];
+    int res, i;
+
+    for (i = 0; i < N; i++)
+        for (word = WORD_LIST; *word; word++) {
+            size_t len = strlen(*word);
+            memcpy(buf, *word, len+1);
+            res = strncmp(buf, *word, len + 1);
+        }
+}
+
+BENCH(strcmp_when_length_is_known)
+{
+    BM_COUNT(6);
+    BM_DISCARD(1);
+    BM_ADD(do_strcmp);
+    BM_ADD(do_strncmp);
+}

data/ext/isomorfeus_ferret_ext/bm_store.c

@@ -0,0 +1,93 @@
+#include <string.h>
+#include "benchmark.h"
+#include "frt_config.h"
+#include "frt_store.h"
+
+#define N 10
+#define write_byte(os, b) os->buf.buf[os->buf.pos++] = (frt_uchar)b
+
+void my_os_write_voff_t(FrtOutStream *os, register off_t num)
+{
+    if (!(num&0x7f)) {
+        if (os->buf.pos >= FRT_BUFFER_SIZE) {
+            frt_os_write_byte(os, (frt_uchar)num);
+        }
+        else {
+            write_byte(os, (frt_uchar)num);
+        }
+    }
+    else if (!(num&0x3fff)) {
+        if (os->buf.pos >= FRT_BUFFER_SIZE - 1) {
+            frt_os_write_byte(os, (frt_uchar)(0x80 | (0x3f & num))); num >>= 6;
+            frt_os_write_byte(os, (frt_uchar)num);
+        }
+        else {
+            write_byte(os, (frt_uchar)(0x80 | (0x3f & num))); num >>= 6;
+            write_byte(os, (frt_uchar)num);
+        }
+    }
+    else if (!(num&0x1fffff)) {
+        if (os->buf.pos >= FRT_BUFFER_SIZE - 2) {
+            frt_os_write_byte(os, (frt_uchar)(0xc0 | (0x1f & num))); num >>= 5;
+            frt_os_write_byte(os, (frt_uchar)(0xff| num)); num >>= 8;
+            frt_os_write_byte(os, (frt_uchar)num);
+        }
+        else {
+            write_byte(os, (frt_uchar)(0xc0 | (0x1f & num))); num >>= 5;
+            write_byte(os, (frt_uchar)(0xff| num)); num >>= 8;
+            write_byte(os, (frt_uchar)num);
+        }
+    }
+    else if (!(num&0xfffff)) {
+        if (os->buf.pos >= FRT_BUFFER_SIZE - 3) {
+            frt_os_write_byte(os, (frt_uchar)(0xe0 | (0x0f & num))); num >>= 4;
+            frt_os_write_byte(os, (frt_uchar)(0xff | num)); num >>= 8;
+            frt_os_write_byte(os, (frt_uchar)(0xff | num)); num >>= 8;
+            frt_os_write_byte(os, (frt_uchar)num);
+        }
+        else {
+            write_byte(os, (frt_uchar)(0xe0 | (0x0f & num))); num >>= 4;
+            write_byte(os, (frt_uchar)(0xff | num)); num >>= 8;
+            write_byte(os, (frt_uchar)(0xff | num)); num >>= 8;
+            write_byte(os, (frt_uchar)num);
+        }
+    }
+}
+
+static void vint_out()
+{
+    int n;
+    off_t i;
+    FrtOutStream *os;
+
+    for (n = 0; n < N; n++) {
+        os = frt_ram_new_buffer();
+        for (i = 0; i < 10000000; i++) {
+            frt_os_write_voff_t(os, i);
+        }
+        frt_ram_destroy_buffer(os);
+    }
+
+}
+
+static void unrolled_vint_out()
+{
+    int n;
+    off_t i;
+    FrtOutStream *os;
+
+    for (n = 0; n < N; n++) {
+        os = frt_ram_new_buffer();
+        for (i = 0; i < 10000000; i++) {
+            frt_os_write_voff_t(os, i);
+        }
+        frt_ram_destroy_buffer(os);
+    }
+
+}
+
+BENCH(vint_io)
+{
+    BM_ADD(vint_out);
+    BM_ADD(unrolled_vint_out);
+}

data/ext/isomorfeus_ferret_ext/email.rl

@@ -0,0 +1,21 @@
+#// email.rl -*-C-*-
+%%{
+    machine Email;
+
+    #// RFC 2822 - matching email addresses
+    NO_WS_CTL       = ( 1..8 | 11 | 12 | 14..31 | 127 );
+    ASCII           = 1..127;
+    atext           = [a-zA-Z0-9!#$%&\'*+\-/=?^_`{|}~];
+    qtext           = ( NO_WS_CTL | 33 | 35..91 | 93..126 );
+    dtext           = ( NO_WS_CTL | 33..90 | 94..126 );
+    dot_atom        = atext+ ('.' atext+)*;
+    text            = ( 1..9 | 11 | 12 | 14..127 );
+    quoted_pair     = '\\' text;
+    quoted_string   = '"' ( qtext | quoted_pair )* '"';
+    domain_literal  = '[' (dtext | quoted_pair)* ']';
+
+    local_part      = dot_atom | quoted_string;
+    domain          = dot_atom | domain_literal;
+
+    email           = local_part '@' domain;
+}%%

data/ext/isomorfeus_ferret_ext/extconf.rb

@@ -0,0 +1,5 @@
+require 'mkmf'
+
+$CFLAGS << ' -O2 -Wall -Wno-sizeof-pointer-div'
+
+create_makefile('isomorfeus_ferret_ext')