isomorfeus-ferret 0.13.0 → 0.13.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be4c84d556459a8ed5d2585068378c156bdb2d68507ca2786844b2c69a4e7f35
-  data.tar.gz: '0096ef29b274ea39567e876d95d8441ff80589a8d3403c5aed3801c62377cffd'
+  metadata.gz: 98439b4a9e6ca849246c6e2ddd2ce1bbf117182025b3651d4f6a95593aff0eb6
+  data.tar.gz: 0e1e90c4bfce1014c9983f4bb5be0f3123ef502788f464e988c4671bdd1758ed
 SHA512:
-  metadata.gz: 55aa9f39fd4971e1a80bbcdeba14906928140fd023a7ee3e581ef2bafc63a8a664f5b0fbaf86dec1119ac62fd441703d33a4b6ee3731f811736a1b2eedcef9a3
-  data.tar.gz: 3843bd29450fb925069733913aee9f66958dfe41aba052d7d14a18cb7f2d4df376e0756d7c97bcd01527120a6a77776d9bee2a1c0a488648fc61f500dfe6e98e
+  metadata.gz: da674772ba34175364d0d4d93023ef380eab06334b4f9c3e1956934289a7a2016387607740fb83cc11753a3bb72de62208390933ff9f4cc341524b7fe6c0c6af
+  data.tar.gz: 8261029020f33cb9fb52453e007defdfd2c1dd029f9da22a02b64ae0047bcd333e62e6d6a8d15263ee71089e203f393f5e865138a31784edf48f66a483a0ffce

data/README.md

@@ -50,6 +50,7 @@ Compression semantics have changed, now Brotli, BZip2 and LZ4 compression codecs
 - BZip2: slow compression, slow decompression, high compression ratio
 - Brotli: slow compression, fast decrompression, high compression ratio, recommended for general purpose.
 - LZ4: fast compression, fast decrompression, low compression ratio
+
 To see performance and compression ratios `rake ferret_compression_bench` can be run from the cloned repo.
 It uses data and code within the misc/ferret_vs_lucene directory.
 
@@ -96,6 +97,7 @@ Ensure your locale is set to C.UTF-8, because the internal c tests don't know ho
 
 ## Benchmarks
 
+### Indexing and Searching
 - clone repo
 - bundle install
 - rake ferret_vs_lucene
@@ -104,20 +106,37 @@ A recent Java JDK must be installed to compile and run lucene benchmarks.
 
 Results on Linux:
 ```
-Ferret:
-Indexing Secs: 7.36  Docs: 19043, 2587 docs/s
-Searching took: 0.3366296s for 8000 queries
-thats 23765 q/s
-
-Lucene:
-Indexing Secs: 4.22  Docs: 19043, 4516 docs/s
-Searching took: 1.48s for 8000 queries
-thats 5420 q/s
----------------------------------------------------
-Lucene 9.0.0 0b18b3b965cedaf5eb129aa41243a44c83ca826d - jpountz - 2021-12-01 14:23:49
-JVM 17.0.1 (Private Build)
+Ferret 0.13.0:
+Indexing: 9.35 secs, Docs: 19043, 2035 docs/s
+Searching took: 0.3133133s for 8000 queries
+thats 25533 q/s
+Total found: 42000
+Index size: 28Mb
+
+Lucene 9.1.0:
+Indexing: 4.20 secs, Docs: 19043, 4538 docs/s
+Searching took: 1.64s for 8000 queries
+thats 4875 q/s
+Total found: 41000
+index size: 35Mb
+
+JVM 11.0.14.1 (Ubuntu)
 ```
 
+### Storing Fields with Compression, Indexing and Retrieval
+- clone repo
+- bundle install
+- rake ferret_compression_benchmark
+
+Results on Linux, 0.13.0:
+
+| Compression | Index & Store | Retrieve      | Index size |
+|-------------|---------------|---------------|------------|
+| none        |   2008 docs/s | 153853 docs/s |      43 MB |
+| brotli      |   1726 docs/s |  58315 docs/s |      36 MB |
+| bzip2       |   1438 docs/s |  15382 docs/s |      38 MB |
+| lz4         |   1932 docs/s | 127100 docs/s |      41 MB |
+
 ## Future
 
 Lots of things to do:

data/ext/isomorfeus_ferret_ext/frt_global.c

@@ -232,71 +232,6 @@ void frt_dummy_free(void *p) {
     (void)p; /* suppress unused argument warning */
 }
 
-#ifdef HAVE_GDB
-#define CMD_BUF_SIZE (128 + FILENAME_MAX)
-/* need to declare this as it is masked by default in linux */
-
-static char *build_shell_command(void) {
-    int   pid = getpid();
-    char *buf = FRT_ALLOC_N(char, CMD_BUF_SIZE);
-    char *command =
-        "gdb -quiet -ex='bt' -ex='quit' %s %d 2>/dev/null | grep '^[ #]'";
-
-    snprintf(buf, CMD_BUF_SIZE, command, frt_progname(), pid);
-    return buf;
-}
-
-#endif
-
-/**
- * Call out to gdb to get our stacktrace.
- */
-char *frt_get_stacktrace(void) {
-#ifdef HAVE_GDB
-    FILE *stream;
-    char *gdb_filename = NULL, *buf = NULL, *stack = NULL;
-    int   offset = -FRT_BUFFER_SIZE;
-
-    if ( !(buf = build_shell_command()) ) {
-        fprintf(EXCEPTION_STREAM,
-                "Unable to build stacktrace shell command\n");
-        goto cleanup;
-    }
-
-    if ( !(stream = popen(buf, "r")) ) {
-        fprintf(EXCEPTION_STREAM,
-                "Unable to exec stacktrace shell command: '%s'\n", buf);
-        goto cleanup;
-    }
-
-    do {
-        offset += FRT_BUFFER_SIZE;
-        FRT_REALLOC_N(stack, char, offset + FRT_BUFFER_SIZE);
-        FRT_ZEROSET_N(stack + offset, char, FRT_BUFFER_SIZE);
-    } while(fread(stack + offset, 1, FRT_BUFFER_SIZE, stream) == FRT_BUFFER_SIZE);
-
-    pclose(stream);
-
- cleanup:
-    if (gdb_filename) free(gdb_filename);
-    if (buf) free(buf);
-    return stack;
-#else
-    return NULL;
-#endif
-}
-
-void frt_print_stacktrace(void) {
-    char *stack = frt_get_stacktrace();
-
-    if (stack) {
-        fprintf(EXCEPTION_STREAM, "Stack trace:\n%s", stack);
-        free(stack);
-    } else {
-        fprintf(EXCEPTION_STREAM, "Stack trace not available\n");
-    }
-}
-
 typedef struct FreeMe {
     void *p;
     frt_free_ft free_func;
@@ -321,55 +256,7 @@ void frt_register_for_cleanup(void *p, frt_free_ft free_func) {
     free_me->free_func = free_func;
 }
 
-#define MAX_PROG_NAME 200
-static char name[MAX_PROG_NAME]; /* program name for error msgs */
-
-/* frt_setprogname: set stored name of program */
-void frt_setprogname(const char *str) {
-    strncpy(name, str, sizeof(name) - 1);
-}
-
-const char *frt_progname(void) {
-    return name;
-}
-
-static const char *signal_to_string(int signum) {
-    switch (signum)
-    {
-        case SIGILL:  return "SIGILL";
-        case SIGABRT: return "SIGABRT";
-        case SIGFPE:  return "SIGFPE";
-#if !defined POSH_OS_WIN32 && !defined POSH_OS_WIN64
-        case SIGBUS:  return "SIGBUS";
-#endif
-        case SIGSEGV: return "SIGSEGV";
-    }
-
-    return "Unknown Signal";
-}
-
-static void sighandler_crash(int signum) {
-    frt_print_stacktrace();
-    FRT_XEXIT("Signal", "Exiting on signal %s (%d)", signal_to_string(signum), signum);
-}
-
-#define SETSIG_IF_UNSET(sig, handler) do { \
-    signal(sig, handler);                  \
-} while(0)
-
 void frt_init(int argc, const char *const argv[]) {
-    if (argc > 0) {
-        frt_setprogname(argv[0]);
-    }
-
-    SETSIG_IF_UNSET(SIGILL , sighandler_crash);
-    SETSIG_IF_UNSET(SIGABRT, sighandler_crash);
-    SETSIG_IF_UNSET(SIGFPE , sighandler_crash);
-#if !defined POSH_OS_WIN32 && !defined POSH_OS_WIN64
-    SETSIG_IF_UNSET(SIGBUS , sighandler_crash);
-#endif
-    SETSIG_IF_UNSET(SIGSEGV, sighandler_crash);
-
     atexit(&frt_hash_finalize);
 
     utf8_encoding = rb_enc_find("UTF-8");
@@ -429,41 +316,3 @@ void frt_init(int argc, const char *const argv[]) {
     FRT_SORT_FIELD_DOC_REV->compare = frt_sort_field_doc_compare; /* compare */
     FRT_SORT_FIELD_DOC_REV->get_val = frt_sort_field_doc_get_val; /* get_val */
 }
-
-/**
- * For general use when testing
- *
- * TODO wrap in #ifdef
- */
-
-static bool p_switch = false;
-static bool p_switch_tmp = false;
-
-void p(const char *format, ...) {
-    va_list args;
-
-    if (!p_switch) return;
-
-    va_start(args, format);
-    vfprintf(stderr, format, args);
-    va_end(args);
-}
-
-void p_on(void) {
-    fprintf(stderr, "> > > > > STARTING PRINT\n");
-    p_switch = true;
-}
-
-void p_off(void) {
-    fprintf(stderr, "< < < < < STOPPING PRINT\n");
-    p_switch = false;
-}
-
-void frt_p_pause(void) {
-    p_switch_tmp = p_switch;
-    p_switch = false;
-}
-
-void frt_p_resume(void) {
-    p_switch = p_switch_tmp;
-}

data/ext/isomorfeus_ferret_ext/frt_global.h

@@ -105,9 +105,6 @@ extern char *frt_dbl_to_s(char *buf, double num);
 extern char *frt_strfmt(const char *fmt, ...);
 extern char *frt_vstrfmt(const char *fmt, va_list args);
 
-extern char *frt_get_stacktrace();
-extern void  frt_print_stacktrace();
-
 extern void frt_register_for_cleanup(void *p, frt_free_ft free_func);
 
 /**
@@ -277,18 +274,6 @@ extern bool frt_x_do_logging;
 #endif
 
 extern void frt_init(int arc, const char *const argv[]);
-extern void frt_setprogname(const char *str);
-extern const char *frt_progname();
 extern void frt_micro_sleep(const int micro_seconds);
 
-/**
- * For general use during testing. Switch this on and off for print statements
- * to only print when p_on is called and not after p_off is called
- */
-extern void p(const char *format, ...);
-extern void p_on();
-extern void p_off();
-extern void frt_p_pause();
-extern void frt_p_resume();
-
 #endif

data/ext/isomorfeus_ferret_ext/frt_lang.c

@@ -61,7 +61,6 @@ void FRT_VEXIT(const char *err_type, const char *fmt, va_list args)
 # endif
 {
     fflush(stdout);
-    fprintf(EXCEPTION_STREAM, "\n%s: ", frt_progname());
 
 # ifdef FRT_HAS_VARARGS
     fprintf(EXCEPTION_STREAM, "%s occurred at <%s>:%d in %s\n",
@@ -76,7 +75,6 @@ void FRT_VEXIT(const char *err_type, const char *fmt, va_list args)
     }
 
     fprintf(EXCEPTION_STREAM, "\n");
-    frt_print_stacktrace();
     if (frt_x_abort_on_exception) {
         exit(2);                 /* conventional value for failed execution */
     }

data/ext/isomorfeus_ferret_ext/frt_q_boolean.c

@@ -1286,6 +1286,7 @@ FrtBooleanClause *frt_bc_alloc(void) {
 FrtBooleanClause *frt_bc_init(FrtBooleanClause *self, FrtQuery *query, FrtBCType occur) {
     self->ref_cnt = 1;
     self->query = query;
+    self->rbc = Qnil;
     frt_bc_set_occur(self, occur);
     return self;
 }

data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c

@@ -250,9 +250,7 @@ void Init_Ferret(void) {
 }
 
 void Init_isomorfeus_ferret_ext(void) {
-    const char *const progname[] = {"ruby"};
-
-    frt_init(1, progname);
+    frt_init(0, NULL);
 
     /* IDs */
     id_new = rb_intern("new");

data/ext/isomorfeus_ferret_ext/test.c

@@ -279,22 +279,10 @@ static void append_to_msg_buf(const char *fmt, ...)
     va_end(args);
 }
 
-
-static void Tstack(void) {
-    if (show_stack) {
-        char *stack = frt_get_stacktrace();
-        if (stack) {
-            append_to_msg_buf("\n\nStack trace:\n%s\n", stack);
-            free(stack);
-        }
-    }
-}
-
 static void vTmsg_nf(const char *fmt, va_list args)
 {
     if (verbose) {
         vappend_to_msg_buf(fmt, args);
-        Tstack();
     }
 }
 
@@ -305,8 +293,6 @@ void vTmsg(const char *fmt, va_list args)
         vappend_to_msg_buf(fmt, args);
         va_end(args);
         append_to_msg_buf("\n");
-
-        Tstack();
     }
 }
 
@@ -348,8 +334,6 @@ void tst_msg(const char *func, const char *fname, int line_num, const char *fmt,
         va_start(args, fmt);
         vappend_to_msg_buf(fmt, args);
         va_end(args);
-
-        Tstack();
     }
 }
 

data/ext/isomorfeus_ferret_ext/test_global.c

@@ -145,50 +145,6 @@ static void test_dbl_to_s(TestCase *tc, void *data)
     Asequal("NaN", frt_dbl_to_s(buf, NAN));
 }
 
-
-/**
- * Generate a stacktrace, make sure it does something
- */
-static void test_stacktrace(TestCase *tc, void *data)
-{
-    FILE *old_stream = frt_x_exception_stream;
-    (void)data; /* suppress warning */
-    int tfd = fio_tmpfile();
-    frt_x_exception_stream = fdopen(tfd, "w+");
-    Atrue(frt_x_exception_stream != NULL);
-    if (frt_x_exception_stream) {
-        frt_print_stacktrace();
-        long int f = ftell(frt_x_exception_stream);
-        Assert(f, "Stream position should not be 0");
-        fclose(frt_x_exception_stream);
-    }
-    frt_x_exception_stream = old_stream;
-}
-
-/**
- * Generate a normally fatal signal, which gets caught
- */
- /*
-static void test_sighandler(TestCase *tc, void *data)
-{
-    bool  old_abort = frt_x_abort_on_exception;
-    FILE *old_stream = frt_x_exception_stream;
-    (void)data;
-    (void)tc;
-
-    frt_x_exception_stream = false;
-    frt_x_exception_stream = tmpfile();
-
-    raise(SIGSEGV);
-
-    Assert(ftell(frt_x_exception_stream), "Stream position should not be 0");
-    fclose(frt_x_exception_stream);
-
-    frt_x_exception_stream = old_stream;
-    frt_x_abort_on_exception = old_abort;
-}
-*/
-
 static void test_count_leading_zeros(TestCase *tc, void *data)
 {
     (void)data;
@@ -284,8 +240,6 @@ TestSuite *ts_global(TestSuite *suite)
     tst_run_test(suite, test_alloc, NULL);
     tst_run_test(suite, test_strfmt, NULL);
     tst_run_test(suite, test_dbl_to_s, NULL);
-    tst_run_test(suite, test_stacktrace, NULL);
-    // tst_run_test(suite, test_sighandler, NULL);
     tst_run_test(suite, test_count_leading_zeros, NULL);
     tst_run_test(suite, test_count_leading_ones, NULL);
     tst_run_test(suite, test_count_trailing_zeros, NULL);

data/ext/isomorfeus_ferret_ext/test_search.c

@@ -266,7 +266,6 @@ void tst_check_hits(TestCase *tc, FrtSearcher *searcher, FrtQuery *query, const
     int i, count;
     int total_hits = s2l(expected_hits, num_array);
     FrtTopDocs *top_docs = frt_searcher_search(searcher, query, 0, total_hits + 1, NULL, NULL, NULL);
-    frt_p_pause();
     if (!tc->failed && !Aiequal(total_hits, top_docs->total_hits)) {
         int i;
         Tmsg_nf("\texpected docs:\n\t    ");
@@ -314,7 +313,6 @@ void tst_check_hits(TestCase *tc, FrtSearcher *searcher, FrtQuery *query, const
         count = frt_searcher_search_unscored(searcher, query, num_array2, ARRAY_SIZE, num_array2[3]);
         Aaiequal(num_array + 3, num_array2, count);
     }
-    frt_p_resume();
 }
 
 void check_match_vector(TestCase *tc, FrtSearcher *searcher, FrtQuery *query,

data/lib/isomorfeus/ferret/version.rb

@@ -1,5 +1,5 @@
 module Isomorfeus
   module Ferret
-    VERSION = '0.13.0'
+    VERSION = '0.13.1'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: isomorfeus-ferret
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.13.1
 platform: ruby
 authors:
 - Jan Biedermann
@@ -154,13 +154,13 @@ files:
 - ext/isomorfeus_ferret_ext/brotli_encode.h
 - ext/isomorfeus_ferret_ext/brotli_port.h
 - ext/isomorfeus_ferret_ext/brotli_types.h
-- ext/isomorfeus_ferret_ext/bzip_blocksort.c
-- ext/isomorfeus_ferret_ext/bzip_huffman.c
 - ext/isomorfeus_ferret_ext/bzlib.c
 - ext/isomorfeus_ferret_ext/bzlib.h
+- ext/isomorfeus_ferret_ext/bzlib_blocksort.c
 - ext/isomorfeus_ferret_ext/bzlib_compress.c
 - ext/isomorfeus_ferret_ext/bzlib_crctable.c
 - ext/isomorfeus_ferret_ext/bzlib_decompress.c
+- ext/isomorfeus_ferret_ext/bzlib_huffman.c
 - ext/isomorfeus_ferret_ext/bzlib_private.h
 - ext/isomorfeus_ferret_ext/bzlib_randtable.c
 - ext/isomorfeus_ferret_ext/extconf.rb