isomorfeus-ferret 0.13.6 → 0.13.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 524b384c2ab35457603726287160b443a50939473bab7852e5e6f5f886ded0a7
-  data.tar.gz: b304d0386bc0b14e68167bfcc1b62346f81c02f6eff25c3a6633bb0a1919083e
+  metadata.gz: 479f3df28144a3ca25194afc36590058e027af38ba43cc42d24ffd83a35a6da7
+  data.tar.gz: 95c80530e4012e0f7219e3b60cc82748f4e6d1824f0958cfda6cd4735c92a82a
 SHA512:
-  metadata.gz: b378ce12aa56bb35e57eec2c8df64152a7a7bd5cb1345d4a7dc0264e236fddcea3ffbc41a7213401cf50ff163c3172169fbf93699a4a7f96505f300782bc9a92
-  data.tar.gz: 6954e297df285f7726bd925a6f4dda4300f06d059dc4b674a494e775b901f7ea9e529e62be8793049c6df20a17515c41741f63155fbdd661de4595694deb4c27
+  metadata.gz: 0a72ddf5c974896ac0e5d11cdab105c02894f12ffac43db12b7651393f0bfd5ebd5f99f4e03e7714a8d230c6e77f71a1f5ee1c243b01608b23555a170eabe5c9
+  data.tar.gz: 58a803a8de3afae0b2bdf6e0eeda41f6a2e29dc649d8e219d2d7d5c667bd27bea3fa8dd80f16af934456e9414cd0f6a2ec9b998d303593db94f43de208a3eaff

data/README.md

@@ -63,14 +63,8 @@ fis.add_field(:compressed_field, :store => :yes, :compression => :brotli, :term_
 
 ### Performance
 
-The encoding support demands its toll, indexing performance dropped a bit in comparision to 0.12, but still thousands of docs per second, depending on machine/docs.
-On Windows the indexing performance is still terrible, but that may be resolved in a future project.
-
-Search performance is still excellent and multiple times faster than Lucene.
-
-Lucene achieves roughly double the indexing performance. This seems to be because of the different way strings and
-encodings are handled in Java. For example, the Java WhitespaceTokenizer code requires only one method call per character (check for whitespace), but for Ruby, to support all the different encodings, several method calls are required per character (retrieve character according to encoding, check character for whitespace).
-Ferret is internally using the standard Ruby string encoding methods.
+For version 0.13.7 the performance bottle-nack has been identified and removed, ferret now delivers excellent indexing perfomance on all platforms, see numbers below.
+On Windows performance is still not as good as on Linux, but that is equally true for Lucene and because of how the Windows filesystem works.
 
 ## Documentation
 
@@ -105,43 +99,35 @@ Ensure your locale is set to C.UTF-8, because the internal c tests don't know ho
 
 A recent Java JDK must be installed to compile and run lucene benchmarks.
 
-Results on Linux:
-```
-Ferret 0.13.0:
-Indexing: 9.35 secs, Docs: 19043, 2035 docs/s
-Searching took: 0.3133133s for 8000 queries
-thats 25533 q/s
-Total found: 42000
-Index size: 28Mb
-
-Lucene 9.1.0:
-Indexing: 4.20 secs, Docs: 19043, 4538 docs/s
-Searching took: 1.64s for 8000 queries
-thats 4875 q/s
-Total found: 41000
-index size: 35Mb
-
-JVM 11.0.14.1 (Ubuntu)
-```
+Results, Ferret 0.13.7 vs. Lucene 9.1.0, WhitespaceAnalyzer, on old Intel Core i5 from 2015:
+
+| OS      | Task       | Ferret          | Lucene*        |
+|---------|------------|-----------------|----------------|
+| Linux   | Indexing   |     4905 docs/s |    4785 docs/s |
+| Windows | Indexing   |     2361 docs/s |    2395 docs/s |
+| Linux   | Searching  | 25664 queries/s | 4708 queries/s |
+| Windows | Searching  |  3646 queries/s |  935 queries/s |
+|         | Index Size |           28 MB |          35 MB |
+
+*Lucene 9.1.0 on JVM 11.0.14.1 (Ubuntu)
 
 ### Storing Fields with Compression, Indexing and Retrieval
 - clone repo
 - bundle install
 - rake ferret_compression_benchmark
 
-Results on Linux, 0.13.0:
+Results on Linux, 0.13.7, on old Intel Core i5 from 2015:
 
 | Compression | Index & Store | Retrieve      | Index size |
 |-------------|---------------|---------------|------------|
-| none        |   2008 docs/s | 153853 docs/s |      43 MB |
-| brotli      |   1726 docs/s |  58315 docs/s |      36 MB |
-| bzip2       |   1438 docs/s |  15382 docs/s |      38 MB |
-| lz4         |   1932 docs/s | 127100 docs/s |      41 MB |
+| none        |   4866 docs/s | 153853 docs/s |      43 MB |
+| brotli      |   3539 docs/s |  58315 docs/s |      36 MB |
+| bzip2       |   2624 docs/s |  15382 docs/s |      38 MB |
+| lz4         |   4639 docs/s | 127100 docs/s |      41 MB |
 
 ## Future
 
 Lots of things to do:
-- Improve indexing performance on Windows (WriteFile is terribly slow, maybe use mapping, see libuv)
 - Bring documentation in order in a docs directory
 - Review code (especially for memory/stack issues, typical c issues)
 - Take care of ruby GVL and threading

data/ext/isomorfeus_ferret_ext/frt_global.h

@@ -13,7 +13,7 @@
 
 #define FRT_MAX_WORD_SIZE 255
 #define FRT_MAX_FILE_PATH 1024
-#define FRT_BUFFER_SIZE 1024
+#define FRT_BUFFER_SIZE 16384
 
 typedef enum {
     FRT_COMPRESSION_NONE = 0,

data/ext/isomorfeus_ferret_ext/frt_store.c

@@ -149,7 +149,7 @@ void frt_os_seek(FrtOutStream *os, off_t new_pos)
  */
 void frt_os_write_byte(FrtOutStream *os, frt_uchar b)
 {
-    if (os->buf.pos >= FRT_BUFFER_SIZE) {
+    if (os->buf.pos >= (FRT_BUFFER_SIZE - 1)) {
         frt_os_flush(os);
     }
     write_byte(os, b);
@@ -157,15 +157,12 @@ void frt_os_write_byte(FrtOutStream *os, frt_uchar b)
 
 void frt_os_write_bytes(FrtOutStream *os, const frt_uchar *buf, int len)
 {
-    if (os->buf.pos > 0) {      /* flush buffer */
-        frt_os_flush(os);
-    }
-
-    if (len < FRT_BUFFER_SIZE) {
-        os->m->flush_i(os, buf, len);
-        os->buf.start += len;
+    if (len < (FRT_BUFFER_SIZE - os->buf.pos)) {
+        memcpy(os->buf.buf + os->buf.pos, buf, len);
+        os->buf.pos += len;
     }
     else {
+        frt_os_flush(os);
         int pos = 0;
         int size;
         while (pos < len) {

data/lib/isomorfeus/ferret/version.rb

@@ -1,5 +1,5 @@
 module Isomorfeus
   module Ferret
-    VERSION = '0.13.6'
+    VERSION = '0.13.7'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: isomorfeus-ferret
 version: !ruby/object:Gem::Version
-  version: 0.13.6
+  version: 0.13.7
 platform: ruby
 authors:
 - Jan Biedermann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-26 00:00:00.000000000 Z
+date: 2022-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake