isomorfeus-ferret 0.13.12 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60ce50bdaf067d95199abf381bd96e0b92fcce70ac7f64ea7e2f73ac1564ff64
-  data.tar.gz: 9c6eba0bc2630a6d95b55ea5887466a041e50425f1472c9008ef82a841194cf6
+  metadata.gz: e6893e7012cf75189d3ff378b6e869a831a5281472f84ea5ab4e354bd92bfcee
+  data.tar.gz: 0a4cad49faae062c29e0bed8fd7f87c5e3875c548cbb6e168907719b52777306
 SHA512:
-  metadata.gz: 878e93d18fe9a2847599be12478baea42fd010d4aa52f639977a7fb4bc8901771035908c9ceebfd34508ccd02f5d909569d06f88e55df36d8345e5bf35942d3e
-  data.tar.gz: c00f2909a8ef647a81deff946bedbfe5e4bfd86c175537a114866ca278a388a36c885a612f3cb1c49b582aaea9c9751f86b439d111e1b72205f46f653d7913cc
+  metadata.gz: 4e48ec64d99af7fe0440480f11f22fd79dd9fca0c6dd09ce58bc0953f7f556ee4f61bdf18810f5cb2f39c8f80c99b406c5319c93235d06974009cefb1c73fccb
+  data.tar.gz: 254960eb7543fb59e1d12087f83feaeb4abd957314f76b56778baf7a2fef2e090922b4b99d653e8532e30690a14d90c661ec5f849588a7e384b2a65346f7f04d

data/README.md

@@ -11,13 +11,29 @@ At the [Isomorfeus Framework Project](https://isomorfeus.com)
 
 ## About this project
 
-Isomorfeus-Ferret is a revived version of the original ferret gem created by Dave Balmain, [https://github.com/dbalmain/ferret](https://github.com/dbalmain/ferret).
+Isomorfeus-Ferret is a revived version of the original ferret gem created by Dave Balmain,
+[https://github.com/dbalmain/ferret](https://github.com/dbalmain/ferret).
 During revival many things havbe been fixed, now all tests pass, no crashes and it
 successfully compiles and runs with rubys >3. Its no longer a goal to have
 a c library available, but instead the usage is meant as ruby gem with a c extension only.
 
 It works on *nixes, *nuxes, *BSDs and also works on Windows and RaspberryPi.
 
+## Improvements and Changes in Version 0.14
+
+### Breaking
+
+- The API for LazyDocs has changed, they are read only now. LazyDoc#to_h may be used to create a hash, that may be changed and reindexed as doc.
+
+### Performance
+
+- LazyDoc is now truly lazy, fields are automatically retrieved. LazyDoc#load is no longer required, but may be used to preload all fields.
+- Index#each is now multiple times faster, depending on use case.
+
+### Other
+
+- The Index class now includes Enumerable
+
 ## Improvements and Changes in Version 0.13
 
 ### Breaking
@@ -99,37 +115,42 @@ Ensure your locale is set to C.UTF-8, because the internal c tests don't know ho
 
 A recent Java JDK must be installed to compile and run lucene benchmarks.
 
-Results, Ferret 0.13.7 vs. Lucene 9.1.0, WhitespaceAnalyzer,
-Linux Ubuntu 20.04, FreeBSD 13.0 and Windows 10 on old Intel Core i5 from 2015,
+Results, Ferret 0.14.0 vs. Lucene 9.1.0, WhitespaceAnalyzer,
+Linux Ubuntu 20.04, FreeBSD 13.1 and Windows 10 on old Intel Core i5 from 2015,
 LinuxPi on RaspberryPi 400:
 
 | OS      | Task       | Ferret          | Lucene*        |
 |---------|------------|-----------------|----------------|
-| Linux   | Indexing   |     4905 docs/s |    4785 docs/s |
-| FreeBSD | Indexing   |     4516 docs/s |       -        |
-| Windows | Indexing   |     2361 docs/s |    2395 docs/s |
-| LinuxPi | Indexing   |     1161 docs/s |     707 docs/s |
-| Linux   | Searching  | 25664 queries/s | 4708 queries/s |
-| FreeBSD | Searching  | 25073 queries/s |    -           |
-| Windows | Searching  |  3646 queries/s |  935 queries/s |
-| LinuxPi | Searching  |  5768 queries/s |  680 queries/s |
+| Linux   | Indexing   |     5125 docs/s |    4671 docs/s |
+| FreeBSD | Indexing   |     4537 docs/s |    3831 docs/s |
+| Windows | Indexing   |     2488 docs/s |    2588 docs/s |
+| LinuxPi | Indexing   |     1200 docs/s |     551 docs/s |
+| Linux   | Searching  | 26610 queries/s | 7165 queries/s |
+| FreeBSD | Searching  | 24167 queries/s | 4288 queries/s |
+| Windows | Searching  |  3901 queries/s | 1033 queries/s |
+| LinuxPi | Searching  |  6194 queries/s |  769 queries/s |
 |         | Index Size |           28 MB |          35 MB |
 
-*Lucene 9.1.0 on JVM 11.0.14.1 (Ubuntu)
+* JVM Versions:
+OpenJDK Runtime Environment (build 18-ea+36-Ubuntu-1) (Linux)
+OpenJDK Runtime Environment (build 17.0.3+7-Raspbian-1deb11u1rpt1) (LinuxPi)
+OpenJDK Runtime Environment Temurin-18.0.1+10 (build 18.0.1+10) (Windows)
+OpenJDK Runtime Environment (build 17.0.2+8-1) (FreeBSD)
 
 ### Storing Fields with Compression, Indexing and Retrieval
+
 - clone repo
 - bundle install
 - rake ferret_compression_benchmark
 
-Results on Linux, 0.13.7, on old Intel Core i5 from 2015:
+Results on Linux, 0.14.0, on old Intel Core i5 from 2015:
 
-| Compression | Index & Store | Retrieve      | Index size |
-|-------------|---------------|---------------|------------|
-| none        |   4866 docs/s | 153853 docs/s |      43 MB |
-| brotli      |   3539 docs/s |  58315 docs/s |      36 MB |
-| bzip2       |   2624 docs/s |  15382 docs/s |      38 MB |
-| lz4         |   4639 docs/s | 127100 docs/s |      41 MB |
+| Compression | Index & Store | Retrieve Title | Index size |
+|-------------|---------------|----------------|------------|
+| none        |   4862 docs/s |  278827 docs/s |      43 MB |
+| brotli      |   3559 docs/s |  178170 docs/s |      36 MB |
+| bzip2       |   2628 docs/s |   81877 docs/s |      38 MB |
+| lz4         |   4648 docs/s |  232236 docs/s |      41 MB |
 
 ## Future
 

data/ext/isomorfeus_ferret_ext/frb_index.c

@@ -1,6 +1,5 @@
 #include "frt_index.h"
 #include "isomorfeus_ferret.h"
-#include <ruby.h>
 
 // #undef close
 
@@ -16,8 +15,6 @@ VALUE cTermVector;
 VALUE cTermEnum;
 VALUE cTermDocEnum;
 
-VALUE cLazyDoc;
-VALUE cLazyDocData;
 VALUE cIndexWriter;
 VALUE cIndexReader;
 
@@ -59,15 +56,17 @@ static VALUE sym_with_positions_offsets;
 static ID fsym_content;
 
 static ID id_term;
-static ID id_fields;
 static ID id_fld_num_map;
 static ID id_field_num;
 static ID id_boost;
 
+extern VALUE sym_each;
 extern rb_encoding *utf8_encoding;
 extern void frb_set_term(VALUE rterm, FrtTerm *t);
 extern FrtAnalyzer *frb_get_cwrapped_analyzer(VALUE ranalyzer);
 extern VALUE frb_get_analyzer(FrtAnalyzer *a);
+extern VALUE frb_get_lazy_doc(FrtLazyDoc *lazy_doc);
+extern void Init_LazyDoc(void);
 
 /****************************************************************************
  *
@@ -1987,125 +1986,6 @@ frb_iw_set_use_compound_file(VALUE self, VALUE rval)
     return rval;
 }
 
-/****************************************************************************
- *
- * LazyDoc Methods
- *
- ****************************************************************************/
-
-static void frb_lzd_data_free(void *p) {
-    frt_lazy_doc_close((FrtLazyDoc *)p);
-}
-
-static size_t frb_lazy_doc_size(const void *p) {
-    return sizeof(FrtLazyDoc);
-    (void)p;
-}
-
-const rb_data_type_t frb_lazy_doc_t = {
-    .wrap_struct_name = "FrbLazyDoc",
-    .function = {
-        .dmark = NULL,
-        .dfree = frb_lzd_data_free,
-        .dsize = frb_lazy_doc_size,
-        .dcompact = NULL,
-        .reserved = {0},
-    },
-    .parent = NULL,
-    .data = NULL,
-    .flags = RUBY_TYPED_FREE_IMMEDIATELY
-};
-
-static VALUE frb_lzd_alloc(VALUE klass) {
-    FrtLazyDoc *ld = FRT_ALLOC(FrtLazyDoc);
-    return TypedData_Wrap_Struct(klass, &frb_lazy_doc_t, ld);
-}
-
-static VALUE frb_lazy_df_load(VALUE self, VALUE rkey, FrtLazyDocField *lazy_df) {
-    VALUE rdata = Qnil;
-    if (lazy_df) {
-        if (lazy_df->size == 1) {
-            char *data = frt_lazy_df_get_data(lazy_df, 0);
-            rdata = rb_str_new(data, lazy_df->data[0].length);
-            rb_enc_associate(rdata, lazy_df->data[0].encoding);
-        } else {
-            int i;
-            VALUE rstr;
-            rdata = rb_ary_new2(lazy_df->size);
-            for (i = 0; i < lazy_df->size; i++) {
-                char *data = frt_lazy_df_get_data(lazy_df, i);
-                rstr = rb_str_new(data, lazy_df->data[i].length);
-                rb_enc_associate(rstr, lazy_df->data[i].encoding);
-                rb_ary_store(rdata, i, rstr);
-            }
-        }
-        rb_hash_aset(self, rkey, rdata);
-    }
-    return rdata;
-}
-
-/*
- *  call-seq:
- *     lazy_doc.default(key) -> string
- *
- *  This method is used internally to lazily load fields. You should never
- *  really need to call it yourself.
- */
-static VALUE frb_lzd_default(VALUE self, VALUE rkey) {
-    FrtLazyDoc *lazy_doc = (FrtLazyDoc *)DATA_PTR(rb_ivar_get(self, id_data));
-    ID field = frb_field(rkey);
-    VALUE rfield = ID2SYM(field);
-
-    return frb_lazy_df_load(self, rfield, frt_lazy_doc_get(lazy_doc, field));
-}
-
-/*
- *  call-seq:
- *     lazy_doc.fields -> array of available fields
- *
- *  Returns the list of fields stored for this particular document. If you try
- *  to access any of these fields in the document the field will be loaded.
- *  Try to access any other field an nil will be returned.
- */
-static VALUE frb_lzd_fields(VALUE self) {
-    return rb_ivar_get(self, id_fields);
-}
-
-/*
- *  call-seq:
- *     lazy_doc.load -> lazy_doc
- *
- *  Load all unloaded fields in the document from the index.
- */
-static VALUE frb_lzd_load(VALUE self) {
-    FrtLazyDoc *lazy_doc = (FrtLazyDoc *)DATA_PTR(rb_ivar_get(self, id_data));
-    int i;
-    for (i = 0; i < lazy_doc->size; i++) {
-        FrtLazyDocField *lazy_df = lazy_doc->fields[i];
-        frb_lazy_df_load(self, ID2SYM(lazy_df->name), lazy_df);
-    }
-    return self;
-}
-
-VALUE frb_get_lazy_doc(FrtLazyDoc *lazy_doc) {
-    int i;
-    VALUE rfields = rb_ary_new2(lazy_doc->size);
-
-    VALUE self, rdata;
-    self = rb_hash_new();
-    OBJSETUP(self, cLazyDoc, T_HASH);
-
-    rdata = TypedData_Wrap_Struct(cLazyDocData, &frb_lazy_doc_t, lazy_doc);
-    rb_ivar_set(self, id_data, rdata);
-
-    for (i = 0; i < lazy_doc->size; i++) {
-        rb_ary_store(rfields, i, ID2SYM(lazy_doc->fields[i]->name));
-    }
-    rb_ivar_set(self, id_fields, rfields);
-
-    return self;
-}
-
 /****************************************************************************
  *
  * IndexReader Methods
@@ -2743,12 +2623,33 @@ frb_ir_tk_fields(VALUE self)
  *  Returns the current version of the index reader.
  */
 static VALUE
-frb_ir_version(VALUE self)
-{
+frb_ir_version(VALUE self) {
     FrtIndexReader *ir = (FrtIndexReader *)DATA_PTR(self);
     return ULL2NUM(ir->sis->version);
 }
 
+static VALUE frb_ir_to_enum(VALUE self) {
+    return rb_enumeratorize(self, sym_each, 0, NULL);
+}
+
+static VALUE frb_ir_each(VALUE self) {
+    FrtIndexReader *ir = (FrtIndexReader *)DATA_PTR(self);
+    if (rb_block_given_p()) {
+        long i;
+        long max_doc = ir->max_doc(ir);
+        VALUE rld;
+        for (i = 0; i < max_doc; i++) {
+            if (ir->is_deleted(ir, i)) continue;
+            rld = frb_get_lazy_doc(ir->get_lazy_doc(ir, i));
+            rb_yield(rld);
+        }
+        return self;
+    } else {
+        return frb_ir_to_enum(self);
+    }
+
+}
+
 /****************************************************************************
  *
  * Init Functions
@@ -3350,48 +3251,6 @@ void Init_IndexWriter(void) {
     rb_define_method(cIndexWriter, "use_compound_file=", frb_iw_set_use_compound_file, 1);
 }
 
-/*
- *  Document-class: Ferret::Index::LazyDoc
- *
- *  == Summary
- *
- *  When a document is retrieved from the index a LazyDoc is returned.
- *  Actually, LazyDoc is just a modified Hash object which lazily adds fields
- *  to itself when they are accessed. You should note that the keys method
- *  will return nothing until you actually access one of the fields. To see
- *  what fields are available use LazyDoc#fields rather than LazyDoc#keys. To
- *  load all fields use the LazyDoc#load method.
- *
- *  == Example
- *
- *    doc = index_reader[0]
- *
- *    doc.keys     #=> []
- *    doc.values   #=> []
- *    doc.fields   #=> [:title, :content]
- *
- *    title = doc[:title] #=> "the title"
- *    doc.keys     #=> [:title]
- *    doc.values   #=> ["the title"]
- *    doc.fields   #=> [:title, :content]
- *
- *    doc.load
- *    doc.keys     #=> [:title, :content]
- *    doc.values   #=> ["the title", "the content"]
- *    doc.fields   #=> [:title, :content]
- */
-void Init_LazyDoc(void) {
-    id_fields = rb_intern("@fields");
-
-    cLazyDoc = rb_define_class_under(mIndex, "LazyDoc", rb_cHash);
-    rb_define_method(cLazyDoc, "default", frb_lzd_default, 1);
-    rb_define_method(cLazyDoc, "load",    frb_lzd_load, 0);
-    rb_define_method(cLazyDoc, "fields",  frb_lzd_fields, 0);
-
-    cLazyDocData = rb_define_class_under(cLazyDoc, "LazyDocData", rb_cObject);
-    rb_define_alloc_func(cLazyDocData, frb_lzd_alloc);
-}
-
 /*
  *  Document-class: Ferret::Index::IndexReader
  *
@@ -3405,36 +3264,38 @@ void Init_LazyDoc(void) {
 void Init_IndexReader(void) {
     cIndexReader = rb_define_class_under(mIndex, "IndexReader", rb_cObject);
     rb_define_alloc_func(cIndexReader, frb_ir_alloc);
-    rb_define_method(cIndexReader, "initialize",     frb_ir_init, 1);
-    rb_define_method(cIndexReader, "set_norm",       frb_ir_set_norm, 3);
-    rb_define_method(cIndexReader, "norms",          frb_ir_norms, 1);
+    rb_define_method(cIndexReader, "initialize",     frb_ir_init,          1);
+    rb_define_method(cIndexReader, "set_norm",       frb_ir_set_norm,      3);
+    rb_define_method(cIndexReader, "norms",          frb_ir_norms,         1);
     rb_define_method(cIndexReader, "get_norms_into", frb_ir_get_norms_into, 3);
-    rb_define_method(cIndexReader, "commit",         frb_ir_commit, 0);
-    rb_define_method(cIndexReader, "close",          frb_ir_close, 0);
+    rb_define_method(cIndexReader, "commit",         frb_ir_commit,        0);
+    rb_define_method(cIndexReader, "close",          frb_ir_close,         0);
     rb_define_method(cIndexReader, "has_deletions?", frb_ir_has_deletions, 0);
-    rb_define_method(cIndexReader, "delete",         frb_ir_delete, 1);
-    rb_define_method(cIndexReader, "deleted?",       frb_ir_is_deleted, 1);
-    rb_define_method(cIndexReader, "max_doc",        frb_ir_max_doc, 0);
-    rb_define_method(cIndexReader, "num_docs",       frb_ir_num_docs, 0);
-    rb_define_method(cIndexReader, "undelete_all",   frb_ir_undelete_all, 0);
-    rb_define_method(cIndexReader, "latest?",        frb_ir_is_latest, 0);
-    rb_define_method(cIndexReader, "get_document",   frb_ir_get_doc, -1);
-    rb_define_method(cIndexReader, "[]",             frb_ir_get_doc, -1);
-    rb_define_method(cIndexReader, "term_vector",    frb_ir_term_vector, 2);
-    rb_define_method(cIndexReader, "term_vectors",   frb_ir_term_vectors, 1);
-    rb_define_method(cIndexReader, "term_docs",      frb_ir_term_docs, 0);
+    rb_define_method(cIndexReader, "delete",         frb_ir_delete,        1);
+    rb_define_method(cIndexReader, "deleted?",       frb_ir_is_deleted,    1);
+    rb_define_method(cIndexReader, "max_doc",        frb_ir_max_doc,       0);
+    rb_define_method(cIndexReader, "num_docs",       frb_ir_num_docs,      0);
+    rb_define_method(cIndexReader, "undelete_all",   frb_ir_undelete_all,  0);
+    rb_define_method(cIndexReader, "latest?",        frb_ir_is_latest,     0);
+    rb_define_method(cIndexReader, "get_document",   frb_ir_get_doc,      -1);
+    rb_define_method(cIndexReader, "[]",             frb_ir_get_doc,      -1);
+    rb_define_method(cIndexReader, "term_vector",    frb_ir_term_vector,   2);
+    rb_define_method(cIndexReader, "term_vectors",   frb_ir_term_vectors,  1);
+    rb_define_method(cIndexReader, "term_docs",      frb_ir_term_docs,     0);
     rb_define_method(cIndexReader, "term_positions", frb_ir_term_positions, 0);
     rb_define_method(cIndexReader, "term_docs_for",  frb_ir_term_docs_for, 2);
     rb_define_method(cIndexReader, "term_positions_for", frb_ir_t_pos_for, 2);
-    rb_define_method(cIndexReader, "doc_freq",       frb_ir_doc_freq, 2);
-    rb_define_method(cIndexReader, "terms",          frb_ir_terms, 1);
-    rb_define_method(cIndexReader, "terms_from",     frb_ir_terms_from, 2);
-    rb_define_method(cIndexReader, "term_count",     frb_ir_term_count, 1);
-    rb_define_method(cIndexReader, "fields",         frb_ir_fields, 0);
-    rb_define_method(cIndexReader, "field_names",    frb_ir_fields, 0);
-    rb_define_method(cIndexReader, "field_infos",    frb_ir_field_infos, 0);
-    rb_define_method(cIndexReader, "tokenized_fields", frb_ir_tk_fields, 0);
-    rb_define_method(cIndexReader, "version",        frb_ir_version, 0);
+    rb_define_method(cIndexReader, "doc_freq",       frb_ir_doc_freq,      2);
+    rb_define_method(cIndexReader, "terms",          frb_ir_terms,         1);
+    rb_define_method(cIndexReader, "terms_from",     frb_ir_terms_from,    2);
+    rb_define_method(cIndexReader, "term_count",     frb_ir_term_count,    1);
+    rb_define_method(cIndexReader, "fields",         frb_ir_fields,        0);
+    rb_define_method(cIndexReader, "field_names",    frb_ir_fields,        0);
+    rb_define_method(cIndexReader, "field_infos",    frb_ir_field_infos,   0);
+    rb_define_method(cIndexReader, "tokenized_fields", frb_ir_tk_fields,   0);
+    rb_define_method(cIndexReader, "version",        frb_ir_version,       0);
+    rb_define_method(cIndexReader, "each",           frb_ir_each,          0);
+    rb_define_method(cIndexReader, "to_enum",        frb_ir_to_enum,       0);
 }
 
 /* rdoc hack

data/ext/isomorfeus_ferret_ext/frb_lazy_doc.c

@@ -0,0 +1,705 @@
+#include "frt_index.h"
+#include "isomorfeus_ferret.h"
+
+extern VALUE rb_hash_update(int argc, VALUE *argv, VALUE self);
+
+extern VALUE sym_each;
+extern ID id_eql;
+
+static VALUE sym_each_key;
+static VALUE sym_each_value;
+static ID id_compact;
+static ID id_equal;
+static ID id_except;
+static ID id_fields;
+static ID id_flatten;
+static ID id_ge;
+static ID id_get;
+static ID id_gt;
+static ID id_inspect;
+static ID id_invert;
+static ID id_le;
+static ID id_merge_bang;
+static ID id_reject;
+static ID id_select;
+static ID id_size;
+static ID id_slice;
+static ID id_to_h;
+static ID id_to_proc;
+static ID id_transform_keys;
+static ID id_transform_values;
+
+FrtLazyDoc empty_lazy_doc = {0};
+VALUE cLazyDoc;
+
+typedef struct rLazyDoc {
+  FrtHash    *hash;
+  FrtLazyDoc *doc;
+} rLazyDoc;
+
+/****************************************************************************
+ *
+ * LazyDoc Methods
+ *
+ ****************************************************************************/
+
+static void frb_ld_free(void *p) {
+  rLazyDoc *rld = (rLazyDoc *)p;
+  if (rld->doc != &empty_lazy_doc) {
+    frt_lazy_doc_close(rld->doc);
+  }
+  frt_h_destroy(rld->hash);
+  free(rld);
+}
+
+static size_t frb_ld_size(const void *p) {
+  return sizeof(rLazyDoc);
+  (void)p;
+}
+
+void rld_mark(void *key, void *value, void *arg) {
+  rb_gc_mark((VALUE)value);
+}
+
+static void frb_ld_mark(void *p) {
+  frt_h_each(((rLazyDoc *)p)->hash, rld_mark, NULL);
+}
+
+const rb_data_type_t frb_ld_t = {
+  .wrap_struct_name = "FrbLazyDoc",
+  .function = {
+    .dmark = frb_ld_mark,
+    .dfree = frb_ld_free,
+    .dsize = frb_ld_size,
+    .dcompact = NULL,
+    .reserved = {0},
+  },
+  .parent = NULL,
+  .data = NULL,
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+VALUE frb_get_lazy_doc(FrtLazyDoc *lazy_doc) {
+  rLazyDoc *rld = FRT_ALLOC(rLazyDoc);
+  rld->hash = frt_h_new_ptr(NULL);
+  rld->doc = lazy_doc;
+  return TypedData_Wrap_Struct(cLazyDoc, &frb_ld_t, rld);
+}
+
+static VALUE frb_ld_alloc(VALUE rclass) {
+  rLazyDoc *rld = FRT_ALLOC(rLazyDoc);
+  rld->hash = frt_h_new_ptr(NULL);
+  rld->doc = &empty_lazy_doc;
+  return TypedData_Wrap_Struct(rclass, &frb_ld_t, rld);
+}
+
+static VALUE frb_ld_df_load(VALUE self, VALUE rkey, FrtLazyDocField *lazy_df) {
+  rLazyDoc *rld = DATA_PTR(self);
+  VALUE rdata;
+  if (lazy_df->size == 1) {
+    char *data = frt_lazy_df_get_data(lazy_df, 0);
+    rdata = rb_str_new(data, lazy_df->data[0].length);
+    rb_enc_associate(rdata, lazy_df->data[0].encoding);
+  } else {
+    int i;
+    VALUE rstr;
+    rdata = rb_ary_new2(lazy_df->size);
+    for (i = 0; i < lazy_df->size; i++) {
+      char *data = frt_lazy_df_get_data(lazy_df, i);
+      rstr = rb_str_new(data, lazy_df->data[i].length);
+      rb_enc_associate(rstr, lazy_df->data[i].encoding);
+      rb_ary_store(rdata, i, rstr);
+    }
+  }
+  frt_h_set(rld->hash, (void *)rkey, (void *)rdata);
+  return rdata;
+}
+
+/*
+ *  call-seq:
+ *     lazy_doc.load -> lazy_doc
+ *
+ *  Load all unloaded fields in the document from the index.
+ */
+static VALUE frb_ld_load(VALUE self) {
+  rLazyDoc *rld = DATA_PTR(self);
+  FrtLazyDoc *ld = rld->doc;
+  if (ld->loaded) return self;
+  int i;
+  FrtLazyDocField *lazy_df;
+  for (i = 0; i < ld->size; i++) {
+    lazy_df = ld->fields[i];
+    if (!(lazy_df->loaded)) frb_ld_df_load(self, ID2SYM(lazy_df->name), lazy_df);
+  }
+  ld->loaded = true;
+  return self;
+}
+
+/*
+ *  call-seq:
+ *     lazy_doc.fields -> array of available fields
+ *
+ *  Returns the list of fields stored for this particular document. If you try
+ *  to access any of these fields in the document the field will be loaded.
+ *  Try to access any other field an nil will be returned.
+ */
+static VALUE frb_ld_fields(VALUE self) {
+  FrtLazyDoc *ld = ((rLazyDoc *)DATA_PTR(self))->doc;
+  VALUE rfields = rb_ivar_get(self, id_fields);
+  if (rfields == Qnil) {
+    int i;
+    rfields = rb_ary_new2(ld->size);
+    for (i = 0; i < ld->size; i++) {
+      rb_ary_store(rfields, i, ID2SYM(ld->fields[i]->name));
+    }
+    rb_ivar_set(self, id_fields, rfields);
+  }
+  return rfields;
+}
+
+void rld_to_hash(void *key, void *value, void *arg) {
+  rb_hash_aset((VALUE)arg, (VALUE)key, (VALUE)value);
+}
+
+static VALUE frb_ld_to_h(VALUE self) {
+  rLazyDoc *rld = (rLazyDoc *)DATA_PTR(self);
+  if (!rld->doc->loaded) frb_ld_load(self);
+  VALUE hash = rb_hash_new();
+  frt_h_each(rld->hash, rld_to_hash, (void *)hash);
+  return hash;
+}
+
+static VALUE frb_ld_lt(VALUE self, VALUE other) {
+  VALUE other_h;
+  if (TYPE(other) == T_HASH) {
+    other_h = other;
+  } else {
+    rLazyDoc *other_rld;
+    TypedData_Get_Struct(other, rLazyDoc, &frb_ld_t, other_rld);
+    other_h = frb_ld_to_h(other);
+  }
+  VALUE self_h = frb_ld_to_h(self);
+  return rb_funcall(self_h, id_lt, 1, other_h);
+}
+
+static VALUE frb_ld_le(VALUE self, VALUE other) {
+  VALUE other_h;
+  if (TYPE(other) == T_HASH) {
+    other_h = other;
+  } else {
+    rLazyDoc *other_rld;
+    TypedData_Get_Struct(other, rLazyDoc, &frb_ld_t, other_rld);
+    other_h = frb_ld_to_h(other);
+  }
+  VALUE self_h = frb_ld_to_h(self);
+  return rb_funcall(self_h, id_le, 1, other_h);
+}
+
+static VALUE frb_ld_equal(VALUE self, VALUE other) {
+  FrtLazyDoc *ld = ((rLazyDoc *)DATA_PTR(self))->doc;
+  int other_size;
+  VALUE other_h;
+  if (TYPE(other) == T_HASH) {
+    other_h = other;
+    other_size = FIX2INT(rb_funcall(other_h, id_size, 0));
+  } else {
+    rLazyDoc *other_rld;
+    TypedData_Get_Struct(other, rLazyDoc, &frb_ld_t, other_rld);
+    other_h = frb_ld_to_h(other);
+    other_size = other_rld->doc->size;
+  }
+  if (ld->size == other_size) {
+    VALUE self_h = frb_ld_to_h(self);
+    return rb_funcall(self_h, id_equal, 1, other_h);
+  }
+  return Qfalse;
+}
+
+static VALUE frb_ld_gt(VALUE self, VALUE other) {
+  VALUE other_h;
+  if (TYPE(other) == T_HASH) {
+    other_h = other;
+  } else {
+    rLazyDoc *other_rld;
+    TypedData_Get_Struct(other, rLazyDoc, &frb_ld_t, other_rld);
+    other_h = frb_ld_to_h(other);
+  }
+  VALUE self_h = frb_ld_to_h(self);
+  return rb_funcall(self_h, id_gt, 1, other_h);
+}
+
+static VALUE frb_ld_ge(VALUE self, VALUE other) {
+  VALUE other_h;
+  if (TYPE(other) == T_HASH) {
+    other_h = other;
+  } else {
+    rLazyDoc *other_rld;
+    TypedData_Get_Struct(other, rLazyDoc, &frb_ld_t, other_rld);
+    other_h = frb_ld_to_h(other);
+  }
+  VALUE self_h = frb_ld_to_h(self);
+  return rb_funcall(self_h, id_ge, 1, other_h);
+}
+
+static VALUE frb_ld_get(VALUE self, VALUE key) {
+  rLazyDoc *rld = (rLazyDoc *)DATA_PTR(self);
+  VALUE rval = (VALUE)frt_h_get(rld->hash, (void *)key);
+  if (rval) return rval;
+  if (TYPE(key) != T_SYMBOL) rb_raise(rb_eArgError, "key, must be a symbol");
+  FrtLazyDocField *df = frt_h_get(rld->doc->field_dictionary, (void *)SYM2ID(key));
+  if (df) return frb_ld_df_load(self, key, df);
+  return Qnil;
+}
+
+void rld_any(void *key, void *value, void *arg) {
+  VALUE *v = arg;
+  *v = rb_yield_values(2, (VALUE)key, (VALUE)value);
+}
+
+static VALUE frb_ld_assoc(VALUE self, VALUE key) {
+  rLazyDoc *rld = DATA_PTR(self);
+  VALUE value = (VALUE)frt_h_get(rld->hash, (void *)key);
+  if (!value) {
+    FrtLazyDoc *ld = rld->doc;
+    FrtLazyDocField *df = frt_h_get(ld->field_dictionary, (void *)SYM2ID(key));
+    if (!df) return Qnil;
+    if (df && !df->loaded) value = frb_ld_df_load(self, key, df);
+  }
+  VALUE a[2] = {key, value};
+  return rb_ary_new_from_values(2, a);
+}
+
+static VALUE frb_ld_any(int argc, VALUE *argv, VALUE self) {
+  rLazyDoc *rld = (rLazyDoc *)DATA_PTR(self);
+  FrtLazyDoc *ld = rld->doc;
+  if (argc == 0) {
+    if (!rb_block_given_p()) {
+      return (ld->size > 0) ? Qtrue : Qfalse;
+    } else {
+      if (!ld->loaded) frb_ld_load(self);
+      VALUE res = Qnil;
+      frt_h_each(rld->hash, rld_any, &res);
+      if (RTEST(res)) return Qtrue;
+      else return Qfalse;
+    }
+  } else if (argc == 1) {
+    VALUE obj = argv[0];
+    VALUE key = rb_funcall(obj, id_get, 1, 0);
+    VALUE a = frb_ld_assoc(self, key);
+    return rb_funcall(a, id_equal, 1, obj);
+  }
+  rb_raise(rb_eArgError, "at most one arg may be given");
+  return Qfalse;
+}
+
+static VALUE frb_ld_compact(VALUE self) {
+  VALUE hash = frb_ld_to_h(self);
+  return rb_funcall(hash, id_compact, 0);
+}
+
+static VALUE frb_ld_dig(int argc, VALUE *argv, VALUE self) {
+  if (argc == 0) rb_raise(rb_eArgError, "at least a key must be given");
+  VALUE key = argv[0];
+  if (TYPE(key) != T_SYMBOL) rb_raise(rb_eArgError, "first arg, key, must be a symbol");
+  VALUE value = frb_ld_get(self, key);
+  if (argc == 1) return value;
+  if (TYPE(value) == T_ARRAY && argc == 2) {
+    return rb_ary_entry(value, NUM2LONG(argv[1]));
+  }
+  return Qnil;
+}
+
+static VALUE frb_ld_to_enum(VALUE self) {
+  return rb_enumeratorize(self, sym_each, 0, NULL);
+}
+
+static VALUE frb_ld_to_key_enum(VALUE self) {
+  return rb_enumeratorize(self, sym_each_key, 0, NULL);
+}
+
+static VALUE frb_ld_to_value_enum(VALUE self) {
+  return rb_enumeratorize(self, sym_each_value, 0, NULL);
+}
+
+void rld_each(void *key, void *value, void *arg) {
+  rb_yield_values(2, (VALUE)key, (VALUE)value);
+}
+
+static VALUE frb_ld_each(VALUE self) {
+  rLazyDoc *rld = (rLazyDoc *)DATA_PTR(self);
+  FrtLazyDoc *ld = rld->doc;
+  if (!ld->loaded) frb_ld_load(self);
+  if (rb_block_given_p()) {
+    frt_h_each(rld->hash, rld_each, NULL);
+    return self;
+  } else {
+    return frb_ld_to_enum(self);
+  }
+}
+
+void rld_each_key(void *key, void *value, void *arg) {
+  rb_yield((VALUE)key);
+}
+
+static VALUE frb_ld_each_key(VALUE self) {
+  rLazyDoc *rld = (rLazyDoc *)DATA_PTR(self);
+  FrtLazyDoc *ld = rld->doc;
+  if (!ld->loaded) frb_ld_load(self);
+  if (rb_block_given_p()) {
+    frt_h_each(rld->hash, rld_each_key, NULL);
+    return self;
+  } else {
+    return frb_ld_to_key_enum(self);
+  }
+}
+
+void rld_each_value(void *key, void *value, void *arg) {
+  rb_yield((VALUE)value);
+}
+
+static VALUE frb_ld_each_value(VALUE self) {
+  rLazyDoc *rld = (rLazyDoc *)DATA_PTR(self);
+  FrtLazyDoc *ld = rld->doc;
+  if (!ld->loaded) frb_ld_load(self);
+  if (rb_block_given_p()) {
+    frt_h_each(rld->hash, rld_each_value, NULL);
+    return self;
+  } else {
+    return frb_ld_to_value_enum(self);
+  }
+}
+
+static VALUE frb_ld_empty(VALUE self) {
+  FrtLazyDoc *ld = ((rLazyDoc *)DATA_PTR(self))->doc;
+  return (ld->size == 0) ? Qtrue : Qfalse;
+}
+
+static VALUE frb_ld_eql(VALUE self, VALUE other) {
+  FrtLazyDoc *ld = ((rLazyDoc *)DATA_PTR(self))->doc;
+  rLazyDoc *other_rld;
+  int other_size;
+  VALUE other_h;
+  if (TYPE(other) == T_HASH) {
+    other_h = other;
+    other_size = FIX2INT(rb_funcall(other_h, id_size, 0));
+  } else {
+    TypedData_Get_Struct(other, rLazyDoc, &frb_ld_t, other_rld);
+    other_h = frb_ld_to_h(other);
+    other_size = other_rld->doc->size;
+  }
+  if (ld->size == other_size) {
+    VALUE self_h = frb_ld_to_h(self);
+    return rb_funcall(self_h, id_eql, 1, other_h);
+  }
+  return Qfalse;
+}
+
+static VALUE frb_ld_except(int argc, VALUE *argv, VALUE self) {
+  VALUE hash = frb_ld_to_h(self);
+  return rb_funcallv(hash, id_except, argc, argv);
+}
+
+static VALUE frb_ld_fetch(int argc, VALUE *argv, VALUE self) {
+  VALUE key = argv[0];
+  if (TYPE(key) != T_SYMBOL) rb_raise(rb_eArgError, "first arg must be a symbol");
+  VALUE res = frb_ld_get(self, key);
+  if (argc == 1) {
+    if (res == Qnil && rb_block_given_p()) return rb_yield(key);
+    return res;
+  }
+  if (argc == 2) {
+    if (res == Qnil) return argv[1];
+    return res;
+  }
+  rb_raise(rb_eArgError, "too many args, only two allowed: key, default_value");
+}
+
+static VALUE frb_ld_fetch_values(int argc, VALUE *argv, VALUE self) {
+  rLazyDoc *rld = DATA_PTR(self);
+  if (!rld->doc->loaded) frb_ld_load(self);
+  VALUE ary = rb_ary_new();
+  int i;
+  VALUE value;
+  for (i=0; i<argc; i++) {
+    value = (VALUE)frt_h_get(rld->hash, (void *)argv[i]);
+    if (value) rb_ary_push(ary, value);
+    else if (rb_block_given_p()) {
+      value = rb_yield(argv[i]);
+      rb_ary_push(ary, value);
+    }
+  }
+  if (FIX2INT(rb_funcall(ary, id_size, 0)) == 0) rb_raise(rb_eException, "nothing found for given keys");
+  return ary;
+}
+
+static VALUE frb_ld_filter(VALUE self) {
+  VALUE hash = frb_ld_to_h(self);
+  return rb_funcall_passing_block(hash, id_select, 0, NULL);
+}
+
+void rld_flatten(void *key, void *value, void *arg) {
+  rb_ary_push((VALUE)arg, (VALUE)key);
+  rb_ary_push((VALUE)arg, (VALUE)value);
+}
+
+static VALUE frb_ld_flatten(int argc, VALUE *argv, VALUE self) {
+  rLazyDoc *rld = (rLazyDoc *)DATA_PTR(self);
+  if (!rld->doc->loaded) frb_ld_load(self);
+  VALUE ary = rb_ary_new();
+  frt_h_each(rld->hash, rld_flatten, (void *)ary);
+  if (argc == 1) {
+    int level = FIX2INT(argv[0]) - 1;
+    VALUE rlevel = INT2FIX(level);
+    rb_funcall(ary, id_flatten, 1, rlevel);
+  }
+  return ary;
+}
+
+static VALUE frb_ld_has_key(VALUE self, VALUE key) {
+  if (TYPE(key) != T_SYMBOL) rb_raise(rb_eArgError, "arg must be a symbol");
+  VALUE hk = Qfalse;
+  FrtLazyDoc *ld = ((rLazyDoc *)DATA_PTR(self))->doc;
+  ID dfkey = SYM2ID(key);
+  FrtLazyDocField *df = frt_h_get(ld->field_dictionary, (void *)dfkey);
+  if (df) hk = Qtrue;
+  return hk;
+}
+
+static VALUE frb_ld_has_value(VALUE self, VALUE value) {
+  rLazyDoc *rld = (rLazyDoc *)DATA_PTR(self);
+  FrtLazyDoc *ld = rld->doc;
+  if (!ld->loaded) frb_ld_load(self);
+  int i;
+  VALUE hvalue;
+  for (i=0; i<ld->size; i++) {
+    hvalue = (VALUE)frt_h_get(rld->hash, (void *)ID2SYM(ld->fields[i]->name));
+    hvalue = rb_funcall(hvalue, id_equal, 1, value);
+    if (hvalue == Qtrue) return Qtrue;
+  }
+  return Qfalse;
+}
+
+static VALUE frb_ld_inspect(VALUE self) {
+  VALUE hash = frb_ld_to_h(self);
+  return rb_funcall(hash, id_inspect, 0);
+}
+
+static VALUE frb_ld_invert(VALUE self) {
+  VALUE hash = frb_ld_to_h(self);
+  return rb_funcall(hash, id_invert, 0);
+}
+
+static VALUE frb_ld_key(VALUE self, VALUE value) {
+  rLazyDoc *rld = (rLazyDoc *)DATA_PTR(self);
+  FrtLazyDoc *ld = rld->doc;
+  if (!ld->loaded) frb_ld_load(self);
+  int i;
+  VALUE hvalue;
+  for (i=0; i<ld->size; i++) {
+    hvalue = (VALUE)frt_h_get(rld->hash, (void *)ID2SYM(ld->fields[i]->name));
+    hvalue = rb_funcall(hvalue, id_equal, 1, value);
+    if (hvalue == Qtrue) return ID2SYM(ld->fields[i]->name);
+  }
+  return Qnil;
+}
+
+static VALUE frb_ld_length(VALUE self) {
+  FrtLazyDoc *ld = ((rLazyDoc *)DATA_PTR(self))->doc;
+  return INT2FIX(ld->size);
+}
+
+static VALUE frb_ld_merge(int argc, VALUE *argv, VALUE self) {
+  rLazyDoc *rld = (rLazyDoc *)DATA_PTR(self);
+  if (!rld->doc->loaded) frb_ld_load(self);
+  VALUE hash = frb_ld_to_h(self);
+  return rb_funcall_passing_block(hash, id_merge_bang, argc, argv);
+}
+
+static VALUE frb_ld_rassoc(VALUE self, VALUE value) {
+  VALUE key = frb_ld_key(self, value);
+  if (key == Qnil) return Qnil;
+  VALUE a[2] = {key, value};
+  return rb_ary_new_from_values(2, a);
+}
+
+static VALUE frb_ld_reject(VALUE self) {
+  VALUE hash = frb_ld_to_h(self);
+  return rb_funcall_passing_block(hash, id_reject, 0, NULL);
+}
+
+static VALUE frb_ld_slice(int argc, VALUE *argv, VALUE self) {
+  VALUE hash = frb_ld_to_h(self);
+  return rb_funcallv(hash, id_slice, argc, argv);
+}
+
+void rld_to_a(void *key, void *value, void *arg) {
+  VALUE ary = rb_ary_new();
+  rb_ary_push(ary, (VALUE)key);
+  rb_ary_push(ary, (VALUE)value);
+  rb_ary_push((VALUE)arg, ary);
+}
+
+static VALUE frb_ld_to_a(VALUE self) {
+  rLazyDoc *rld = DATA_PTR(self);
+  if (!rld->doc->loaded) frb_ld_load(self);
+  VALUE ary = rb_ary_new();
+  frt_h_each(rld->hash, rld_to_a, (void *)ary);
+  return ary;
+}
+
+static VALUE frb_ld_to_ha(VALUE self) {
+  VALUE hash = frb_ld_to_h(self);
+  if (!rb_block_given_p()) return hash;
+  return rb_funcall_passing_block(hash, id_to_h, 0, NULL);
+}
+
+static VALUE frb_ld_to_proc(VALUE self) {
+  VALUE hash = frb_ld_to_h(self);
+  return rb_funcall(hash, id_to_proc, 0);
+}
+
+static VALUE frb_ld_transform_keys(int argc, VALUE *argv, VALUE self) {
+  VALUE hash = frb_ld_to_h(self);
+  return rb_funcall_passing_block(hash, id_transform_keys, argc, argv);
+}
+
+static VALUE frb_ld_transform_values(VALUE self) {
+  VALUE hash = frb_ld_to_h(self);
+  return rb_funcall_passing_block(hash, id_transform_values, 0, NULL);
+}
+
+void rld_values(void *key, void *value, void *arg) {
+  rb_ary_push((VALUE)arg, (VALUE)value);
+}
+
+static VALUE frb_ld_values(VALUE self) {
+  rLazyDoc *rld = DATA_PTR(self);
+  if (!rld->doc->loaded) frb_ld_load(self);
+  VALUE ary = rb_ary_new();
+  frt_h_each(rld->hash, rld_values, (void *)ary);
+  return ary;
+}
+
+static VALUE frb_ld_values_at(int argc, VALUE *argv, VALUE self) {
+  rLazyDoc *rld = DATA_PTR(self);
+  if (!rld->doc->loaded) frb_ld_load(self);
+  VALUE ary = rb_ary_new();
+  int i;
+  VALUE value;
+  for (i=0; i<argc; i++) {
+    value = (VALUE)frt_h_get(rld->hash, (void *)argv[i]);
+    if (value) rb_ary_push(ary, value);
+  	else rb_ary_push(ary, Qnil);
+  }
+  return ary;
+}
+
+/*
+ *  Document-class: Ferret::Index::LazyDoc
+ *
+ *  == Summary
+ *
+ *  When a document is retrieved from the index a LazyDoc is returned.
+ *  It inherits from rubys Hash class, however it is read only.
+ *  LazyDoc lazily adds fields to itself when they are accessed or
+ *  automatically loads all fields if needed.
+ *  To load all fields use the LazyDoc#load method.
+ *  Methods from the Hash class, that would modify the LazyDoc itself,
+ *  are not supported, .
+ *
+ *  == Example
+ *
+ *    doc = index_reader[0]
+ *
+ *    doc.keys     #=> []
+ *    doc.values   #=> []
+ *    doc.fields   #=> [:title, :content]
+ *
+ *    title = doc[:title] #=> "the title"
+ *    doc.keys     #=> [:title]
+ *    doc.values   #=> ["the title"]
+ *    doc.fields   #=> [:title, :content]
+ *
+ *    doc.load
+ *    doc.keys     #=> [:title, :content]
+ *    doc.values   #=> ["the title", "the content"]
+ *    doc.fields   #=> [:title, :content]
+ */
+void Init_LazyDoc(void) {
+  sym_each_key = ID2SYM(rb_intern("each_key"));
+  sym_each_value = ID2SYM(rb_intern("each_value"));
+  id_compact = rb_intern("compact");
+  id_equal = rb_intern("==");
+  id_except = rb_intern("except");
+  id_fields = rb_intern("@fields");
+  id_flatten = rb_intern("flatten");
+  id_ge = rb_intern(">=");
+  id_get = rb_intern("[]");
+  id_gt = rb_intern(">");
+  id_inspect = rb_intern("inspect");
+  id_invert = rb_intern("invert");
+  id_le = rb_intern("<=");
+  id_merge_bang = rb_intern("merge!");
+  id_reject = rb_intern("reject");
+  id_select = rb_intern("select");
+  id_size = rb_intern("size");
+  id_slice = rb_intern("slice");
+  id_to_h = rb_intern("to_h");
+  id_to_proc = rb_intern("to_proc");
+  id_transform_keys = rb_intern("transform_keys");
+  id_transform_values = rb_intern("transform_values");
+
+  cLazyDoc = rb_define_class_under(mIndex, "LazyDoc", rb_cObject);
+  rb_include_module(cLazyDoc, rb_mEnumerable);
+  rb_define_alloc_func(cLazyDoc, frb_ld_alloc);
+  rb_define_method(cLazyDoc, "load",     frb_ld_load,     0);
+  rb_define_method(cLazyDoc, "fields",   frb_ld_fields,   0);
+  rb_define_method(cLazyDoc, "keys",     frb_ld_fields,   0);
+  rb_define_method(cLazyDoc, "<",        frb_ld_lt,       1);
+  rb_define_method(cLazyDoc, "<=",       frb_ld_le,       1);
+  rb_define_method(cLazyDoc, "==",       frb_ld_equal,    1);
+  rb_define_method(cLazyDoc, ">",        frb_ld_gt,       1);
+  rb_define_method(cLazyDoc, ">=",       frb_ld_ge,       1);
+  rb_define_method(cLazyDoc, "[]",       frb_ld_get,      1);
+  rb_define_method(cLazyDoc, "any?",     frb_ld_any,     -1);
+  rb_define_method(cLazyDoc, "assoc",    frb_ld_assoc,    1);
+  rb_define_method(cLazyDoc, "compact",  frb_ld_compact,  0);
+  rb_define_method(cLazyDoc, "dig",      frb_ld_dig,     -1);
+  rb_define_method(cLazyDoc, "each",     frb_ld_each,     0);
+  rb_define_method(cLazyDoc, "each_key", frb_ld_each_key, 0);
+  rb_define_method(cLazyDoc, "each_pair", frb_ld_each,    0);
+  rb_define_method(cLazyDoc, "each_value", frb_ld_each_value, 0);
+  rb_define_method(cLazyDoc, "empty?",   frb_ld_empty,    0);
+  rb_define_method(cLazyDoc, "eql?",     frb_ld_eql,      1);
+  rb_define_method(cLazyDoc, "except",   frb_ld_except,  -1);
+  rb_define_method(cLazyDoc, "fetch",    frb_ld_fetch,   -1);
+  rb_define_method(cLazyDoc, "fetch_values", frb_ld_fetch_values, -1);
+  rb_define_method(cLazyDoc, "filter",   frb_ld_filter,   0);
+  rb_define_method(cLazyDoc, "flatten",  frb_ld_flatten, -1);
+  rb_define_method(cLazyDoc, "has_key?", frb_ld_has_key,  1);
+  rb_define_method(cLazyDoc, "has_value?", frb_ld_has_value, 1);
+  rb_define_method(cLazyDoc, "include?", frb_ld_has_key,  1);
+  rb_define_method(cLazyDoc, "inspect",  frb_ld_inspect,  0);
+  rb_define_method(cLazyDoc, "invert",   frb_ld_invert,   0);
+  rb_define_method(cLazyDoc, "key",      frb_ld_key,      1);
+  rb_define_method(cLazyDoc, "key?",     frb_ld_has_key,  1);
+  rb_define_method(cLazyDoc, "length",   frb_ld_length,   0);
+  rb_define_method(cLazyDoc, "member?",  frb_ld_has_key,  1);
+  rb_define_method(cLazyDoc, "merge",    frb_ld_merge,   -1);
+  rb_define_method(cLazyDoc, "rassoc",   frb_ld_rassoc,   1);
+  rb_define_method(cLazyDoc, "reject",   frb_ld_reject,   0);
+  rb_define_method(cLazyDoc, "select",   frb_ld_filter,   0);
+  rb_define_method(cLazyDoc, "size",     frb_ld_length,   0);
+  rb_define_method(cLazyDoc, "slice",    frb_ld_slice,   -1);
+  rb_define_method(cLazyDoc, "to_a",     frb_ld_to_a,     0);
+  rb_define_method(cLazyDoc, "to_enum",  frb_ld_to_enum,  0);
+  rb_define_method(cLazyDoc, "to_h",     frb_ld_to_ha,     0);
+  rb_define_method(cLazyDoc, "to_hash",  frb_ld_to_h,     0);
+  rb_define_method(cLazyDoc, "to_proc",  frb_ld_to_proc,  0);
+  rb_define_method(cLazyDoc, "to_s",     frb_ld_inspect,  0);
+  rb_define_method(cLazyDoc, "transform_keys", frb_ld_transform_keys, -1);
+  rb_define_method(cLazyDoc, "transform_values", frb_ld_transform_values, 0);
+  rb_define_method(cLazyDoc, "value?",   frb_ld_has_value, 1);
+  rb_define_method(cLazyDoc, "values",   frb_ld_values,   0);
+  rb_define_method(cLazyDoc, "values_at", frb_ld_values_at, -1);
+}

data/ext/isomorfeus_ferret_ext/frt_hash.h

@@ -58,10 +58,10 @@ typedef struct FrtHash {
      * used outside of the Hash methods */
     FrtHashEntry *(*lookup_i)(struct FrtHash *self,
                               register const void *key);
-    unsigned long  (*hash_i)(const void *key);
-    int                 (*eq_i)(const void *key1, const void *key2);
-    void                (*free_key_i)(void *p);
-    void                (*free_value_i)(void *p);
+    unsigned long (*hash_i)(const void *key);
+    int           (*eq_i)(const void *key1, const void *key2);
+    void          (*free_key_i)(void *p);
+    void          (*free_value_i)(void *p);
 } FrtHash;
 
 /**
@@ -140,8 +140,7 @@ extern FrtHash *frt_h_new(frt_hash_ft hash,
  *    pass NULL in place of this parameter the value will not be destroyed.
  * @return A newly allocated Hash
  */
-extern FrtHash *frt_h_new_str(frt_free_ft free_key,
-                              frt_free_ft free_value);
+extern FrtHash *frt_h_new_str(frt_free_ft free_key, frt_free_ft free_value);
 
 /**
  * Create a new Hash that uses integers as its keys. The Hash will store all
@@ -258,8 +257,7 @@ extern void *frt_h_rem(FrtHash *self, const void *key, bool del_key);
  *                                  the existing key so no key was freed
  *   </pre>
  */
-extern FrtHashKeyStatus frt_h_set(FrtHash *self,
-                                     const void *key, void *value);
+extern FrtHashKeyStatus frt_h_set(FrtHash *self, const void *key, void *value);
 
 /**
  * Add the value +value+ to the Hash referencing it with key +key+. If

data/ext/isomorfeus_ferret_ext/frt_index.c

@@ -170,9 +170,9 @@ static char *fn_for_gen_field(char *buf,
  *
  ***************************************************************************/
 
-static unsigned long long co_hash(const void *key)
+static unsigned long co_hash(const void *key)
 {
-    return (unsigned long long)key;
+    return (unsigned long)key;
 }
 
 static int co_eq(const void *key1, const void *key2)
@@ -1163,6 +1163,7 @@ static FrtLazyDocField *lazy_df_new(ID name, const int size, FrtCompressionType
     self->data = FRT_ALLOC_AND_ZERO_N(FrtLazyDocFieldData, size);
     self->compression = compression;
     self->decompressed = false;
+    self->loaded = false;
     return self;
 }
 
@@ -1400,6 +1401,7 @@ char *frt_lazy_df_get_data(FrtLazyDocField *self, int i) {
                 frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)text, read_len);
                 text[read_len - 1] = '\0';
             }
+            self->loaded = true;
         }
     }
 
@@ -1473,6 +1475,7 @@ static FrtLazyDoc *lazy_doc_new(int size, FrtInStream *fdt_in)
     self->size = size;
     self->fields = FRT_ALLOC_AND_ZERO_N(FrtLazyDocField *, size);
     self->fields_in = frt_is_clone(fdt_in);
+    self->loaded = false;
     return self;
 }
 

data/ext/isomorfeus_ferret_ext/frt_index.h

@@ -529,7 +529,7 @@ extern FrtTVTerm *frt_tv_get_tv_term(FrtTermVector *tv, const char *term);
 
 /* * * FrtLazyDocField * * */
 typedef struct FrtLazyDocFieldData {
-    frt_off_t              start;
+    frt_off_t          start;
     int                length;
     rb_encoding        *encoding;
     FrtCompressionType compression; /* as stored */
@@ -545,6 +545,7 @@ typedef struct FrtLazyDocField {
     int                 len;  /* length of data elements concatenated */
     FrtCompressionType  compression; /* as configured */
     bool                decompressed;
+    bool                loaded;
 } FrtLazyDocField;
 
 extern char *frt_lazy_df_get_data(FrtLazyDocField *self, int i);
@@ -556,6 +557,7 @@ struct FrtLazyDoc {
     int             size;
     FrtLazyDocField **fields;
     FrtInStream     *fields_in;
+    bool            loaded;
 };
 
 extern void frt_lazy_doc_close(FrtLazyDoc *self);

data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c

@@ -29,6 +29,7 @@ VALUE sym_true;
 VALUE sym_false;
 VALUE sym_path;
 VALUE sym_dir;
+VALUE sym_each;
 
 /* Modules */
 VALUE mIsomorfeus;
@@ -272,12 +273,13 @@ void Init_isomorfeus_ferret_ext(void) {
     id_data = rb_intern("@data");
 
     /* Symbols */
-    sym_yes = ID2SYM(rb_intern("yes"));;
-    sym_no = ID2SYM(rb_intern("no"));;
-    sym_true = ID2SYM(rb_intern("true"));;
-    sym_false = ID2SYM(rb_intern("false"));;
-    sym_path = ID2SYM(rb_intern("path"));;
-    sym_dir = ID2SYM(rb_intern("dir"));;
+    sym_yes = ID2SYM(rb_intern("yes"));
+    sym_no = ID2SYM(rb_intern("no"));
+    sym_true = ID2SYM(rb_intern("true"));
+    sym_false = ID2SYM(rb_intern("false"));
+    sym_path = ID2SYM(rb_intern("path"));
+    sym_dir = ID2SYM(rb_intern("dir"));
+    sym_each = ID2SYM(rb_intern("each"));
 
     mIsomorfeus = rb_define_module("Isomorfeus");
     mFerret = rb_define_module_under(mIsomorfeus, "Ferret");

data/lib/isomorfeus/ferret/index/index.rb

@@ -5,6 +5,7 @@ module Isomorfeus
       # information on how to use this class.
       class Index
         include MonitorMixin
+        include Enumerable
         include Isomorfeus::Ferret::Store
         include Isomorfeus::Ferret::Search
 
@@ -485,15 +486,11 @@ module Isomorfeus
           end
         end
 
-        # iterate through all documents in the index. This method preloads the
-        # documents so you don't need to call #load on the document to load all the
-        # fields.
-        def each
+        # iterate through all documents in the index.
+        def each(&block)
           @dir.synchronize do
             ensure_reader_open
-            (0...@reader.max_doc).each do |i|
-              yield @reader[i].load unless @reader.deleted?(i)
-            end
+            @reader.each(&block)
           end
         end
 
@@ -679,7 +676,7 @@ module Isomorfeus
             docs_to_add = []
             query = do_process_query(query)
             @searcher.search_each(query, :limit => :all) do |id, score|
-              document = @searcher[id].load
+              document = @searcher[id].to_h
               if new_val.is_a?(Hash)
                 document.merge!(new_val)
               else new_val.is_a?(String) or new_val.is_a?(Symbol)
@@ -850,6 +847,12 @@ module Isomorfeus
           end
         end
 
+        def to_enum
+          @dir.synchronize do
+            ensure_reader_open
+            @reader.to_enum
+          end
+        end
 
         protected
           def ensure_writer_open()

data/lib/isomorfeus/ferret/version.rb

@@ -1,5 +1,5 @@
 module Isomorfeus
   module Ferret
-    VERSION = '0.13.12'
+    VERSION = '0.14.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: isomorfeus-ferret
 version: !ruby/object:Gem::Version
-  version: 0.13.12
+  version: 0.14.0
 platform: ruby
 authors:
 - Jan Biedermann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-20 00:00:00.000000000 Z
+date: 2022-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oj
@@ -181,6 +181,7 @@ files:
 - ext/isomorfeus_ferret_ext/fio_tmpfile.h
 - ext/isomorfeus_ferret_ext/frb_analysis.c
 - ext/isomorfeus_ferret_ext/frb_index.c
+- ext/isomorfeus_ferret_ext/frb_lazy_doc.c
 - ext/isomorfeus_ferret_ext/frb_qparser.c
 - ext/isomorfeus_ferret_ext/frb_search.c
 - ext/isomorfeus_ferret_ext/frb_store.c