ferret 0.10.11 → 0.10.12

data/ext/multimapper.h

@@ -0,0 +1,51 @@
+#ifndef FRT_MAPPER_H
+#define FRT_MAPPER_H
+
+#include "hash.h"
+
+typedef struct State
+{
+    int  (*next)(struct State *self, int c, int *states);
+    void (*destroy_i)(struct State *self);
+    int  (*is_match)(struct State *self, char **mapping);
+} State;
+
+typedef struct DeterministicState
+{
+    struct DeterministicState *next[256];
+    int longest_match;
+    char *mapping;
+    int mapping_len;
+} DeterministicState;
+
+typedef struct Mapping
+{
+    char *pattern;
+    char *replacement;
+} Mapping;
+
+typedef struct MultiMapper
+{
+    Mapping **mappings;
+    int size;
+    int capa;
+    DeterministicState **dstates;
+    int d_size;
+    int d_capa;
+    unsigned char alphabet[256];
+    int a_size;
+    HashTable *dstates_map;
+    State **nstates;
+    int nsize;
+    int *next_states;
+    int ref_cnt;
+} MultiMapper;
+
+extern MultiMapper *mulmap_new();
+extern void mulmap_add_mapping(MultiMapper *self, const char *p, const char *r);
+extern void mulmap_compile(MultiMapper *self);
+extern char *mulmap_map(MultiMapper *self, char *to, char *from, int capa);
+extern int mulmap_map_len(MultiMapper *self, char *to, char *from, int capa);
+extern void mulmap_destroy(MultiMapper *self);
+
+#endif

data/ext/r_analysis.c

@@ -20,6 +20,7 @@ static VALUE cRegExpTokenizer;
 static VALUE cAsciiLowerCaseFilter;
 static VALUE cLowerCaseFilter;
 static VALUE cStopFilter;
+static VALUE cMappingFilter;
 static VALUE cHyphenFilter;
 static VALUE cStemFilter;
 
@@ -48,13 +49,11 @@ static VALUE object_space;
 extern int ruby_re_search(struct re_pattern_buffer *, const char *, int, int,
                           int, struct re_registers *);
 
-/*
-static int
+int
 frt_rb_hash_size(VALUE hash)
 {
     return RHASH(hash)->tbl->num_entries;
 }
-*/
 
 /****************************************************************************
  *
@@ -468,8 +467,8 @@ frt_ts_get_text(VALUE self)
     VALUE rtext = Qnil;
     TokenStream *ts; 
     Data_Get_Struct(self, TokenStream, ts);
-    if (ts->text) {
-        if ((rtext = object_get(&ts->text)) == Qnil) {
+    if ((rtext = object_get(&ts->text)) == Qnil) {
+        if (ts->text) {
             rtext = rb_str_new2(ts->text);
             object_set(&ts->text, rtext);
         } 
@@ -539,7 +538,7 @@ typedef struct CWrappedTokenStream {
 static void
 cwrts_destroy_i(TokenStream *ts)
 {
-    rb_hash_delete(object_space, LONG2NUM(CWTS(ts)->rts));
+    rb_hash_delete(object_space, ((long)ts)|1);
     /*printf("rb_hash_size = %d\n", frt_rb_hash_size(object_space)); */
     free(ts);
 }
@@ -563,7 +562,8 @@ static TokenStream *
 cwrts_clone_i(TokenStream *orig_ts)
 {
     TokenStream *new_ts = ts_clone_size(orig_ts, sizeof(CWrappedTokenStream));
-    CWTS(new_ts)->rts = rb_funcall(CWTS(orig_ts)->rts, id_clone, 0);
+    VALUE rts = CWTS(new_ts)->rts = rb_funcall(CWTS(orig_ts)->rts, id_clone, 0);
+    rb_hash_aset(object_space, ((long)new_ts)|1, rts);
     return new_ts;
 }
 
@@ -583,7 +583,7 @@ frt_get_cwrapped_rts(VALUE rts)
         ts->clone_i = &cwrts_clone_i;
         ts->destroy_i = &cwrts_destroy_i;
         /* prevent from being garbage collected */
-        rb_hash_aset(object_space, LONG2NUM(rts), rts);
+        rb_hash_aset(object_space, ((long)ts)|1, rts);
         ts->ref_cnt = 1;
     }
     return ts;
@@ -621,6 +621,8 @@ typedef struct RegExpTokenStream {
 static void
 rets_destroy_i(TokenStream *ts)
 {
+    rb_hash_delete(object_space, ((long)ts)|1);
+    /*printf("rb_hash_size = %d\n", frt_rb_hash_size(object_space)); */
     free(ts);
 }
 
@@ -656,6 +658,7 @@ frt_rets_set_text(VALUE self, VALUE rtext)
     TokenStream *ts;
     GET_TS(ts, self);
 
+    rb_hash_aset(object_space, ((long)ts)|1, rtext);
     StringValue(rtext);
     RETS(ts)->rtext = rtext;
     RETS(ts)->curr_ind = 0;
@@ -723,12 +726,12 @@ rets_clone_i(TokenStream *orig_ts)
 static TokenStream *
 rets_new(VALUE rtext, VALUE regex, VALUE proc)
 {
-    TokenStream *ts;
+    TokenStream *ts = ts_new(RegExpTokenStream);
 
     if (rtext != Qnil) {
         rtext = StringValue(rtext);
+        rb_hash_aset(object_space, ((long)ts)|1, rtext);
     }
-    ts = ts_new(RegExpTokenStream);
     ts->reset = &rets_reset;
     ts->next = &rets_next;
     ts->clone_i = &rets_clone_i;
@@ -769,9 +772,6 @@ frt_rets_init(int argc, VALUE *argv, VALUE self)
 
     Frt_Wrap_Struct(self, &frt_rets_mark, &frt_rets_free, ts);
     object_add(ts, self);
-    /* no need to add to object space as it is going to ruby space
-     * rb_hash_aset(object_space, LONG2NUM((long)self), self);
-     */
     return self;
 }
 
@@ -973,6 +973,96 @@ frt_stop_filter_init(int argc, VALUE *argv, VALUE self)
     return self;
 }
 
+static __inline void frt_add_mapping_i(TokenStream *mf, VALUE from, char *to)
+{
+    switch (TYPE(from)) {
+        case T_STRING:
+            mapping_filter_add(mf, RSTRING(from)->ptr, to);
+            break;
+        case T_SYMBOL:
+            mapping_filter_add(mf, rb_id2name(SYM2ID(from)), to);
+            break;
+        default:
+            rb_raise(rb_eArgError,
+                     "cannot map from %s with MappingFilter",
+                     RSTRING(rb_obj_as_string(from))->ptr);
+            break;
+    }
+}
+
+static int frt_add_mappings_i(VALUE key, VALUE value, VALUE arg)
+{
+    if (key == Qundef) {
+        return ST_CONTINUE;
+    } else {
+        TokenStream *mf = (TokenStream *)arg;
+        char *to;
+        switch (TYPE(value)) {
+            case T_STRING:
+                to = RSTRING(value)->ptr;
+                break;
+            case T_SYMBOL:
+                to = rb_id2name(SYM2ID(value));
+                break;
+            default:
+                rb_raise(rb_eArgError,
+                         "cannot map to %s with MappingFilter",
+                         RSTRING(rb_obj_as_string(key))->ptr);
+                break;
+        }
+        if (TYPE(key) == T_ARRAY) {
+            int i;
+            for (i = RARRAY(key)->len - 1; i >= 0; i--) {
+                frt_add_mapping_i(mf, RARRAY(key)->ptr[i], to);
+            }
+        }
+        else {
+            frt_add_mapping_i(mf, key, to);
+        }
+    }
+    return ST_CONTINUE;
+}
+
+
+/* 
+ *  call-seq:
+ *     MappingFilter.new(token_stream, mapping) -> token_stream
+ *
+ *  Create an MappingFilter which maps strings in tokens. This is usually used
+ *  to map UTF-8 characters to ascii characters for easier searching and
+ *  better searche recall. The mapping is compiled into a Deterministic Finite
+ *  Automata so it is super fast. This Filter can therefor be used for
+ *  indexing very large datasets. Currently regular expressions are not
+ *  supported. If you are really interested in the feature, please contact me
+ *  at dbalmain@gmail.com.
+ *
+ *  token_stream:: TokenStream to be filtered
+ *  mapping::      Hash of mappings to apply to tokens. The key can be a
+ *                 String or an Array of Strings. The value must be a String
+ *
+ *  == Example
+ *
+ *     filt = MappingFilter.new(token_stream,
+ *                              {
+ *                                ['à','á','â','ã','ä','å'] => 'a',
+ *                                ['è','é','ê','ë','ē','ę'] => 'e'
+ *                              })
+ */
+static VALUE
+frt_mapping_filter_init(VALUE self, VALUE rsub_ts, VALUE mapping) 
+{
+    TokenStream *ts;
+    ts = frt_get_cwrapped_rts(rsub_ts);
+    ts = mapping_filter_new(ts);
+    rb_hash_foreach(mapping, frt_add_mappings_i, (VALUE)ts);
+    mulmap_compile(((MappingFilter *)ts)->mapper);
+    object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
+
+    Frt_Wrap_Struct(self, &frt_tf_mark, &frt_tf_free, ts);
+    object_add(ts, self);
+    return self;
+}
+
 /* 
  *  call-seq:
  *     StemFilter.new(token_stream) -> token_stream
@@ -1031,7 +1121,7 @@ typedef struct CWrappedAnalyzer
 static void
 cwa_destroy_i(Analyzer *a)
 {
-    rb_hash_delete(object_space, LONG2NUM(CWA(a)->ranalyzer));
+    rb_hash_delete(object_space, ((long)a)|1);
     /*printf("rb_hash_size = %d\n", frt_rb_hash_size(object_space)); */
     free(a);
 }
@@ -1059,7 +1149,7 @@ frt_get_cwrapped_analyzer(VALUE ranalyzer)
         a->ref_cnt   = 1;
         ((CWrappedAnalyzer *)a)->ranalyzer = ranalyzer;
         /* prevent from being garbage collected */
-        rb_hash_aset(object_space, LONG2NUM(ranalyzer), ranalyzer);
+        rb_hash_aset(object_space, ((long)a)|1, ranalyzer);
     }
     return a;
 }
@@ -1100,6 +1190,8 @@ frt_get_analyzer(Analyzer *a)
 static VALUE
 frt_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
 {
+    /* NOTE: Any changes made to this method may also need to be applied to
+     * frt_re_analyzer_token_stream */
     TokenStream *ts;
     Analyzer *a;
     GET_A(a, self);
@@ -1121,7 +1213,7 @@ lower = (argc ? RTEST(rlower) : dflt)
 
 /*
  *  call-seq:
- *     AsciiWhiteSpaceAnalyzer.new(lower = true) -> analyzer
+ *     AsciiWhiteSpaceAnalyzer.new(lower = false) -> analyzer
  *
  *  Create a new AsciiWhiteSpaceAnalyzer which downcases tokens by default
  *  but can optionally leave case as is. Lowercasing will only be done to
@@ -1142,7 +1234,7 @@ frt_a_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
 
 /*
  *  call-seq:
- *     WhiteSpaceAnalyzer.new(lower = true) -> analyzer
+ *     WhiteSpaceAnalyzer.new(lower = false) -> analyzer
  *
  *  Create a new WhiteSpaceAnalyzer which downcases tokens by default but can
  *  optionally leave case as is. Lowercasing will be done based on the current
@@ -1220,7 +1312,7 @@ get_rstopwords(const char **stop_words)
 
 /*
  *  call-seq:
- *     AsciiStandardAnalyzer.new(lower = true, stop_words = ENGLISH_STOP_WORDS)
+ *     AsciiStandardAnalyzer.new(lower = true, stop_words = FULL_ENGLISH_STOP_WORDS)
  *     -> analyzer
  *
  *  Create a new AsciiStandardAnalyzer which downcases tokens by default but
@@ -1253,7 +1345,7 @@ frt_a_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
 
 /*
  *  call-seq:
- *     StandardAnalyzer.new(stop_words=ENGLISH_STOP_WORDS, lower=true)
+ *     StandardAnalyzer.new(stop_words = FULL_ENGLISH_STOP_WORDS, lower=true)
  *     -> analyzer
  *
  *  Create a new StandardAnalyzer which downcases tokens by default but can
@@ -1377,7 +1469,6 @@ frt_re_analyzer_init(int argc, VALUE *argv, VALUE self)
 
     ts = rets_new(Qnil, regex, proc);
     rets = Data_Wrap_Struct(cRegExpTokenizer, &frt_rets_mark, &frt_rets_free, ts);
-    /* rb_hash_aset(object_space, LONG2NUM((long)rets), rets); */
     object_add(ts, rets);
 
     if (lower != Qfalse) {
@@ -1392,6 +1483,41 @@ frt_re_analyzer_init(int argc, VALUE *argv, VALUE self)
     return self;
 }
 
+/*
+ *  call-seq:
+ *     analyzer.token_stream(field_name, input) -> token_stream
+ *
+ *  Create a new TokenStream to tokenize +input+. The TokenStream created may
+ *  also depend on the +field_name+. Although this parameter is typically
+ *  ignored.
+ *
+ *  field_name:: name of the field to be tokenized
+ *  input::      data from the field to be tokenized
+ */
+static VALUE
+frt_re_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rtext)
+{
+    TokenStream *ts;
+    Analyzer *a;
+    GET_A(a, self);
+
+    StringValue(rtext);
+
+    ts = a_get_ts(a, frt_field(rfield), RSTRING(rtext)->ptr);
+
+    /* Make sure that there is no entry already */
+    object_set(&ts->text, rtext);
+    if (ts->next == &rets_next) {
+        RETS(ts)->rtext = rtext;
+        rb_hash_aset(object_space, ((long)ts)|1, rtext);
+    }
+    else {
+        RETS(((TokenFilter*)ts)->sub_ts)->rtext = rtext;
+        rb_hash_aset(object_space, ((long)((TokenFilter*)ts)->sub_ts)|1, rtext);
+    }
+    return get_rb_token_stream(ts);
+}
+
 /****************************************************************************
  *
  * Locale stuff
@@ -1728,6 +1854,55 @@ static void Init_HyphenFilter(void)
     rb_define_method(cHyphenFilter, "initialize", frt_hyphen_filter_init, 1);
 }
 
+/*
+ *  Document-class: Ferret::Analysis::MappingFilter
+ *
+ *  A MappingFilter maps strings in tokens. This is usually used to map UTF-8
+ *  characters to ascii characters for easier searching and better searche
+ *  recall. The mapping is compiled into a Deterministic Finite Automata so it
+ *  is super fast. This Filter can therefor be used for indexing very large
+ *  datasets. Currently regular expressions are not supported. If you are
+ *  really interested in the feature, please contact me at dbalmain@gmail.com.
+ *
+ *  == Example
+ *
+ *     mapping = {
+ *       ['à','á','â','ã','ä','å','ā','ă']         => 'a',
+ *       'æ'                                       => 'ae',
+ *       ['ď','đ']                                 => 'd',
+ *       ['ç','ć','č','ĉ','ċ']                     => 'c',
+ *       ['è','é','ê','ë','ē','ę','ě','ĕ','ė',]    => 'e',
+ *       ['ƒ']                                     => 'f',
+ *       ['ĝ','ğ','ġ','ģ']                         => 'g',
+ *       ['ĥ','ħ']                                 => 'h',
+ *       ['ì','ì','í','î','ï','ī','ĩ','ĭ']         => 'i',
+ *       ['į','ı','ij','ĵ']                         => 'j',
+ *       ['ķ','ĸ']                                 => 'k',
+ *       ['ł','ľ','ĺ','ļ','ŀ']                     => 'l',
+ *       ['ñ','ń','ň','ņ','ʼn','ŋ']                 => 'n',
+ *       ['ò','ó','ô','õ','ö','ø','ō','ő','ŏ','ŏ'] => 'o',
+ *       ['œ']                                     => 'oek',
+ *       ['ą']                                     => 'q',
+ *       ['ŕ','ř','ŗ']                             => 'r',
+ *       ['ś','š','ş','ŝ','ș']                     => 's',
+ *       ['ť','ţ','ŧ','ț']                         => 't',
+ *       ['ù','ú','û','ü','ū','ů','ű','ŭ','ũ','ų'] => 'u',
+ *       ['ŵ']                                     => 'w',
+ *       ['ý','ÿ','ŷ']                             => 'y',
+ *       ['ž','ż','ź']                             => 'z'
+ *     }
+ *     filt = MappingFilter.new(token_stream, mapping)
+ */
+static void Init_MappingFilter(void)
+{
+    cMappingFilter =
+        rb_define_class_under(mAnalysis, "MappingFilter", cTokenStream);
+    frt_mark_cclass(cMappingFilter);
+    rb_define_alloc_func(cMappingFilter, frt_data_alloc);
+    rb_define_method(cMappingFilter, "initialize",
+                     frt_mapping_filter_init, 2);
+}
+
 /*
  *  Document-class: Ferret::Analysis::StopFilter
  *
@@ -1999,7 +2174,7 @@ static void Init_WhiteSpaceAnalyzer(void)
  *  ascii-analyzers. If it were implemented in Ruby it would look like this;
  *
  *    class AsciiStandardAnalyzer
- *      def initialize(stop_words = ENGLISH_STOP_WORDS, lower = true)
+ *      def initialize(stop_words = FULL_ENGLISH_STOP_WORDS, lower = true)
  *        @lower = lower
  *        @stop_words = stop_words
  *      end
@@ -2036,7 +2211,7 @@ static void Init_AsciiStandardAnalyzer(void)
  *  it were implemented in Ruby it would look like this;
  *
  *    class StandardAnalyzer
- *      def initialize(stop_words = ENGLISH_STOP_WORDS, lower = true)
+ *      def initialize(stop_words = FULL_ENGLISH_STOP_WORDS, lower = true)
  *        @lower = lower
  *        @stop_words = stop_words
  *      end
@@ -2131,6 +2306,8 @@ static void Init_RegExpAnalyzer(void)
     rb_define_alloc_func(cRegExpAnalyzer, frt_data_alloc);
     rb_define_method(cRegExpAnalyzer, "initialize",
                      frt_re_analyzer_init, -1);
+    rb_define_method(cRegExpAnalyzer, "token_stream",
+                     frt_re_analyzer_token_stream, 2);
 }
 
 /* rdoc hack
@@ -2244,6 +2421,7 @@ Init_Analysis(void)
     Init_LowerCaseFilter();
     Init_HyphenFilter();
     Init_StopFilter();
+    Init_MappingFilter();
     Init_StemFilter();
 
     Init_Analyzer();

data/ext/r_search.c

@@ -124,7 +124,6 @@ extern VALUE cIndexReader;
 extern void frt_ir_free(void *p);
 extern void frt_ir_mark(void *p);
 
-
 extern void frt_set_term(VALUE rterm, Term *t);
 extern VALUE frt_get_analyzer(Analyzer *a);
 extern HashSet *frt_get_fields(VALUE rfields);
@@ -223,6 +222,113 @@ frt_td_to_s(int argc, VALUE *argv, VALUE self)
     return rstr;
 }
 
+/*
+ * Json Exportation - Loading each LazyDoc and formatting them into json
+ * This code is designed to get a VERY FAST json string, the goal was speed,
+ * not sexyness.
+ * Jeremie 'ahFeel' BORDIER
+ * ahFeel@rift.Fr
+ */
+__inline char *
+json_concat_string(char *s, char *field)
+{
+    *(s++) = '"';
+	while (*field) {
+		if (*field == '\"') {
+            *(s++) = '\'';
+            *(s++) = *(field++);
+            *(s++) = '\'';
+        }
+        else {
+            *(s++) = *(field++);
+        }
+    }
+    *(s++) = '"';
+    return s;
+}
+
+inline char *
+frt_lzd_load_to_json(LazyDoc *lzd, char **str, char *s, int *slen)
+{
+	int i, j;
+	int diff = s - *str;
+	int len = diff, l;
+	LazyDocField *f;
+	
+	for (i = 0; i < lzd->size; i++) {
+		f = lzd->fields[i];
+        /* 3 times length of field to make space for quoted quotes ('"') and
+         * 4 x field length to make space for '"' around fields and ','
+         * between fields. Add 100 for '[', ']' and good safety.
+         */
+        len += strlen(f->name) + f->len * 3 + 100 + 4 * f->size;
+    }
+
+    if (len > *slen) {
+        while (len > *slen) *slen = *slen << 1;
+        REALLOC_N(*str, char, *slen);
+        s = *str + diff;
+    }
+
+	for (i = 0; i < lzd->size; i++) {
+		f = lzd->fields[i];
+		if (i)  *(s++) = ',';
+        *(s++) = '"';
+        l = strlen(f->name);
+        memcpy(s, f->name, l);
+        s += l;
+        *(s++) = '"';
+        *(s++) = ':';
+        if (f->size > 1)  *(s++) = '[';
+		for (j = 0; j < f->size; j++) {
+			if (j) *(s++) = ',';
+			s = json_concat_string(s, lazy_df_get_data(f, j));
+		}
+        if (f->size > 1)  *(s++) = ']';
+	}
+	return s;
+}
+
+/*
+ *  call-seq:
+ *     top_doc.to_json() -> string
+ *
+ *  Returns a json represention of the top_doc.
+ */
+static VALUE
+frt_td_to_json(VALUE self)
+{
+	int i;
+	VALUE rhits = rb_funcall(self, id_hits, 0);
+	VALUE rhit;
+	LazyDoc *lzd;
+	Searcher *sea = (Searcher *)DATA_PTR(rb_funcall(self, id_searcher, 0));
+	const int num_hits = RARRAY(rhits)->len;
+	int doc_id;
+    int len = 32768;
+	char *str = ALLOC_N(char, len);
+    char *s = str;
+	VALUE rstr;
+
+    *(s++) = '[';
+	for (i = 0; i < num_hits; i++) {
+        if (i) *(s++) = ',';
+        *(s++) = '{';
+		rhit = RARRAY(rhits)->ptr[i];
+		doc_id = FIX2INT(rb_funcall(rhit, id_doc, 0));
+		lzd = sea->get_lazy_doc(sea, doc_id);
+		s = frt_lzd_load_to_json(lzd, &str, s, &len);
+        lazy_doc_close(lzd);
+        *(s++) = '}';
+	}
+    *(s++) = ']';
+    *(s++) = '\0';
+	rstr = rb_str_new2(str);
+	free(str);
+	return rstr;
+}
+
+
 /****************************************************************************
  *
  * Explanation Methods
@@ -1901,6 +2007,7 @@ frt_sf_init(int argc, VALUE *argv, VALUE self)
     VALUE rval;
     int type = SORT_TYPE_AUTO;
     int is_reverse = false;
+    char *field;
 
     if (rb_scan_args(argc, argv, "11", &rfield, &roptions) == 2) {
         if (Qnil != (rval = rb_hash_aref(roptions, sym_type))) {
@@ -1914,11 +2021,11 @@ frt_sf_init(int argc, VALUE *argv, VALUE self)
         }
     }
     if (NIL_P(rfield)) rb_raise(rb_eArgError, "must pass a valid field name");
-    rfield = rb_obj_as_string(rfield);
+    field = frt_field(rfield);
 
-    sf = sort_field_new(RSTRING(rfield)->ptr, type, is_reverse);
-    if (sf->field == NULL && RSTRING(rfield)->ptr != NULL) {
-        sf->field = estrdup(RSTRING(rfield)->ptr);
+    sf = sort_field_new(field, type, is_reverse);
+    if (sf->field == NULL && field) {
+        sf->field = estrdup(field);
     }
 
     Frt_Wrap_Struct(self, NULL, &frt_sf_free, sf);
@@ -2017,7 +2124,6 @@ frt_sort_free(void *p)
 {
     Sort *sort = (Sort *)p;
     object_del(sort);
-    object_del(sort->sort_fields);
     sort_destroy(sort);
 }
 
@@ -2025,7 +2131,10 @@ static void
 frt_sort_mark(void *p)
 {
     Sort *sort = (Sort *)p;
-    frt_gc_mark(sort->sort_fields);
+    int i;
+    for (i = 0; i < sort->size; i++) {
+        frt_gc_mark(sort->sort_fields[i]);
+    }
 }
 
 static VALUE
@@ -2147,11 +2256,6 @@ frt_sort_init(int argc, VALUE *argv, VALUE self)
                 sort_add_sort_field(sort, (SortField *)&SORT_FIELD_SCORE);
                 sort_add_sort_field(sort, (SortField *)&SORT_FIELD_DOC);
     }
-    rfields = rb_ary_new2(sort->size);
-    for (i = 0; i < sort->size; i++) {
-        rb_ary_store(rfields, i, object_get(sort->sort_fields[i]));
-    }
-    object_add(sort->sort_fields, rfields);
 
     return self;
 }
@@ -2166,7 +2270,12 @@ static VALUE
 frt_sort_get_fields(VALUE self)
 {
     GET_SORT();
-    return object_get(sort->sort_fields);
+    VALUE rfields = rb_ary_new2(sort->size);
+    int i;
+    for (i = 0; i < sort->size; i++) {
+        rb_ary_store(rfields, i, object_get(sort->sort_fields[i]));
+    }
+    return rfields;
 }
 
 
@@ -2374,9 +2483,9 @@ frt_sea_search_internal(Query *query, VALUE roptions, Searcher *sea)
             sea->arg = (void *)rval;
         }
         if (Qnil != (rval = rb_hash_aref(roptions, sym_sort))) {
-            if (TYPE(rval) != T_DATA) {
+            if (TYPE(rval) != T_DATA || CLASS_OF(rval) == cSortField) {
                 rval = frt_sort_init(1, &rval, frt_sort_alloc(cSort));
-            }
+            } 
             Data_Get_Struct(rval, Sort, sort);
         }
     }
@@ -2801,6 +2910,7 @@ Init_TopDocs(void)
     rb_set_class_path(cTopDocs, mSearch, td_class);
     rb_const_set(mSearch, rb_intern(td_class), cTopDocs);
     rb_define_method(cTopDocs, "to_s", frt_td_to_s, -1);
+    rb_define_method(cTopDocs, "to_json", frt_td_to_json, 0);
     id_hits = rb_intern("hits");
     id_total_hits = rb_intern("total_hits");
     id_max_score = rb_intern("max_score");