ferret 0.11.3 → 0.11.4

data/Rakefile

@@ -131,7 +131,7 @@ file "ext/#{EXT}" => ["ext/Makefile"] do
       puts
       puts "**********************************************************************"
       puts "You may need to call VCVARS32.BAT to set the environment variables."
-      puts '  c:\Program Files\Microsoft Visual Studio\VC98\Bin\VCVARS32.BAT'
+      puts '  "f:\Program Files\Microsoft Visual Studio\VC98\Bin\VCVARS32.BAT"'
       puts "**********************************************************************"
       puts
       raise e
@@ -175,6 +175,9 @@ PKG_FILES = FileList[
   '[-A-Z]*',
   'ext/**/*.[ch]', 
   'lib/**/*.rb', 
+  'lib/**/*.rhtml', 
+  'lib/**/*.css', 
+  'lib/**/*.js', 
   'test/**/*.rb',
   'test/**/wordfile',
   'rake_utils/**/*.rb',
@@ -205,6 +208,9 @@ else
     s.extensions << "ext/extconf.rb"
     s.require_path = 'lib'
     s.autorequire = 'ferret'
+    s.bindir = 'bin'
+    s.executables = ['ferret-browser']
+    s.default_executable = 'ferret-browser'
 
     #### Author and project details.
     s.author = "David Balmain"

data/bin/ferret-browser

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+
+$: << File.expand_path(File.join(File.basename(__FILE__), '../lib'))
+require 'ferret'
+require 'ferret/browser'
+
+require 'optparse'
+require 'ostruct'
+
+SERVER_OPTIONS = ['webrick']
+conf = OpenStruct.new(:host => '0.0.0.0', :port => 3301)
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename($0)} /path/to/index"
+  opts.separator ""
+  opts.separator "Specific Options:"
+
+  opts.on("-h", "--host HOSTNAME",
+          "Host for web server to bind to (default is all IPs)") { |conf.host| }
+  opts.on("-p", "--port NUM",
+          "Port for web server (defaults to #{conf.port})") { |conf.port| }
+  opts.on("-s", "--server NAME",
+          "Server to force (#{SERVER_OPTIONS.join(', ')}).") { |s| conf.server = s.to_sym }
+
+  opts.separator ""
+  opts.separator "Common options:"
+  
+  opts.on_tail("-?", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+  
+  opts.on_tail("-v", "--version", "Show version") do
+    puts Ferret::VERSION
+    exit
+  end
+end
+
+opts.parse! ARGV
+if ARGV.length != 1
+  puts opts
+  exit
+end
+@path = ARGV[0]
+
+# Load the Ferret index
+begin
+  @reader = Ferret::Index::IndexReader.new(@path)
+rescue Ferret::FileNotFoundError => e
+  puts "\033[31mCannot start Ferret. No index exists at \"\033[m" + 
+    "\033[33m#{@path}\033[m\033[31m\".\033[m"
+  exit
+rescue Exception => e
+  puts "\033[31mCannot start Ferret.\n\033[m\033[33m#{e.to_s}\031[m"
+  exit
+end
+
+unless conf.server 
+  conf.server = :webrick 
+end
+
+case conf.server.to_s
+when 'webrick'
+  require 'webrick/httpserver'
+  require 'ferret/browser/webrick'
+
+  # Mount the root
+  s = WEBrick::HTTPServer.new(:BindAddress => conf.host, :Port => conf.port)
+  s.mount "/s", WEBrick::HTTPServlet::FileHandler, Ferret::Browser::Controller::STATIC_DIR, true
+  s.mount "/", WEBrick::FerretBrowserHandler, @reader, @path
+
+  # Server up
+  trap(:INT) do
+    s.shutdown
+  end
+  s.start
+else
+  raise "server #{conf.server} not known. Must be one of [#{SERVER_OPTIONS.join(', ')}]"
+end

data/ext/analysis.c

@@ -161,8 +161,9 @@ __inline int mb_next_char(wchar_t *wchr, const char *s, mbstate_t *state)
             t++;
             ZEROSET(state, mbstate_t);
             num_bytes = (int)mbrtowc(wchr, t, MB_CUR_MAX, state);
-        } while ((num_bytes < 0) && (*wchr != 0) && (*t != 0));
+        } while ((num_bytes < 0) && (*t != 0));
         num_bytes = t - s;
+        if (*t == 0) *wchr = 0;
     }
     return num_bytes;
 }
@@ -1301,12 +1302,14 @@ Token *mb_lcf_next(TokenStream *ts)
 {
     wchar_t wbuf[MAX_WORD_SIZE + 1], *wchr;
     Token *tk = TkFilt(ts)->sub_ts->next(TkFilt(ts)->sub_ts);
+    int x;
+    wbuf[MAX_WORD_SIZE] = 0;
 
     if (tk == NULL) {
         return tk;
     }
 
-    if (mbstowcs(wbuf, tk->text, MAX_WORD_SIZE) <= 0) return tk;
+    if ((x=mbstowcs(wbuf, tk->text, MAX_WORD_SIZE)) <= 0) return tk;
     wchr = wbuf;
     while (*wchr != 0) {
         *wchr = towlower(*wchr);

data/ext/config.h

@@ -2,6 +2,7 @@
 #define FRT_DEFINES_H
 
 #include <sys/types.h>
+#include <limits.h>
 #include "posh.h"
 
 #ifndef false
@@ -21,7 +22,7 @@ typedef posh_i32_t f_i32;
 typedef posh_u64_t f_u64;
 typedef posh_i64_t f_i64;
 
-#if defined(_FILE_OFFSET_BITS) && (_FILE_OFFSET_BITS == 64)
+#if ( LONG_MAX == 2147483647 ) && defined(_FILE_OFFSET_BITS) && (_FILE_OFFSET_BITS == 64)
 #define F_OFF_T_PFX "ll"
 #else
 #define F_OFF_T_PFX "l"

data/ext/ferret.c

@@ -65,9 +65,9 @@ void
 object_add2(void *key, VALUE obj, const char *file, int line)
 {
     if (h_get(object_map, key))
-        printf("failed adding %x to %d; already contains %x. %s:%d\n",
-               (int)obj, (int)key, (int)h_get(object_map, key), file, line);
-    //printf("adding %d. now contains %d %s:%d\n", (int)key, ++hash_cnt, file, line);
+        printf("failed adding %lx to %ld; already contains %lx. %s:%d\n",
+               (long)obj, (long)key, (long)h_get(object_map, key), file, line);
+    //printf("adding %ld. now contains %d %s:%d\n", (long)key, ++hash_cnt, file, line);
     h_set(object_map, key, (void *)obj);
 }
 
@@ -76,7 +76,7 @@ void
 object_set2(void *key, VALUE obj, const char *file, int line)
 {
     //if (!h_get(object_map, key))
-      //printf("adding %d. now contains %d %s:%d\n", (int)key, ++hash_cnt, file, line);
+      //printf("adding %ld. now contains %d %s:%d\n", (long)key, ++hash_cnt, file, line);
     h_set(object_map, key, (void *)obj);
 }
 
@@ -85,8 +85,8 @@ void
 object_del2(void *key, const char *file, int line)
 {
     if (object_get(key) == Qnil) 
-        printf("failed deleting %d. %s:%d\n", (int)key, file, line);
-    //printf("deleting %d. now contains %d, %s:%d\n", (int)key, --hash_cnt, file, line);
+        printf("failed deleting %ld. %s:%d\n", (long)key, file, line);
+    //printf("deleting %ld. now contains %ld, %s:%d\n", (long)key, --hash_cnt, file, line);
     h_del(object_map, key);
 }
 
@@ -189,6 +189,31 @@ frt_field(VALUE rfield)
     return NULL;
 }
 
+/*
+ * Json Exportation - Loading each LazyDoc and formatting them into json
+ * This code is designed to get a VERY FAST json string, the goal was speed,
+ * not sexyness.
+ * Jeremie 'ahFeel' BORDIER
+ * ahFeel@rift.Fr
+ */
+__inline char *
+json_concat_string(char *s, char *field)
+{
+    *(s++) = '"';
+	while (*field) {
+		if (*field == '"') {
+            *(s++) = '\'';
+            *(s++) = *(field++);
+            *(s++) = '\'';
+        }
+        else {
+            *(s++) = *(field++);
+        }
+    }
+    *(s++) = '"';
+    return s;
+}
+
 static VALUE error_map;
 
 VALUE frt_get_error(const char *err_type)
@@ -355,7 +380,7 @@ void Init_ferret_ext(void)
     cStateError =
         rb_define_class_under(mFerret, "StateError", rb_eStandardError);
     cFileNotFoundError =
-        rb_define_class_under(rb_cObject, "FileNotFoundError", rb_eIOError);
+        rb_define_class_under(mFerret, "FileNotFoundError", rb_eIOError);
 
     error_map = rb_hash_new();
     rb_hash_aset(error_map, rb_intern("Exception"),         rb_eStandardError);

data/ext/ferret.h

@@ -65,6 +65,7 @@ extern VALUE frt_hs_to_rb_ary(HashSet *hs);
 extern void *frt_rb_data_ptr(VALUE val);
 extern char * frt_field(VALUE rfield);
 extern VALUE frt_get_term(const char *field, const char *term);
+extern __inline char *json_concat_string(char *s, char *field);
 extern char *rs2s(VALUE rstr);
 extern char *nstrdup(VALUE rstr);
 #define Frt_Make_Struct(klass)\

data/ext/index.c

@@ -6,7 +6,6 @@
 #include <string.h>
 #include <limits.h>
 #include <ctype.h>
-#include <unistd.h>
 
 #define GET_LOCK(lock, name, store, err_msg) do {\
     lock = store->open_lock(store, name);\
@@ -117,10 +116,10 @@ char *fn_for_generation(char *buf, char *base, char *ext, f_i64 gen)
     }
 }
 
-char *segfn_for_generation(char *buf, int generation)
+char *segfn_for_generation(char *buf, f_u64 generation)
 {
     char b[SEGMENT_NAME_MAX_LENGTH];
-    char *u = u64_to_str36(b, SEGMENT_NAME_MAX_LENGTH, (f_u64)generation);
+    char *u = u64_to_str36(b, SEGMENT_NAME_MAX_LENGTH, generation);
     sprintf(buf, SEGMENTS_FILE_NAME"_%s", u);
     return buf;
 }
@@ -393,25 +392,36 @@ FieldInfo *fis_by_number(FieldInfos *fis, int num)
 
 FieldInfos *fis_read(InStream *is)
 {
-    int store_val, index_val, term_vector_val;
-    int i;
-    union { f_u32 i; float f; } tmp;
-    FieldInfo *fi;
-    FieldInfos *fis;
-
-    store_val = is_read_vint(is);
-    index_val = is_read_vint(is);
-    term_vector_val = is_read_vint(is);
-    fis = fis_new(store_val, index_val, term_vector_val);
-    for (i = is_read_vint(is); i > 0; i--) {
-        fi = ALLOC(FieldInfo);
-        fi->name = is_read_string(is);
-        tmp.i = is_read_u32(is);
-        fi->boost = tmp.f;
-        fi->bits = is_read_vint(is);
-        fis_add_field(fis, fi);
-        fi->ref_cnt = 1;
-    }
+    FieldInfos *volatile fis;
+    TRY
+        do {
+            int store_val, index_val, term_vector_val;
+            int i;
+            union { f_u32 i; float f; } tmp;
+            FieldInfo *volatile fi;
+
+            store_val = is_read_vint(is);
+            index_val = is_read_vint(is);
+            term_vector_val = is_read_vint(is);
+            fis = fis_new(store_val, index_val, term_vector_val);
+            for (i = is_read_vint(is); i > 0; i--) {
+                fi = ALLOC_AND_ZERO(FieldInfo);
+                TRY
+                    fi->name = is_read_string_safe(is);
+                    tmp.i = is_read_u32(is);
+                    fi->boost = tmp.f;
+                    fi->bits = is_read_vint(is);
+                XCATCHALL
+                    free(fi->name);
+                    free(fi);
+                XENDTRY
+                fis_add_field(fis, fi);
+                fi->ref_cnt = 1;
+            }
+        } while (0);
+    XCATCHALL
+        fis_deref(fis);
+    XENDTRY
 
     return fis; 
 }
@@ -557,21 +567,26 @@ SegmentInfo *si_new(char *name, int doc_cnt, Store *store)
 
 SegmentInfo *si_read(Store *store, InStream *is)
 {
-    SegmentInfo *si = ALLOC_AND_ZERO(SegmentInfo);
-    si->store = store;
-    si->name = is_read_string(is);
-    si->doc_cnt = is_read_vint(is);
-    si->del_gen = is_read_vint(is);
-    si->norm_gens_size = is_read_vint(is);
-    si->ref_cnt = 1;
-    if (0 < si->norm_gens_size) {
-        int i;
-        si->norm_gens = ALLOC_N(int, si->norm_gens_size);
-        for (i = si->norm_gens_size - 1; i >= 0; i--) {
-            si->norm_gens[i] = is_read_vint(is);
+    SegmentInfo *volatile si = ALLOC_AND_ZERO(SegmentInfo);
+    TRY
+        si->store = store;
+        si->name = is_read_string_safe(is);
+        si->doc_cnt = is_read_vint(is);
+        si->del_gen = is_read_vint(is);
+        si->norm_gens_size = is_read_vint(is);
+        si->ref_cnt = 1;
+        if (0 < si->norm_gens_size) {
+            int i;
+            si->norm_gens = ALLOC_N(int, si->norm_gens_size);
+            for (i = si->norm_gens_size - 1; i >= 0; i--) {
+                si->norm_gens[i] = is_read_vint(is);
+            }
         }
-    }
-    si->use_compound_file = (bool)is_read_byte(is);
+        si->use_compound_file = (bool)is_read_byte(is);
+    XCATCHALL
+        free(si->name);
+        free(si);
+    XENDTRY
     return si;
 }
 
@@ -4008,17 +4023,23 @@ static void bv_write(BitVector *bv, Store *store, char *name)
 static BitVector *bv_read(Store *store, char *name)
 {
     int i;
-    BitVector *bv = ALLOC_AND_ZERO(BitVector);
-    InStream *is = store->open_input(store, name);
+    bool success = false;
+    InStream *volatile is = store->open_input(store, name);
+    BitVector *volatile bv = ALLOC_AND_ZERO(BitVector);
     bv->size = (int)is_read_vint(is);
     bv->capa = (bv->size >> 5) + 1;
     bv->bits = ALLOC_AND_ZERO_N(f_u32, bv->capa);
     bv->ref_cnt = 1;
-    for (i = (bv->size >> 5); i >= 0; i--) {
-        bv->bits[i] = is_read_u32(is);
-    }
-    is_close(is);
-    bv_recount(bv);
+    TRY
+        for (i = (bv->size >> 5); i >= 0; i--) {
+            bv->bits[i] = is_read_u32(is);
+        }
+        bv_recount(bv);
+        success = true;
+    XFINALLY
+        is_close(is);
+        if (!success && bv) bv_destroy(bv);
+    XENDTRY
     return bv;
 }
 
@@ -4297,6 +4318,7 @@ static IndexReader *sr_setup_i(SegmentReader *sr)
             sr->fr_bucket = ary_new();
         }
     XCATCHALL
+        ir->sis = NULL;
         ir_close(ir);
     XENDTRY
 
@@ -4306,10 +4328,9 @@ static IndexReader *sr_setup_i(SegmentReader *sr)
 static IndexReader *sr_open(SegmentInfos *sis, FieldInfos *fis, int si_num,
                             bool is_owner)
 {
-    IndexReader *ir;
     SegmentReader *sr = ALLOC_AND_ZERO(SegmentReader);
     sr->si = sis->segs[si_num];
-    ir = ir_setup(IR(sr), sr->si->store, sis, fis, is_owner);
+    ir_setup(IR(sr), sr->si->store, sis, fis, is_owner);
     return sr_setup_i(sr);
 }
 
@@ -5187,10 +5208,10 @@ HashTable *dw_invert_field(DocWriter *dw,
     if (fld_inv->is_tokenized) {
         Token *tk;
         int pos = -1, num_terms = 0;
-        TokenStream *ts = a_get_ts(a, df->name, "");
 
         for (i = 0; i < df_size; i++) {
-            ts->reset(ts, df->data[i]);
+            TokenStream *ts = a_get_ts(a, df->name, df->data[i]);
+            /* ts->reset(ts, df->data[i]); no longer being called */
             if (store_offsets) {
                 while (NULL != (tk = ts->next(ts))) {
                     pos += tk->pos_inc;
@@ -5212,8 +5233,8 @@ HashTable *dw_invert_field(DocWriter *dw,
                     }
                 }
             }
+            ts_deref(ts);
         }
-        ts_deref(ts);
         fld_inv->length = num_terms;
     }
     else {

data/ext/q_boolean.c

@@ -1555,18 +1555,25 @@ Query *bq_new(bool coord_disabled)
     return self;
 }
 
+Query *bq_new_max(bool coord_disabled, int max)
+{
+    Query *q = bq_new(coord_disabled);
+    BQ(q)->max_clause_cnt = max;
+    return q;
+}
+
 BooleanClause *bq_add_clause_nr(Query *self, BooleanClause *bc)
 {
-    if (BQ(self)->clause_cnt >= BQ(self)->clause_capa) {
-        BQ(self)->clause_capa *= 2;
-        REALLOC_N(BQ(self)->clauses, BooleanClause *, BQ(self)->clause_capa);
-    }
-    if (BQ(self)->clause_cnt > BQ(self)->max_clause_cnt) {
+    if (BQ(self)->clause_cnt >= BQ(self)->max_clause_cnt) {
         RAISE(STATE_ERROR, "Two many clauses. The max clause limit is set to "
               "<%d> but your query has <%d> clauses. You can try increasing "
               ":max_clause_count for the BooleanQuery or using a different "
               "type of query.", BQ(self)->clause_cnt, BQ(self)->max_clause_cnt);
     }
+    if (BQ(self)->clause_cnt >= BQ(self)->clause_capa) {
+        BQ(self)->clause_capa *= 2;
+        REALLOC_N(BQ(self)->clauses, BooleanClause *, BQ(self)->clause_capa);
+    }
     BQ(self)->clauses[BQ(self)->clause_cnt] = bc;
     BQ(self)->clause_cnt++;
     return bc;
@@ -1580,9 +1587,16 @@ BooleanClause *bq_add_clause(Query *self, BooleanClause *bc)
 
 BooleanClause *bq_add_query_nr(Query *self, Query *sub_query, enum BC_TYPE occur)
 {
-    BooleanClause *bc = bc_new(sub_query, occur);
+    BooleanClause *bc;
+    if (BQ(self)->clause_cnt >= BQ(self)->max_clause_cnt) {
+        RAISE(STATE_ERROR, "Two many clauses. The max clause limit is set to "
+              "<%d> but your query has <%d> clauses. You can try increasing "
+              ":max_clause_count for the BooleanQuery or using a different "
+              "type of query.", BQ(self)->clause_cnt, BQ(self)->max_clause_cnt);
+    }
+    bc = bc_new(sub_query, occur);
     bq_add_clause(self, bc);
-    bc_deref(bc); /* bc would have been referenced unnecessarily */
+    bc_deref(bc); /* bc was referenced unnecessarily */
     return bc;
 }