ferret 0.1.3 → 0.1.4

data/lib/ferret/index/segment_infos.rb

@@ -61,9 +61,9 @@ module Ferret
         @counter = 0
       end
 
-      def clone
-        clone = self.clone
-        self.each_index {|i| clone[i] = self[i].clone}
+      def initialize_copy(o)
+        super
+        o.each_index {|i| self[i] = o[i].clone}
       end
 
       def read(directory)

data/lib/ferret/index/segment_merger.rb

@@ -31,6 +31,7 @@ module Ferret::Index
     def add(reader) 
       @readers << reader
     end
+    alias :<< :add
 
     # 
     # i:: The index of the reader to return
@@ -61,24 +62,24 @@ module Ferret::Index
            
       cfs_writer = CompoundFileWriter.new(@directory, file_name)
 
-      files = Array.new(IndexFileNames::COMPOUND_EXTENSIONS.length + @field_infos.size)
+      files = []
       
       # Basic files
-      IndexFileNames::COMPOUND_EXTENSIONS.times do |i|
-        files << @segment + "." + IndexFileNames::COMPOUND_EXTENSIONS[i]
+      IndexFileNames::COMPOUND_EXTENSIONS.each do |ext|
+        files << "#{@segment}.#{ext}"
       end
 
       # Field norm files
       @field_infos.each_with_index do |fi, i|
         if (fi.indexed?) 
-          files << @segment + ".f#{i}"
+          files << "#{@segment}.f#{i}"
         end
       end
 
       # Vector files
       if @field_infos.has_vectors?
-        IndexFileNames::VECTOR_EXTENSIONS.length.times do |i|
-          files << @segment + "." + IndexFileNames::VECTOR_EXTENSIONS[i]
+        IndexFileNames::VECTOR_EXTENSIONS.each do |ext|
+          files << "#{@segment}.#{ext}"
         end
       end
 

data/lib/ferret/index/segment_reader.rb

@@ -37,8 +37,10 @@ module Ferret::Index
       @prox_stream = cfs.open_input(@segment + '.prx')
       @norms = {}
       @norms.extend(MonitorMixin)
+      @norms_dirty = false
       open_norms(cfs)
 
+      @tv_reader_orig = nil
       if @field_infos.has_vectors? then
         @tv_reader_orig = TermVectorsReader.new(cfs, @segment, @field_infos)
       end
@@ -65,7 +67,9 @@ module Ferret::Index
     end
     
     def do_close()
-      Thread.current["#{self.object_id}-tv_reader"] = nil # clear the cache
+      # clear the cache
+      Thread.current["#{self.object_id}-#{@segment}-tv_reader"] = nil
+
       @fields_reader.close()
       @term_infos.close()
 
@@ -88,11 +92,12 @@ module Ferret::Index
 
 
     def SegmentReader.uses_compound_file?(si)
-      return si.dir.exists?(si.name + ".cfs")
+      return si.directory.exists?(si.name + ".cfs")
     end
     
     def SegmentReader.has_separate_norms?(si)
-      return (si.dir.list.select {|f| f =~ /^#{si.name}\.s/}).size > 0
+      si.directory.each {|f| return true if f =~ /^#{si.name}\.s/}
+      return false
     end
 
     def do_delete(doc_num) 
@@ -294,11 +299,22 @@ module Ferret::Index
     # in the Thread
     # returns:: TermVectorsReader
     def get_term_vectors_reader() 
-      #return @xtv_reader ||= @tv_reader_orig.clone()
-      tv_reader = Thread.current["#{self.object_id}-tv_reader"]
-      if (tv_reader == nil) 
+      #tvr_cache = Thread.current["tv_reader"]
+      #if (tvr_cache == nil) 
+      #  tvr_cache = Thread.current["tv_reader"] = Ferret::Utils::WeakKeyHash.new
+      #end
+      #tvr_cache.synchronize do
+      #  tv_reader = tvr_cache[self]
+      #  if tv_reader == nil
+      #    tv_reader = @tv_reader_orig.clone()
+      #    tvr_cache[self] = tv_reader
+      #  end
+      #  return tv_reader
+      #end
+      tv_reader = Thread.current.get_local(self)
+      if tv_reader.nil?
         tv_reader = @tv_reader_orig.clone()
-        Thread.current["#{self.object_id}-tv_reader"] = tv_reader
+        Thread.current.set_local(self, tv_reader)
       end
       return tv_reader
     end

data/lib/ferret/index/segment_term_enum.rb

@@ -64,13 +64,12 @@ module Ferret::Index
     protected :input, :input=, :term_buffer,
       :term_buffer=, :prev_buffer, :prev_buffer=
 
-    def clone() 
-      clone = super
-      clone.input = @input.clone
-      clone.term_info = @term_info.clone
-      clone.term_buffer = @term_buffer.clone
-      clone.prev_buffer = @prev_buffer.clone
-      return clone
+    def initialize_copy(o)
+      super
+      @input = o.input.clone
+      @term_info = o.term_info.clone
+      @term_buffer = o.term_buffer.clone
+      @prev_buffer = o.prev_buffer.clone
     end
 
     def seek(pointer, position, term, term_info)

data/lib/ferret/index/term_buffer.rb

@@ -71,12 +71,10 @@ module Ferret::Index
     end
     alias :term :to_term
 
-    def clone() 
-      clone = TermBuffer.new()
-      clone.set!(self)
-      return clone
+    def initialize_copy(o)
+      set!(o)
     end
-    
+
     def text_str()
       @text[0,@text_length]
     end

data/lib/ferret/index/term_doc_enum.rb

@@ -110,8 +110,10 @@ module Ferret::Index
 
     def close()
       @freq_stream.close()
+      @freq_stream = nil
       if (@skip_stream != nil)
         @skip_stream.close()
+        @skip_stream = nil
       end
       @parent = nil
     end
@@ -143,11 +145,11 @@ module Ferret::Index
     # Optimized implementation. 
     def read(docs, freqs, start = 0)
       i = start
-      needed=docs.length
+      needed = docs.length
 
       while (i < needed and @count < @doc_freq) 
 
-        # manually inlined call to next() for speed
+        # manually inlined call to next?() for speed
         doc_code = @freq_stream.read_vint()
         @doc += doc_code >> 1              # shift off low bit
         if ((doc_code & 1) != 0)           # if low bit is set
@@ -155,6 +157,7 @@ module Ferret::Index
         else
           @freq = @freq_stream.read_vint() # else read freq
         end
+
         @count += 1
 
         if (@deleted_docs == nil or not @deleted_docs[@doc]) 
@@ -162,6 +165,8 @@ module Ferret::Index
           freqs[i] = @freq
           i += 1
         end
+
+        skipping_doc()
       end
       return i
     end

data/lib/ferret/index/term_infos_io.rb

@@ -115,8 +115,6 @@ module Ferret::Index
     def initialize(dir, seg, fis)
       super()
 
-      Thread.current["#{self.object_id}-term_enum"] = nil
-
       @directory = dir
       @segment = seg
       @field_infos = fis
@@ -134,8 +132,6 @@ module Ferret::Index
 
     def close()
       # clear this threads cache 
-      Thread.current["#{self.object_id}-term_enum"] = nil
-
       @orig_enum.close() if (@orig_enum != nil)
       @index_enum.close() if (@index_enum != nil)
     end
@@ -211,10 +207,21 @@ module Ferret::Index
     private
 
       def enum() 
-        term_enum = Thread.current["#{self.object_id}-term_enum"]
-        if (term_enum == nil) 
-          term_enum = terms()
-          @xterm_enum = Thread.current["#{self.object_id}-term_enum"] = term_enum 
+        #te_cache = Thread.current["term_enum"]
+        #if (te_cache == nil) 
+        #  te_cache = Thread.current["term_enum"] = Ferret::Utils::WeakKeyHash.new
+        #end
+        #te_cache.synchronize do
+        #  term_enum = te_cache[self]
+        #  if term_enum == nil
+        #    term_enum = terms()
+        #    te_cache[self] = term_enum
+        #  end
+        #  return term_enum
+        #end
+        term_enum = Thread.current.get_local(self)
+        if term_enum.nil?
+          Thread.current.set_local(self, term_enum = terms())
         end
         return term_enum
       end

data/lib/ferret/query_parser/query_parser.tab.rb

@@ -1,7 +1,7 @@
 #
 # DO NOT MODIFY!!!!
 # This file is automatically generated by racc 1.4.4
-# from racc grammer file "query_parser.y".
+# from racc grammer file "lib/ferret/query_parser/query_parser.y".
 #
 
 require 'racc/parser'
@@ -11,12 +11,12 @@ module Ferret
 
   class QueryParser < Racc::Parser
 
-module_eval <<'..end query_parser.y modeval..id2aa118b008', 'query_parser.y', 126
+module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id9e08d44076', 'lib/ferret/query_parser/query_parser.y', 126
   attr_accessor :default_field, :fields
 
   # true if you want to downcase wild card queries. This is set to try by
   # default.
-  attr_accessor :wild_lower
+  attr_writer :wild_lower
 
   def wild_lower?() @wild_lower end
 
@@ -369,8 +369,12 @@ module_eval <<'..end query_parser.y modeval..id2aa118b008', 'query_parser.y', 12
     end
   end
 
+  def QueryParser.parse(query, default_field = "*", options = {})
+    qp = QueryParser.new(default_field, options)
+    return qp.parse(query)
+  end
 
-..end query_parser.y modeval..id2aa118b008
+..end lib/ferret/query_parser/query_parser.y modeval..id9e08d44076
 
 ##### racc 1.4.4 generates ###
 
@@ -604,49 +608,49 @@ Racc_debug_parser = false
 
  # reduce 0 omitted
 
-module_eval <<'.,.,', 'query_parser.y', 19
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 19
   def _reduce_1( val, _values)
                     get_boolean_query(val[0])
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 24
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 24
   def _reduce_2( val, _values)
                     [val[0]]
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 28
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 28
   def _reduce_3( val, _values)
                     add_and_clause(val[0], val[2])
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 32
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 32
   def _reduce_4( val, _values)
                     add_or_clause(val[0], val[2])
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 36
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 36
   def _reduce_5( val, _values)
                     add_default_clause(val[0], val[1])
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 41
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 41
   def _reduce_6( val, _values)
                     get_boolean_clause(val[1], BooleanClause::Occur::MUST)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 45
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 45
   def _reduce_7( val, _values)
                     get_boolean_clause(val[1], BooleanClause::Occur::MUST_NOT)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 49
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 49
   def _reduce_8( val, _values)
                     get_boolean_clause(val[0], BooleanClause::Occur::SHOULD)
   end
@@ -654,7 +658,7 @@ module_eval <<'.,.,', 'query_parser.y', 49
 
  # reduce 9 omitted
 
-module_eval <<'.,.,', 'query_parser.y', 51
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 51
   def _reduce_10( val, _values)
  val[0].boost = val[2].to_f; return val[0]
   end
@@ -662,7 +666,7 @@ module_eval <<'.,.,', 'query_parser.y', 51
 
  # reduce 11 omitted
 
-module_eval <<'.,.,', 'query_parser.y', 58
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 58
   def _reduce_12( val, _values)
                     get_boolean_query(val[1])
   end
@@ -676,187 +680,187 @@ module_eval <<'.,.,', 'query_parser.y', 58
 
  # reduce 16 omitted
 
-module_eval <<'.,.,', 'query_parser.y', 67
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 67
   def _reduce_17( val, _values)
                     _get_term_query(val[0])
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 71
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 71
   def _reduce_18( val, _values)
                     _get_fuzzy_query(val[0], val[2])
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 75
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 75
   def _reduce_19( val, _values)
                     _get_fuzzy_query(val[0])
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 80
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 80
   def _reduce_20( val, _values)
                     _get_wild_query(val[0])
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 81
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 81
   def _reduce_21( val, _values)
 @field = @default_field
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 85
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 85
   def _reduce_22( val, _values)
                     val[2]
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 85
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 85
   def _reduce_23( val, _values)
 @field = "*"
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 85
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 85
   def _reduce_24( val, _values)
 @field = @default_field
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 89
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 89
   def _reduce_25( val, _values)
                     val[3]
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 90
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 90
   def _reduce_26( val, _values)
  @field = [val[0]]
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 91
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 91
   def _reduce_27( val, _values)
  @field = val[0] += [val[2]]
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 97
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 97
   def _reduce_28( val, _values)
                     get_phrase_query(val[1])
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 101
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 101
   def _reduce_29( val, _values)
                     get_phrase_query(val[1], val[4].to_i)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 101
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 101
   def _reduce_30( val, _values)
  nil
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 102
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 102
   def _reduce_31( val, _values)
  nil
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 104
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 104
   def _reduce_32( val, _values)
  [val[0]]
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 105
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 105
   def _reduce_33( val, _values)
  val[0] << val[1]
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 106
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 106
   def _reduce_34( val, _values)
  val[0] << nil
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 107
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 107
   def _reduce_35( val, _values)
  add_multi_word(val[0], val[2])
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 109
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 109
   def _reduce_36( val, _values)
  _get_range_query(val[1], val[2], true, true)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 110
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 110
   def _reduce_37( val, _values)
  _get_range_query(val[1], val[2], true, false)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 111
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 111
   def _reduce_38( val, _values)
  _get_range_query(val[1], val[2], false, true)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 112
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 112
   def _reduce_39( val, _values)
  _get_range_query(val[1], val[2], false, false)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 113
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 113
   def _reduce_40( val, _values)
  _get_range_query(nil,    val[1], false, false)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 114
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 114
   def _reduce_41( val, _values)
  _get_range_query(nil,    val[1], false, true)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 115
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 115
   def _reduce_42( val, _values)
  _get_range_query(val[1], nil,    true, false)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 116
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 116
   def _reduce_43( val, _values)
  _get_range_query(val[1], nil,    false, false)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 117
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 117
   def _reduce_44( val, _values)
  _get_range_query(nil,    val[1], false, false)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 118
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 118
   def _reduce_45( val, _values)
  _get_range_query(nil,    val[2], false, true)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 119
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 119
   def _reduce_46( val, _values)
  _get_range_query(val[2], nil,    true, false)
   end
 .,.,
 
-module_eval <<'.,.,', 'query_parser.y', 120
+module_eval <<'.,.,', 'lib/ferret/query_parser/query_parser.y', 120
   def _reduce_47( val, _values)
  _get_range_query(val[1], nil,    false, false)
   end