ferret 0.1.0

data/lib/ferret/utils.rb

@@ -0,0 +1,7 @@
+require 'ferret/utils/string_helper'
+require 'ferret/utils/parameter'
+require 'ferret/utils/priority_queue'
+require 'ferret/utils/bit_vector'
+require 'ferret/utils/date_tools'
+require 'ferret/utils/number_tools'
+require 'ferret/utils/weak_key_hash'

data/lib/ferret/utils/bit_vector.rb

@@ -0,0 +1,105 @@
+module Ferret::Utils
+  # Optimized implementation of a vector of bits.
+  #
+  #  * a count() method, which efficiently computes the number of one bits
+  #  * optimized read from and write to disk
+  #  * inlinable get() method
+  class BitVector 
+    attr_reader :size
+    attr_accessor :bits
+
+    def initialize
+      @bits = 0
+      @count = -1
+    end
+
+    # Sets the value of _bit_ to one. 
+    def set(bit) 
+      @bits |= 1 << bit
+      @count = -1
+    end
+
+    # Sets the value of _bit_ to zero. 
+    def clear(bit) 
+      @bits &= ~(1 << bit)
+      @count = -1
+    end
+
+    # Returns _true_ if _bit_ is one and
+    # _false_ if it is zero. 
+    def get(bit) 
+      return (@bits & (1 << bit)) != 0
+    end
+    alias :[] :get
+
+    # Returns the total number of one bits in this vector.  This is
+    # efficiently computed and cached, so that, if the vector is not
+    # changed, no recomputation is done for repeated calls. 
+    def count() 
+      # if the vector has been modified
+      if (@count == -1) 
+        c = 0
+        tmp = @bits
+        while tmp > 0
+          c += BYTE_COUNTS[tmp & 0xFF] # sum bits per byte
+          tmp >>= 8
+        end
+        @count = c
+      end
+      return @count
+    end
+
+    BYTE_COUNTS = [ # table of bits/byte
+      0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
+      1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+      1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+      2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+      1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+      2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+      2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+      3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+      1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+      2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+      2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+      3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+      2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+      3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+      3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+      4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
+    ] 
+
+
+    # Writes this vector to the file _name_ in Directory _d_, in a format
+    # that can be read by the constructor
+    def write(d, name)
+      output = d.create_output(name)
+      begin 
+        output.write_vint(@bits)
+      ensure 
+        output.close()
+      end
+    end
+
+    # Constructs a bit vector from the file _name_ in Directory _d_, as
+    # written by the @link #writeendmethod.
+    def BitVector.read(d, name)
+      bv = BitVector.new
+      input = d.open_input(name)
+      begin 
+        bv.bits = input.read_vint()
+      ensure 
+        input.close()
+      end
+      return bv
+    end
+
+    def to_s
+      i = @bits
+      while i > 0
+        print(i&1)
+        i >>= 1
+      end
+      puts ""
+    end
+  end
+end

data/lib/ferret/utils/date_tools.rb

@@ -0,0 +1,138 @@
+require 'date'
+module Ferret::Utils
+  # Provides support for converting dates to strings and vice-versa.  The
+  # strings are structured so that lexicographic sorting orders them by
+  # date, which makes them suitable for use as field values and search
+  # terms.
+  # 
+  # This class also helps you to limit the resolution of your dates. Do not
+  # save dates with a finer resolution than you really need, as then
+  # RangeQuery and PrefixQuery will require more memory and become slower.
+  # 
+  # Compared to the serialize methods the strings generated by the to_s
+  # methods in this class take slightly more space, unless your selected
+  # resolution is set to _Resolution.DAY_ or lower.
+
+  # Provides support for converting dates to strings and vice-versa.  The
+  # strings are structured so that lexicographic sorting orders by date,
+  # which makes them suitable for use as field values and search terms.
+  # 
+  # Note:: dates before 1970 cannot be used, and therefore cannot be indexed
+  # when using this class.
+  module DateTools
+    # make date strings long enough to last a millenium
+    SERIALIZED_DATE_LEN = (1000*365*24*60*60*1000).to_s(36).length
+
+    # The latest date that can be stored in this format
+    MAX_SERIALIZED_DATE_STRING = Array.new(SERIALIZED_DATE_LEN, "z").to_s.to_i(36)
+
+    # Converts a Date to a string suitable for indexing.  Throws Exception
+    # if the date specified in the method argument is before 1970 This
+    # method is unsupported. Please use Time instead of Date
+    def DateTools.serialize_date(date)
+      return serialize_time(Time.parse(date))
+    end
+
+    # Converts a millisecond time to a string suitable for indexing.
+    # Accepts a Time object or a time in milliseconds.
+    #
+    # Throws Exception if the time specified in the method argument is
+    # negative, that is, before 1970 It is recommended that you store the
+    # date as a string if you don't need the time to the nearest
+    # millisecond. That makes things a lot easier.
+    def DateTools.serialize_time(time)
+      if time.instance_of?(Time) then time = time.to_i end
+
+      if (time < 0) then raise("time too early") end
+
+      # convert to milliseconds before serialization
+      s = (time*1000).to_s(36)
+
+      if (s.length() > SERIALIZED_DATE_LEN) then raise("time too late") end
+
+      # pad to 16 charactors
+      s = "0" + s while (s.length() < SERIALIZED_DATE_LEN)
+
+      return s
+    end
+
+    # The earliest date that can be stored in this format.
+    MIN_SERIALIZED_DATE_STRING = DateTools.serialize_time(0)
+
+    # Converts a string-encoded date into a millisecond time.
+    def DateTools.deserialize_time(s)
+      # remember to convert back to seconds
+      return Time.at(s.to_i(36)/1000)
+    end
+
+    def DateTools.date_to_s(date, resolution = Resolution::MILLISECOND)
+      return time_to_s(Time.parse(date), resolution)
+    end
+    
+    
+    # Converts a millisecond time to a string suitable for indexing.
+    # 
+    # time:: the date expressed as milliseconds since January 1, 1970,
+    #     00:00:00 GMT resolution:: the desired resolution, see
+    #     #round(long, DateTools.Resolution)
+    # return:: a string in format _%Y%m%d%H%M%SSSS_ or shorter,
+    #          depending on _resolution_
+    def DateTools.time_to_s(time, resolution = Resolution::MILLISECOND)
+      if time.instance_of?(Date) then time = Time.parse(time) end
+      suffix = ""
+      if (resolution == Resolution::MILLISECOND)
+        # the suffix is the number of milliseconds if needed.
+        suffix = ((time.to_f-time.to_f.floor)*1000).round.to_s
+      end
+      return time.strftime(resolution.format) + suffix
+    end
+
+    # Converts a string produced by _time_to_s_ or _date_to_s_ back to a
+    # time, represented as the number of milliseconds since January 1, 1970,
+    # 00:00:00 GMT.
+    # 
+    # str:: the date string to be converted
+    # return:: the number of milliseconds since January 1, 1970, 00:00:00GMT
+    def DateTools.s_to_time(str)
+      year =        str.size >=  4 ? str[ 0.. 3].to_i : nil
+      month =       str.size >=  6 ? str[ 4.. 5].to_i : nil
+      day =         str.size >=  8 ? str[ 6.. 7].to_i : nil
+      hour =        str.size >= 10 ? str[ 8.. 9].to_i : nil
+      minute =      str.size >= 12 ? str[10..11].to_i : nil
+      second =      str.size >= 14 ? str[12..13].to_i : nil
+      microsecond = str.size >= 17 ? str[14..17].to_i*1000 : nil
+      return Time.mktime(year, month, day, hour, minute, second, microsecond)
+    end
+
+    # Limit a date's resolution. For example, the date _2004-09-21 13:50:11_
+    # will be changed to _2004-09-01 00:00:00_ when using
+    # _Resolution.MONTH_. 
+    # 
+    # resolution:: The desired resolution of the date to be returned
+    # return:: the date with all values more precise than _resolution_
+    #  set to 0 or 1
+    def DateTools.round(time, resolution)
+      return s_to_time(time_to_s(time, resolution))
+    end
+    
+    class Resolution < Parameter
+      attr_accessor :format
+
+      private :initialize
+
+      def initialize(name, format)
+          super(name)
+          @format = format
+      end
+
+      YEAR        = Resolution.new("year",   "%Y")
+      MONTH       = Resolution.new("month",  "%Y%m")
+      DAY         = Resolution.new("day",    "%Y%m%d")
+      HOUR        = Resolution.new("hour",   "%Y%m%d%H")
+      MINUTE      = Resolution.new("minute", "%Y%m%d%H%M")
+      SECOND      = Resolution.new("second", "%Y%m%d%H%M%S")
+      MILLISECOND = Resolution.new("millisecond", "%Y%m%d%H%M%S")
+
+    end
+  end
+end

data/lib/ferret/utils/number_tools.rb

@@ -0,0 +1,91 @@
+class Float
+  def =~(o)
+    return (1 - self/o).abs < 0.0000000001
+  end
+end
+
+module Ferret::Utils
+  # Provides support for converting longs to Strings, and back again. The
+  # strings are structured so that lexicographic sorting order is preserved.
+  # 
+  # That is, if long1 is less than long2 for any two longs long1 and long2,
+  # then NumberTools.long_to_s(long1) is lexicographically less than
+  # NumberTools.long_to_s(long2). (Similarly for "greater than" and "equals".)
+  # 
+  # This class handles all long values
+  module NumberTools
+    RADIX = 36
+    NEGATIVE_PREFIX = '-'
+
+    # NB: NEGATIVE_PREFIX must be < POSITIVE_PREFIX
+    POSITIVE_PREFIX = '0'
+
+    # The following constants are from Java
+    LONG_MAX_VALUE =  9223372036854775807
+    LONG_MIN_VALUE = -9223372036854775808 
+
+    # NB: This function is used to match the java equivalent. Actually
+    # ruby allows much larger numbers than Java so this is just so that we
+    # can read the Java Lucene created indexes.
+    MIN_STRING_VALUE = NEGATIVE_PREFIX + "0000000000000"
+    MAX_STRING_VALUE = POSITIVE_PREFIX + "1y2p0ij32e8e7"
+
+    # The length of the long field
+    STR_SIZE = MIN_STRING_VALUE.length()
+
+    # Converts a long to a String suitable for indexing.
+    def NumberTools.long_to_s(l)
+      if (l == LONG_MIN_VALUE)
+          # special case, because long is not symetric around zero
+          return MIN_STRING_VALUE;
+      end
+
+      s = ""
+      if (l < 0)
+        s << NEGATIVE_PREFIX
+        l = LONG_MAX_VALUE + l + 1
+      else
+        s << POSITIVE_PREFIX
+      end
+      num = l.to_s(RADIX)
+
+      pad_len = STR_SIZE - num.length() - s.length()
+      while ((pad_len -= 1) >= 0)
+          s << '0'
+      end
+      s << num
+
+      return s
+    end
+
+    # Converts a String that was returned by #long_to_s back to a long.
+    # 
+    # Throws:: ArgumentError if the input is nil
+    def NumberTools.s_to_long(s)
+      if (s == nil)
+        raise ArgumentError, "string cannot be nil"
+      end
+      if (s.length() != STR_SIZE)
+        raise ArgumentError, "string is the wrong size"
+      end
+
+      if (s == MIN_STRING_VALUE)
+        return LONG_MIN_VALUE
+      end
+
+      prefix = s[0,1]
+      l = s[1..-1].to_i(36)
+
+      if (prefix == POSITIVE_PREFIX)
+          # nop
+      elsif (prefix == NEGATIVE_PREFIX)
+          l = l - LONG_MAX_VALUE - 1
+      else
+        raise ArgumentError, "string <" + prefix +
+          "> does not begin with the correct prefix"
+      end
+
+      return l
+    end
+  end
+end

data/lib/ferret/utils/parameter.rb

@@ -0,0 +1,41 @@
+module Ferret::Utils
+  class Parameter
+    def to_s() return @name end
+      
+    def _dump(arg)
+      @name
+    end
+
+    def Parameter._load(var)
+      name = var
+      key = make_key(name)
+      if (@@all_parameters.has_key?(key))
+        return @@all_parameters[key]
+      else
+        return self.new(name)
+      end
+    end
+
+    def hash
+      return self.class.make_key(@name).hash
+    end
+
+    protected
+      @@all_parameters = {}
+
+      def initialize(name)
+        @name = name
+        key = self.class.make_key(name)
+        
+        if (@@all_parameters.has_key?(key))
+          raise ArgumentError, "key already in use"
+        end
+
+        @@all_parameters[key] = self
+      end
+    
+      def Parameter.make_key(name)
+        return self.to_s + " " + name
+      end
+  end
+end

data/lib/ferret/utils/priority_queue.rb

@@ -0,0 +1,120 @@
+module Ferret::Utils
+  # A PriorityQueue maintains a partial ordering of its objects such that
+  # the least object can always be found in constant time. push()'s and
+  # pop()'s require log(size) time. The objects in this priority queue must
+  # be Comparable
+  class PriorityQueue 
+    attr_reader :size
+
+    def less_than(a, b)
+      a < b
+    end
+
+    # Subclass constructors must call this. 
+    def initialize(max_size)
+      @size = 0
+      @heap = Array.new(max_size + 1)
+      @max_size = max_size
+    end
+
+    # Adds an Object to a PriorityQueue in log(size) time.
+    #
+    # If one tries to add more objects than max_size from initialize a
+    # RuntimeException (ArrayIndexOutOfBound) is thrown.
+    def push(object) 
+      @size += 1
+      @heap[@size] = object
+      up_heap()
+    end
+    alias :<< :push
+
+    # Adds object to the PriorityQueue in log(size) time if either the
+    # PriorityQueue is not full, or not less_than(object, top()).
+    #
+    # object:: the object to be inserted
+    # return true if object is added, false otherwise.
+    def insert(object)
+      if(@size < @max_size)
+        push(object)
+        return true
+      elsif (@size > 0 and less_than(top, object))
+        @heap[1] = object
+        down_heap()
+        return true
+      else
+        return false
+      end
+     end
+
+    # Returns the least object of the PriorityQueue in constant time. 
+    def top
+      return @heap[1]
+    end
+
+    # Removes and returns the least object of the PriorityQueue in log(size)
+    # time. 
+    def pop() 
+      if (@size > 0) 
+        result = @heap[1]        # save first value
+        @heap[1] = @heap[@size]  # move last to first
+        @heap[@size] = nil;      # permit GC of objects
+        @size -= 1
+        down_heap()              # adjust heap
+        return result
+      else
+        return nil
+      end
+    end
+
+    # Removes all entries from the PriorityQueue. 
+    def clear() 
+      (1..@size).each do |i|
+        @heap[i] = nil
+      end
+      @size = 0
+    end
+
+    def put_heap
+      puts @heap
+    end
+
+    # resets the queue after the top has been changed
+    def adjust_top()
+      down_heap()
+    end
+
+    private
+
+      def up_heap() 
+        i = @size
+        node = @heap[i]             # save bottom node
+        j = i >> 1
+        while (j > 0 and less_than(node, @heap[j])) 
+          @heap[i] = @heap[j];      # shift parents down
+          i = j
+          j = j >> 1
+        end
+        @heap[i] = node;            # install saved node
+      end
+
+      def down_heap() 
+        i = 1
+        node = @heap[i]             # save top node
+        j = i << 1                  # find smaller child
+        k = j + 1
+        if k <= @size and less_than(@heap[k], @heap[j])
+          j = k
+        end
+        while (j <= @size and less_than(@heap[j], node)) 
+          @heap[i] = @heap[j]       # shift up child
+          i = j
+          j = i << 1
+          k = j + 1
+          if k <= @size and less_than(@heap[k], @heap[j]) 
+            j = k
+          end
+        end
+        @heap[i] = node;            # install saved node
+      end
+  end
+end