ferret 0.1.0

data/lib/ferret/store/index_io.rb

@@ -0,0 +1,259 @@
+module Ferret::Store
+  # Ferret's IO Input methods are defined here. The methods read_byte and
+  # read_bytes need to be defined before this class is of any use. 
+  class IndexInput
+
+    # Reads and returns a single byte.
+    def read_byte()
+      raise NotImplementedError
+    end
+
+    # Reads a specified number of bytes into an array at the specified offset.
+    # buf:: the array to read bytes into
+    # offset:: the offset in the array to start storing bytes
+    # len:: the number of bytes to read
+    def read_bytes(buf, offset, len)
+      raise NotImplementedError
+    end
+     
+
+    # Reads four bytes and returns an int. read_uint should be used for
+    # unsigned integers for performance reasons.
+    def read_int
+      # This may be slow. I'm not sure if this is the best way to get
+      # integers from files but this is the only way I could find to get
+      # signed integers.
+      #i = read_byte
+      #return (((i&0x80)==0 ? 0 : -1) << 32) |
+             #(i << 24) |
+             #((read_byte) << 16) |
+             #((read_byte) << 8) |
+             #(read_byte)
+      i1 = read_byte
+      i2 = read_byte
+      i3 = read_byte
+      i4 = read_byte
+      res =  (((i1&0x80) == 0 ? 0 : -0x100000000)) +
+             ((i1 << 24) + (i2 << 16) + (i3 << 8) + (i4))
+      return res
+    end
+
+    # Reads eight bytes and returns a long.
+    def read_long
+      return (read_int << 32) + (read_int & 0xFFFFFFFF)
+    end
+
+    # Reads four bytes and returns a positive integer
+    def read_uint
+      return ((read_byte) << 24) | ((read_byte) << 16) |
+             ((read_byte) <<  8) |  (read_byte)
+    end
+    
+    # Reads eight bytes and returns a positive integer.
+    def read_ulong
+      return (read_uint << 32) | (read_uint & 0xFFFFFFFF)
+    end
+
+    # Reads an int stored in variable-length format.  Reads between one and
+    # five bytes.  Smaller values take fewer bytes.  Negative numbers are not
+    # supported.
+    def read_vint
+      b = read_byte
+      i = b & 0x7F # 0x7F = 0b01111111
+      shift = 7
+      
+      while b & 0x80 != 0 # 0x80 = 0b10000000
+        b = read_byte
+        i |= (b & 0x7F) << shift
+        shift += 7
+      end
+      
+      return i
+    end
+    alias :read_vlong :read_vint
+
+    # Reads a string. A string is stored as a single vint which describes
+    # the length of the string, followed by the actually string itself.
+    def read_string
+      length = read_vint
+      
+      chars = Array.new(length, ' ')
+      read_chars(chars, 0, length)
+      
+      chars.to_s
+    end
+
+    # Reads UTF-8 encoded characters into an array.
+    # buf:: the array to read characters into
+    # start:: the offset in the array to start storing characters
+    # length:: the number of characters to read
+    #
+    # TODO: Test on some actual UTF-8 documents.
+    def read_chars(buf, start, length)
+      if buf.length < (start + length)
+        # make room for the characters to read
+        buf << " " * (start + length - buf.length)
+      end
+      last = start + length
+      (start...last).each do |i|
+        buf[i] = read_byte.chr
+      end
+#        last = start + length
+#        
+#        (start...last).each do |i|
+#          b = read_byte
+#          if (b & 0x80) == 0
+#            buf[i] = (b & 0x7F).chr # don't need to worry about UTF-8 here
+#          else
+#            if (b & 0xE0) != 0xE0
+#              tmp_int = (((b & 0x1F) << 6) | (read_byte & 0x3F))
+#              buf[i] = [tmp_int].pack("C") # pack into a UTF-8 string
+#            else
+#              buf[i] = [
+#                         ((b & 0x0F) << 12) |
+#                         ((read_byte & 0x3F) << 6) |
+#                         (read_byte & 0x3F)
+#                       ].pack("U") # pack into a UTF-8 string
+#            end
+#          end
+#        end
+    end
+    
+    # Closes the stream to futher operations. 
+    def close
+      raise NotImplementedError
+    end
+
+    # Returns the current position in this file, where the next read will
+    # occur.
+    def pos
+      raise NotImplementedError
+    end
+
+    # Sets current position in this file, where the next read will occur.
+    def seek(pos)
+      raise NotImplementedError
+    end
+
+    # The number of bytes in the file. 
+    def length
+      raise NotImplementedError
+    end
+
+    # Returns a clone of this stream.
+    # 
+    # Clones of a stream access the same data, and are positioned at the same
+    # point as the stream they were cloned from.
+    # 
+    # Expert:: Subclasses must ensure that clones may be positioned at
+    # different points in the input from each other and from the stream they
+    # were cloned from.
+    #   def clone
+    #     raise NotImplementedError
+    #   end
+
+  end
+
+  # Ferret's IO Output methods are defined here. The methods write_byte and
+  # write_bytes need to be defined before this class is of any use. 
+  class IndexOutput 
+
+    # Writes a single byte.
+    def write_byte(b)
+      raise NotImplementedError
+    end
+
+    # Writes an array of bytes.
+    # buf:: the bytes to write
+    # len:: the number of bytes to write
+    def write_bytes(buf, len)
+      raise NotImplementedError
+    end
+
+    # Writes an int as four bytes.
+    def write_int(i)
+      write_byte((i >> 24) & 0xFF)
+      write_byte((i >> 16) & 0xFF)
+      write_byte((i >>  8) & 0xFF)
+      write_byte(i & 0xFF)
+    end
+    alias :write_uint :write_int
+
+    # Writes an int in a variable-length format.  Writes between one and
+    # five bytes.  Smaller values take fewer bytes.  Negative numbers are not
+    # supported.
+    def write_vint(i)
+      while i > 127
+        write_byte((i & 0x7f) | 0x80)
+        i >>= 7
+      end
+      write_byte(i)
+    end
+    alias :write_vlong :write_vint
+
+    # Writes a long as eight bytes.
+    def write_long(i)
+      write_int(i >> 32)
+      write_int(i)
+    end
+    alias :write_ulong :write_long
+
+    # Writes a string.
+    def write_string(s)
+      length = s.length()
+      write_vint(length)
+      write_chars(s, 0, length)
+    end
+
+    # Writes a sequence of UTF-8 encoded characters from a string.
+    # buf:: the source of the characters
+    # start:: the first character in the sequence
+    # length:: the number of characters in the sequence
+    def write_chars(buf, start, length)
+      last = start + length
+      (start ... last).each do |i|
+        write_byte(buf[i])
+#          code = buf[i]
+#          if code >= 0x01 and code <= 0x7F
+#            write_byte(code)
+#          else
+#            # We need to write unicode characters. ToDo: test that this works.
+#            if code > 0x80 and code <= 0x7FF or code == 0
+#              write_byte(0xC0 | code >> 6)
+#              write_byte(0x80 | code & 0x3F)
+#            else
+#              write_byte(0xE0 | (code >> 12))
+#              write_byte(0x80 | ((code >> 6) & 0x3F))
+#              write_byte(0x80 | (code & 0x3F))
+#            end
+#          end
+      end
+    end
+
+    # Forces any buffered output to be written. 
+    def flush
+      raise NotImplementedError
+    end
+
+    # Closes this stream to further operations. 
+    def close
+      raise NotImplementedError
+    end
+
+    # Returns the current position in this file, where the next write will
+    # occur.
+    def pos
+      raise NotImplementedError
+    end
+
+    # Sets current position in this file, where the next write will occur.
+    def seek(pos)
+      raise NotImplementedError
+    end
+
+    # The number of bytes in the file. 
+    def length
+      raise NotImplementedError
+    end
+  end
+end

data/lib/ferret/store/ram_store.rb

@@ -0,0 +1,282 @@
+module Ferret::Store
+  require 'monitor'
+
+  class RAMDirectory < Directory
+    include MonitorMixin
+
+    def initialize(dir = nil, close_dir = false)
+      super()
+      @files = Hash.new
+      unless dir.nil?
+        dir.each do |file|
+          os = create_output(file)    # make a place on ram disk
+          is = dir.open_input(file)   # read the current file
+          len = is.length             # and copy the file to ram disk
+          buf = Array.new(len)
+          is.read_bytes(buf, 0, len)
+          os.write_bytes(buf, len)
+          is.close()
+          os.close()
+        end
+        dir.close() if close_dir
+      end
+    end
+
+    # returns an array of strings, one for each file in the directory
+    def each()
+      @files.each do |path, file|
+        next if file =~ Regexp.new('^rubylock-')
+        yield file
+      end
+    end
+
+    # Returns true if a file with the given name exists.
+    def exists?(name)
+      @files.has_key?(name)
+    end
+
+    # Returns the time the named file was last modified.
+    def modified(name)
+      @files[name].mtime
+    end
+
+    # Set the modified time of an existing file to now.
+    def touch(name)
+      if @files[name].nil?
+        @files[name] = RAMFile.new(name)
+      end
+      @files[name].mtime = Time.now
+    end
+
+    # Removes an existing file in the directory.
+    def delete(name)
+      @files.delete(name)
+    end
+
+    # Renames an existing file in the directory.
+    # If a file already exists with the new name, then it is replaced.
+    # This replacement should be atomic.
+    def rename(from, to)
+      @files[to] = @files[from]
+      @files.delete(from)
+    end
+
+    # Returns the length of a file in the directory.
+    def length(name)
+      @files[name].length
+    end
+
+    # Creates a new, empty file in the directory with the given name.
+    # Returns a stream writing this file.
+    def create_output(name)
+      file = RAMFile.new(name)
+      @files[name] = file
+      RAMIndexOutput.new(file)
+    end
+    
+    # Returns a stream reading an existing file.
+    def open_input(name)
+      raise IOError, "No file #{name}" if @files[name].nil?
+      RAMIndexInput.new(@files[name])
+    end
+    
+    def print_file(name)
+      input = RAMIndexInput.new(@files[name])
+      buf = " " * input.length
+      input.read_internal(buf, 0, input.length)
+      puts buf
+    end
+
+    # Construct a Lock.
+    def make_lock(name) 
+      RAMLock.new("rubylock-" + name, self)
+    end
+
+
+    # Closes the store.
+    def close()
+    end
+
+    def to_s
+      str = "The files in this directory are: \n"
+      @files.each do |path,file|
+        str << path + " - " + file.size.to_s + "\n"
+      end
+      str
+    end
+ 
+    class RAMIndexOutput < BufferedIndexOutput
+      def initialize(f)
+        @file = f
+        @pointer = 0
+        super()
+      end
+      
+      def length
+        return @file.length
+      end
+
+      def flush_buffer(src, len)
+        buffer_number = (@pointer / BUFFER_SIZE).to_i
+        buffer_offset = @pointer % BUFFER_SIZE
+        bytes_in_buffer = BUFFER_SIZE - buffer_offset
+        bytes_to_copy = [bytes_in_buffer, len].min
+        
+        extend_buffer_if_necessary(buffer_number)
+        
+        buffer = @file.buffers[buffer_number]
+        buffer[buffer_offset, bytes_to_copy] = src[0, bytes_to_copy]
+        
+        if bytes_to_copy < len
+          src_offset = bytes_to_copy
+          bytes_to_copy = len - bytes_to_copy
+          buffer_number += 1
+          extend_buffer_if_necessary(buffer_number)
+          buffer = @file.buffers[buffer_number]
+          buffer[0, bytes_to_copy] = src[src_offset, bytes_to_copy]
+        end
+        @pointer += len
+        @file.length = @pointer unless @pointer < @file.length
+        @file.mtime = Time.now
+      end
+
+      def reset
+        seek(0)
+        @file.length = 0
+      end
+
+      def seek(pos)
+        super(pos)
+        @pointer = pos
+      end
+
+      def close
+        super()
+        @file.mtime = Time.new
+      end
+
+      def write_to(output)
+        flush()
+        last_buffer_number = (@file.length / BUFFER_SIZE).to_i
+        last_buffer_offset = @file.length % BUFFER_SIZE
+        @file.buffers.each_with_index do |buffer, i|
+          len = (i == last_buffer_number ? last_buffer_offset : BUFFER_SIZE)
+          output.write_bytes(buffer, len)
+        end
+      end
+      
+      private
+
+      def extend_buffer_if_necessary(buffer_number)
+        if buffer_number == @file.buffers.size
+          @file.buffers << RAMFile::BUFFER.clone
+        end
+      end
+
+    end
+    
+    class RAMIndexInput < BufferedIndexInput
+
+      def initialize(f)
+        @pointer = 0
+        @file = f
+        super()
+      end
+      
+      def length
+        return @file.length
+      end
+
+      def read_internal(b, offset, length)
+        remainder = length
+        start = @pointer
+        
+        while remainder != 0
+          buffer_number = (start / BUFFER_SIZE).to_i
+          buffer_offset = start % BUFFER_SIZE
+          bytes_in_buffer = BUFFER_SIZE - buffer_offset
+          
+          if bytes_in_buffer >= remainder
+            bytes_to_copy = remainder
+          else
+            bytes_to_copy = bytes_in_buffer
+          end
+          buffer = @file.buffers[buffer_number]
+          bo2 = buffer_offset
+          do2 = offset
+          b[do2, bytes_to_copy] = buffer[bo2, bytes_to_copy]
+          offset += bytes_to_copy
+          start += bytes_to_copy
+          remainder -= bytes_to_copy
+        end
+        
+        @pointer += length
+      end
+
+      def seek_internal(pos)
+        @pointer = pos
+      end
+
+      def close
+      end
+    end
+    
+    # This class contains an array of byte arrays which act as buffers to
+    # store the data in.
+    class RAMFile
+      BUFFER = " " * BUFFER_SIZE
+
+      attr_reader :buffers
+      attr_accessor :mtime
+      #attr_accessor :name
+      attr_accessor :length
+
+
+      def initialize(name)
+        @buffers = Array.new
+        @mtime = Time.now
+        #@name = name
+        @length = 0
+      end
+    end
+
+    # A Lock is used to lock a data source (in this case a file) so that
+    # not more than one output stream can access a data source at one time.
+    class RAMLock < Lock
+      # pass the name of the file that we are going to lock
+      def initialize(lock_file, dir)
+        @lock_file = lock_file
+        @dir = dir
+      end
+
+      # obtain the lock on the data source 
+      def obtain(lock_timeout = 1) 
+        MAX_ATTEMPTS.times do
+          @dir.synchronize do
+            # create a file if none exists. If one already exists
+            # then someone beat us to the lock so return false
+            if (! locked?) then
+              @dir.create_output(@lock_file)
+              return true
+            end
+          end
+          # lock was not obtained so sleep for timeout then try again.
+          sleep(lock_timeout)
+        end
+        # lock could not be obtained so raise an exception
+        raise "could not obtain lock: " + @lock_file.to_s
+      end 
+
+      # Release the lock on the data source. Returns true if successful.
+      def release 
+        @dir.delete(@lock_file)
+        return true
+      end
+
+      # returns true if there is a lock on the data source
+      def locked?
+        @dir.exists?(@lock_file)
+      end
+    end
+  end
+end