cuboid 0.0.0 → 0.0.3

data/lib/cuboid/support/database/base.rb

@@ -0,0 +1,177 @@
+module Cuboid
+module Support::Database
+
+# Base class for Database data structures
+#
+# Provides helper methods for data structures to be implemented related to
+# objecting dumping, loading, unique filename generation, etc.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+#
+# @abstract
+class Base
+
+    DISK_SPACE_FILE = 'Database_disk_space'
+
+    class <<self
+
+        def reset
+            @@disk_space = 0
+            set_disk_space @@disk_space
+        end
+
+        def increment_disk_space( int )
+            set_disk_space @@disk_space + int
+        end
+
+        def decrement_disk_space( int )
+            set_disk_space @@disk_space - int
+        end
+
+        def disk_space
+            @@disk_space
+        end
+
+        def disk_directory
+            Options.paths.tmpdir
+        end
+
+        def disk_space_file
+            disk_space_file_for Process.pid
+        end
+
+        def disk_space_for( pid )
+            return 0 if !Dir.exists?( Options.paths.tmp_dir_for( pid ) )
+
+            IO.read( disk_space_file_for( pid ) ).to_i
+        end
+
+        def disk_space_file_for( pid )
+            "#{Options.paths.tmp_dir_for( pid )}/#{DISK_SPACE_FILE}"
+        end
+
+        private
+
+        def set_disk_space( int )
+            if !File.exist?( disk_directory )
+                # Could be caught in #at_exit callbacks, the tmpdir has already
+                # been deleted.
+                return
+            end
+
+            synchronize do
+                @@disk_space = int
+                IO.write( disk_space_file, @@disk_space.to_s )
+            end
+        end
+
+        def synchronize( &block )
+            (@@mutex ||= Mutex.new).synchronize( &block )
+        end
+
+    end
+    reset
+
+    # @param    [Hash]    options
+    #   Any object that responds to 'dump' and 'load'.
+    def initialize( options = {} )
+        @options = options.dup
+
+        @options[:dumper] ||= Marshal
+        @options[:loader] ||= @options[:dumper]
+
+        @filename_counter = 0
+
+        at_exit do
+            clear
+        end
+    end
+
+    def serialize( obj )
+        compress(
+            @options[:dumper].respond_to?( :dump ) ?
+                @options[:dumper].dump( obj ) :
+                @options[:dumper].call( obj )
+        )
+    end
+
+    def unserialize( data )
+        data = decompress( data )
+
+        @options[:loader].respond_to?( :load ) ?
+            @options[:loader].load( data ) :
+            @options[:loader].call( data )
+    end
+
+    private
+
+    # Dumps the object to a unique file and returns its path.
+    #
+    # The path can be used as a reference to the original value
+    # by way of passing it to load().
+    #
+    # @param    [Object]    obj
+    #
+    # @return   [String]
+    #   Filepath
+    def dump( obj, &block )
+        File.open( get_unique_filename, 'wb' ) do |f|
+            serialized = serialize( obj )
+            self.class.increment_disk_space f.write( serialized )
+
+            block.call( serialized ) if block_given?
+
+            f.path
+        end
+    end
+
+    # Loads the object stored in filepath.
+    #
+    # @param    [String]    filepath
+    #
+    # @return   [Object]
+    def load( filepath )
+        unserialize( IO.binread( filepath ) )
+    end
+
+    # Deletes a file.
+    #
+    # @param    [String]    filepath
+    def delete_file( filepath )
+        return if !File.exist?( filepath )
+
+        self.class.decrement_disk_space File.size( filepath )
+        File.delete( filepath )
+    end
+
+    # Loads the object in file and then removes it from the file-system.
+    #
+    # @param    [String]    filepath
+    #
+    # @return   [Object]
+    def load_and_delete_file( filepath )
+        obj = load( filepath )
+        delete_file( filepath )
+        obj
+    end
+
+    def compress( string )
+        Zlib::Deflate.deflate string
+    end
+
+    def decompress( string )
+        Zlib::Inflate.inflate string
+    end
+
+    def get_unique_filename
+        # Should be unique enough...
+        ("#{self.class.disk_directory}/#{self.class.name}_#{object_id}_" <<
+            @filename_counter.to_s).gsub( '::', '_' )
+    ensure
+        @filename_counter += 1
+    end
+
+end
+
+end
+end

data/lib/cuboid/support/database/categorized_queue.rb

@@ -0,0 +1,195 @@
+module Cuboid
+module Support::Database
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class CategorizedQueue < Base
+
+    # Default {#max_buffer_size}.
+    DEFAULT_MAX_BUFFER_SIZE = 100
+
+    # @return   [Integer]
+    #   How many entries to keep in memory before starting to off-load to disk.
+    attr_accessor :max_buffer_size
+
+    attr_accessor :prefer
+
+    # @see Cuboid::Database::Base#initialize
+    def initialize( options = {}, &block )
+        super( options )
+
+        @prefer = block
+        @max_buffer_size = options[:max_buffer_size] || DEFAULT_MAX_BUFFER_SIZE
+
+        @categories ||= {}
+        @waiting = []
+        @mutex   = Mutex.new
+
+        @buffer_size = 0
+        @disk_size   = 0
+    end
+
+    # @note Defaults to {DEFAULT_MAX_BUFFER_SIZE}.
+    #
+    # @return   [Integer]
+    #   How many entries to keep in memory before starting to off-load to disk.
+    def max_buffer_size
+        @max_buffer_size
+    end
+
+    def categories
+        @categories.keys
+    end
+
+    def data_for( category )
+        @categories[category.to_s] ||= {
+            disk:   [],
+            buffer: []
+        }
+    end
+
+    def insert_to_disk( category, path )
+        data_for( category )[:disk] << path
+        @disk_size += 1
+    end
+
+    # @param    [Object]    obj
+    #   Object to add to the queue.
+    #   Must respond to #category.
+    def <<( obj )
+        fail ArgumentError, 'Missing #prefer block.' if !@prefer
+
+        if !obj.respond_to?( :category )
+            fail ArgumentError, "#{obj.class} does not respond to #category."
+        end
+
+        synchronize do
+            data = data_for( obj.category )
+
+            if data[:buffer].size < max_buffer_size
+                @buffer_size += 1
+                data[:buffer] << obj
+            else
+                @disk_size += 1
+                data[:disk] << dump( obj )
+            end
+
+            begin
+                t = @waiting.shift
+                t.wakeup if t
+            rescue ThreadError
+                retry
+            end
+        end
+    end
+    alias :push :<<
+    alias :enq :<<
+
+    # @return   [Object]
+    #   Removes an object from the queue and returns it.
+    def pop( non_block = false )
+        fail ArgumentError, 'Missing #prefer block.' if !@prefer
+
+        synchronize do
+            loop do
+                if internal_empty?
+                    raise ThreadError, 'queue empty' if non_block
+                    @waiting.push Thread.current
+                    @mutex.sleep
+                else
+                    # Get preferred category, hopefully there'll be some data
+                    # for it.
+                    category = @prefer.call( @categories.keys )
+
+                    # Get all other available categories just in case the
+                    # preferred one is empty.
+                    categories = @categories.keys
+                    categories.delete category
+
+                    data = nil
+                    # Check if our category has data and pick another if not.
+                    loop do
+                        data = data_for( category )
+                        if data[:buffer].any? || data[:disk].any?
+                            break
+                        end
+
+                        category = categories.pop
+                    end
+
+                    if data[:buffer].any?
+                        @buffer_size -= 1
+                        return data[:buffer].shift
+                    end
+
+                    @disk_size -= 1
+                    return load_and_delete_file( data[:disk].shift )
+                end
+            end
+        end
+    end
+    alias :deq :pop
+    alias :shift :pop
+
+    # @return   [Integer]
+    #   Size of the queue, the number of objects it currently holds.
+    def size
+        buffer_size + disk_size
+    end
+    alias :length :size
+
+    def free_buffer_size
+        max_buffer_size - buffer_size
+    end
+
+    def buffer_size
+        @buffer_size
+    end
+
+    def disk_size
+        @disk_size
+    end
+
+    # @return   [Bool]
+    #   `true` if the queue if empty, `false` otherwise.
+    def empty?
+        synchronize do
+            internal_empty?
+        end
+    end
+
+    # Removes all objects from the queue.
+    def clear
+        synchronize do
+            @categories.values.each do |data|
+                data[:buffer].clear
+
+                while !data[:disk].empty?
+                    path = data[:disk].pop
+                    next if !path
+                    delete_file path
+                end
+            end
+
+            @buffer_size = 0
+            @disk_size   = 0
+        end
+    end
+
+    def num_waiting
+        @waiting.size
+    end
+
+    private
+
+    def internal_empty?
+        @buffer_size == 0 && @disk_size == 0
+    end
+
+    def synchronize( &block )
+        @mutex.synchronize( &block )
+    end
+
+end
+
+end
+end

data/lib/cuboid/support/database/hash.rb

@@ -0,0 +1,300 @@
+require 'digest/sha1'
+
+module Cuboid
+module Support::Database
+
+# Flat-file Hash implementation
+#
+# Behaves pretty much like a Ruby Hash however it transparently serializes and
+# saves its values to the file-system under the OS's temp directory.
+#
+# It's not interchangeable with Ruby's Hash as it lacks a lot of the
+# stdlib methods.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+# @version 0.1
+class Hash < Base
+
+    # @see Cuboid::Database::Base#initialize
+    def initialize( *args )
+        super( *args )
+
+        # holds the internal representation of the Hash
+        # same keys as self but the values are actually pointing to filepaths
+        # where the real values are being stores
+        @h = ::Hash.new
+
+        # holds a key-value pair of self with digests as values
+        # in order to allow comparisons without requiring to load
+        # the actual values from their files.
+        @eql_h = ::Hash.new
+    end
+
+    # Associates the given value  with the given key.
+    #
+    # @param    [Object]    k
+    #   Key.
+    # @param    [Object]    v
+    #   Value.
+    #
+    # @return   [Object]
+    #   `v`.
+    def []=( k, v )
+        @h[k] = dump( v ) do |serialized|
+            @eql_h[k] = eql_hash( serialized )
+        end
+    end
+    alias :store :[]=
+
+    # @param    [Obj]   k   key
+    #
+    # @return   [Object]
+    #   Object corresponding to the key object, `nil` otherwise.
+    def []( k )
+        load( @h[k] ) if @h[k]
+    end
+
+    # @param    [Object]    k
+    #   Key.
+    #
+    # @return   [Array]
+    #   Array containing the given key and its value.
+    def assoc( k )
+        return if !@h[k]
+        [ k, self[k] ]
+    end
+
+    # @param    [Object]    v
+    #   Value.
+    #
+    # @return   [Array]
+    #   Array containing the key for the given value and that value.
+    def rassoc( v )
+        return if !value?( v )
+        [ key( v ), v ]
+    end
+
+    # Removes an entry by key and returns its value.
+    #
+    # If the key doesn't exist and a block has been provided it's passed
+    # the key and the method returns the result of that block.
+    #
+    # @param    [Object]    k
+    #   Key.
+    #
+    # @return   [Object]
+    def delete( k, &block )
+        if @h[k]
+            obj = load_and_delete_file( @h[k] )
+            @h.delete( k )
+            @eql_h.delete( k )
+            return obj
+        else
+            block.call( k ) if block_given?
+        end
+    end
+
+    # Removes the first key-value pair from the hash and returns it as a array,
+    #
+    # @return   [Array]
+    def shift
+        k, v = @h.first
+        [ k, delete( k ) ]
+    end
+
+    # Calls block with each key-value pair.
+    #
+    # If a block has been given it retuns self.
+    # If no block has been given it returns an enumerator.
+    #
+    # @param    [Proc]      block
+    def each( &block )
+        if block_given?
+            @h.each { |k, v| block.call( [ k, self[k] ] ) }
+            self
+        else
+            enum_for( :each )
+        end
+    end
+    alias :each_pair :each
+
+    # Calls block with each key.
+    #
+    # If a block has been given it returns self.
+    # If no block has been given it returns an enumerator.
+    #
+    # @param    [Proc]      block
+    def each_key( &block )
+        if block_given?
+            @h.each_key( &block )
+            self
+        else
+            enum_for( :each_key )
+        end
+    end
+
+    # Calls block with each value.
+    #
+    # If a block has been given it returns `self`.
+    # If no block has been given it returns an enumerator.
+    #
+    # @param    [Proc]      block
+    def each_value( &block )
+        if block_given?
+            @h.keys.each { |k| block.call( self[k] ) }
+            self
+        else
+            enum_for( :each_value )
+        end
+    end
+
+    # @return   [Array]
+    #   Keys.
+    def keys
+        @h.keys
+    end
+
+    # @param    [Object]    val
+    #
+    # @return   [Object]    key
+    #   key for the given value.
+    def key( val )
+        return if !value?( val )
+        each { |k, v| return k if val == self[k] }
+        nil
+    end
+
+    # @return   [Array]
+    #   Values.
+    def values
+        each_value.to_a
+    end
+
+    # @return   [Bool]
+    #   `true` if the given key exists in the hash, `false` otherwise.
+    def include?( k )
+        @h.include?( k )
+    end
+    alias :member? :include?
+    alias :key? :include?
+    alias :has_key? :include?
+
+    # @return   [Bool]
+    #   `true` if the given value exists in the hash, `false` otherwise.
+    def value?( v )
+        each_value { |val| return true if val == v }
+        false
+    end
+
+    # Merges the contents of self with the contents of the given hash and
+    # returns them in a new object.
+    #
+    # @param    [Hash]  h
+    #
+    # @return   [Cuboid::Database::Set]
+    def merge( h )
+        self.class.new( @options ).merge!( self ).merge!( h )
+    end
+
+    # Merges self with the contents of the given hash and returns self.
+    #
+    # If the given Hash is of the same type as self then the values will
+    # not be loaded during the merge in order to keep memory usage down.
+    #
+    # If the given Hash is any other kind of object it will be coerced
+    # to a Hash by calling 'to_hash' on it and the merging it with self.
+    #
+    # @param    [Hash]  h
+    def merge!( h )
+        if !h.is_a?( self.class )
+            h.to_hash.each do |k, v|
+                delete( k ) if @h.include?( k )
+                self[k] = v
+            end
+        else
+            h._internal.each do |k, v|
+                delete( k ) if @h.include?( k )
+                @h[k] = v
+            end
+            @eql_h.merge!( h._eql_h )
+        end
+        self
+    end
+    alias :update :merge!
+
+    # @return   [Hash]
+    #   `self` as Ruby Hash
+    def to_hash
+        h = {}
+        each { |k, v| h[k] = v }
+        h
+    end
+    alias :to_h :to_hash
+
+    # @return   [Array]
+    #   `self` as a Ruby Array.
+    def to_a
+        to_hash.to_a
+    end
+
+    # @return   [Integer]
+    #   Number of objects.
+    def size
+        @h.size
+    end
+    alias :length :size
+
+    # @return   [Bool]
+    #   `true` if the Hash if empty, `false` otherwise.
+    def empty?
+        @h.empty?
+    end
+
+    # Removes all objects.
+    def clear
+        @h.values.each { |filepath| delete_file( filepath ) }
+        @h.clear
+    end
+
+    # @note If the given hash is not of the same type as self it will be coerced
+    #   to a Ruby Hash by calling 'to_hash' on it.
+    #
+    # @return  [Bool]
+    #   `true` if self and the given hash contain the same key-pair values.
+    def ==( h )
+        if !h.is_a?( self.class )
+            eql = {}
+            h.to_hash.each { |k, v| eql[k] = eql_hash( serialize( v ) ) }
+            @eql_h == eql
+        else
+            @eql_h == h._eql_h
+        end
+    end
+    alias :eql? :==
+
+    # It will return a Ruby Hash with the same values as self but
+    # with filepaths as values (pointing to the files that store them).
+    #
+    # This is used for efficient merging, i.e. without requiring to load
+    # the actual values when merging 2 objects.
+    #
+    # @return   [Hash]
+    #   Internal representation of `self`.
+    def _internal
+        @h.dup
+    end
+
+    def _eql_h
+        @eql_h.dup
+    end
+
+    private
+
+    def eql_hash( str )
+        Digest::SHA1.hexdigest( str )
+    end
+
+end
+
+end
+end