IOWA 1.0.0

data/microprojects/DiskCache/src/iowa/caches/LRUCache.rb

@@ -0,0 +1,287 @@
+require 'enumerator'
+require 'iowa/Mutex'
+require 'iowa/Constants'
+require 'iowa/Hash'
+
+module Iowa
+	module Caches
+
+		#++
+		# Iowa::Caches::LRUCache is an LRU cache based on a linked list.
+		# It provides efficient random access to elements, like a hash,
+		# along with efficient stack or queue-like behaviors -- elements can
+		# be pushed, poped, shifted, and unshifted from the cache.
+		
+		class LRUCache
+			include Enumerable
+
+			# * Iowa::Caches::LRUCache.new
+			# * Iowa::Caches::LRUCache.new(:maxsize => 20)
+			# * Iowa::Caches::LRUCache.new(20)
+			# * Iowa::Caches::LRUCache.new(:maxsize => 20, :ttl => 3600)
+			# * Iowa::Caches::LRUCache.new(20,3600)
+			#
+			# If called with no arguments, the cache will default to a max size
+			# of 20 elements, with no max time to live (ttl) on the elements.
+			#
+			# If called using a hash with keyword arguments, the :maxsize is the
+			# maximum number of elements to keep in the hash, and :ttl is the 
+			# maximum age allowed for elements in the hash.
+			#
+			# If called with list arguments, the first is the max size, and the
+			# second is the TTL.
+			
+			def initialize(args = {Cmaxsize => 20, Cttl => nil}, ttlarg = nil)
+				if args.kind_of? ::Hash
+					oargs = args
+					args = Iowa::Hash.new
+					args.step_merge!(oargs)
+				end
+				args.stringify_keys! if args.respond_to? :stringify_keys!
+				begin
+					@max = args[Cmaxsize] || 20
+					@maxttl = args[Cttl]
+					@mutex = Iowa::Mutex.new
+				rescue Exception
+					args = {Cmaxsize => args || 20, Cttl => ttlarg}
+					retry
+				end
+				@finalizers = []
+				@head = Node.new
+				@tail = Node.new
+				@lookup = Hash.new
+				node_join(@head,@tail)
+			end
+
+			
+			# Check to see if the given key is in the cache.
+
+			def include?(v)
+				@lookup.has_key?(v)
+			end
+
+			def [](v)
+				@mutex.lock
+				if @lookup.has_key?(v)
+					n = @lookup[v]
+					n.age = Time.now
+					node_delete(n)
+					node_join(n,@head.next_node)
+					node_join(@head,n)
+					node_prune
+					@mutex.unlock
+					n.value
+				else
+					node_prune
+					@mutex.unlock
+					nil
+				end
+			end
+
+			def []=(k,v)
+				@mutex.lock
+				if @lookup.has_key?(k)
+					n = @lookup[k]
+					n.value = v
+					n.age = Time.now
+					node_delete(n)
+					node_join(n,@head.next_node)
+					node_join(@head,n)
+				else
+					n = Node.new(k,v,Time.now,@head,@head.next_node)
+					node_join(n,@head.next_node)
+					node_join(@head,n)
+					@lookup[k] = n
+				end
+				node_prune
+				@mutex.unlock
+				v
+			end
+
+			def empty?
+				@lookup.empty?
+			end
+
+			def delete(k)
+				@mutex.lock
+				n = @lookup.delete(k)
+				node_purge(n) if n
+				node_prune
+				@mutex.unlock
+				n
+			end
+
+			def first
+				@head.next_node.value
+			end
+
+			def last
+				@tail.prev_node.value
+			end
+
+			def shift
+				@mutex.lock
+				k = @head.next_node.key
+				n = @lookup.delete(k)
+				node_delete(n) if n
+				node_prune
+				@mutex.unlock
+			end
+
+			def unshift(v)
+				@mutex.lock
+				if @lookup.has_key?(v)
+					n = @lookup[v]
+					node_delete(n)
+					node_join(n,@head.next_node)
+					node_join(@head,n)
+				else
+					n = Node.new(v,v,Time.now,@head,@head.next_node)
+					node_join(n,@head.next_node)
+					node_join(@head,n)
+					@lookup[v] = n
+				end
+				node_prune
+				@mutex.unlock
+				v
+			end
+
+			def pop
+				@mutex.lock
+				k = @tail.prev_node.key
+				n = @lookup.delete(k)
+				node_delete(n) if n
+				node_prune
+				@mutex.unlock
+			end
+
+			def push(v)
+				@mutex.lock
+				if @lookup.has_key?(v)
+					n = @lookup[v]
+					node_delete(n)
+					node_join(@tail.prev_node,n)
+					node_join(n,@tail)
+				else
+					n = Node.new(v,v,Time.now,@tail.prev_node,@tail)
+					node_join(@tail.prev_node,n)
+					node_join(n,@tail)
+					@lookup[v] = n
+				end
+				node_prune
+				@mutex.unlock
+				v
+			end
+
+			def queue
+				r = []
+				n = @head
+				while (n = n.next_node) and n != @tail
+					r << n.key
+				end
+				r
+			end
+	
+			def to_a
+				r = []
+				n = @head
+				while (n = n.next_node) and n != @tail
+					r << n.value
+				end
+				r
+			end
+	
+			def length
+				@lookup.length
+			end
+			alias :size :length
+	
+			def each
+				n = @head
+				while (n = n.next_node) and n != @tail
+					yield(n.key,n.value)
+				end
+			end
+
+			def maxsize
+				@max
+			end
+
+			def maxsize=(v)
+				@max = v
+				node_prune
+			end
+			alias :size= :maxsize=
+
+			def ttl
+				@maxttl
+			end
+
+			def ttl=(v)
+				@maxttl = v
+				node_prune
+			end
+
+			# Adds a finalization method to the cache that will be called before
+			# the object in the cache is expired.	
+
+			def add_finalizer(*args,&block)
+				@finalizers.push [block,args]
+			end
+			
+			private
+	
+			def node_delete(n)
+				node_join(n.prev_node,n.next_node)
+			end
+	
+			def node_purge(n)
+				node_join(n.prev_node,n.next_node)
+				v = n.value
+				k = n.key
+				n.value = nil
+				n.key = nil
+				n.next_node = nil
+				n.prev_node = nil
+				do_finalization(k,v) if @finalizers
+				v
+			end
+
+			def node_join(a,b)
+				a.next_node = b
+				b.prev_node = a
+			end
+
+			def node_prune
+				node_purge(@lookup.delete(@tail.prev_node.key)) while @lookup.size > @max
+				if @maxttl
+					expiration_date = Time.now - @maxttl
+					while (n = @tail.prev_node).age < expiration_date
+						node_purge(@lookup.delete(n.key))
+					end
+				end
+			end
+
+			# Is called when an object in the cache is expired.  Iterates through
+			# the defined finalization methods, if any.
+			def do_finalization(key,obj)
+				@finalizers.each do |f|
+					f[0].call(key,obj,*f[1])
+				end
+			end
+
+			class Node
+				attr_accessor :key, :value, :age, :prev_node, :next_node
+
+				def initialize(key=nil,value=nil,age=Time.now,prev_node=nil,next_node=nil)
+					@key = key
+					@value = value
+					@age = age
+					@prev_node = prev_node
+					@next_node = next_node
+				end
+			end
+
+		end
+	end
+end

data/microprojects/DiskCache/test/TC_DiskCache.rb

@@ -0,0 +1,218 @@
+require 'test/unit'
+require 'external/test_support'
+IWATestSupport.set_src_dir
+require 'iowa/caches/DiskCache'
+require 'iowa/caches/DiskStore'
+require 'tmpdir'
+require 'fileutils'
+require 'benchmark'
+require 'pstore'
+
+TMPDIR = Dir::tmpdir
+
+class TC_DiskCache < Test::Unit::TestCase
+	def make_tmpdir_path; FileUtils.mkdir("#{TMPDIR}/tc_diskcache.#{Time.now.to_i}#{rand}").first; end
+	def make_shm_tmpdir_path; FileUtils.mkdir("/dev/shm/tc_diskcache.#{Time.now.to_i}#{rand}").first; end
+	@@testdir = IWATestSupport.test_dir(__FILE__)
+
+	def setup
+		Dir.chdir(@@testdir)
+		IWATestSupport.announce(:diskcache,"Iowa::Caches::DiskCache / Iowa::Caches::DiskStore")
+	end
+
+	def test_creation
+		tmpdir = make_tmpdir_path
+		assert_nothing_raised("Failed while creating an Iowa::Caches::DiskCache object") do
+			@cache = Iowa::Caches::DiskCache.new({:directory => tmpdir, :maxsize => 12})
+		end
+		FileUtils.rm_rf(tmpdir)
+	end
+
+	def test_cache_size
+		tmpdir = make_tmpdir_path
+		@cache = Iowa::Caches::DiskCache.new({:directory => tmpdir, :maxsize => 12})
+		(1..14).each {|n| @cache[n] = n}
+		s = @cache.size
+		assert_equal(12,s,"The expected size (12) did not match the returned size (#{s}).")
+		FileUtils.rm_rf(tmpdir)
+	end
+
+	def test_cache_size2
+		tmpdir = make_tmpdir_path
+		@cache = Iowa::Caches::DiskCache.new({:directory => tmpdir, :maxsize => 12})
+		(1..14).each {|n| @cache[n] = n}
+		@cache.size=3
+		s = @cache.size
+		assert_equal(3,s,"The cache size does not appear to to have change as expected; want size of 3, have size of #{s}.")
+		FileUtils.rm_rf(tmpdir)
+	end
+
+	def test_values
+		tmpdir = make_tmpdir_path
+		@cache = Iowa::Caches::DiskCache.new({:directory => tmpdir, :maxsize => 3})
+		@cache['a'] = 1
+		@cache['b'] = 2
+		@cache['c'] = 3
+		s = @cache['b']
+		assert_equal(2,s,"The value retrieved from the cache (#{s}) did not match what was expected (2).")
+		FileUtils.rm_rf(tmpdir)
+	end
+
+	def test_expire1
+		tmpdir = make_tmpdir_path
+		@cache = Iowa::Caches::DiskCache.new({:directory => tmpdir, :maxsize => 3})
+		@cache['a'] = 1
+		@cache['b'] = 2
+		@cache['c'] = 3
+		s = @cache['b']
+		@cache['d'] = 4
+		s = true
+		s = false if @cache.include? 'a'
+		assert_equal(true,s,"Key 'a' should have fallen out of the cache.  It did not.")
+		FileUtils.rm_rf(tmpdir)
+	end
+
+	def test_expire2
+		tmpdir = make_tmpdir_path
+		@cache = Iowa::Caches::DiskCache.new({:directory => tmpdir, :maxsize => 3})
+		@cache['a'] = 1
+		@cache['b'] = 2
+		@cache['c'] = 3
+		s = @cache['b']
+		@cache['d'] = 4
+		s = false
+		s = true if @cache.include? 'b'
+		assert_equal(true,s,"Key 'b' appears to have fallen out of the cache.  It should not have.")
+		FileUtils.rm_rf(tmpdir)
+	end
+
+	def test_transactions
+		tmpdir = make_tmpdir_path
+		@cache = Iowa::Caches::DiskCache.new({:directory => tmpdir, :maxsize => 20})
+		assert_nothing_raised("Something blew up when trying to test transactions.") do
+			@cache[:a] = 1
+			@cache[:b] = 2
+			@cache.transaction do
+				@cache[:c] = 3
+			end
+			assert_equal(3,@cache[:c],"Transaction failed.")
+			@cache.transaction do
+				@cache[:d] = 4
+				@cache.commit
+				@cache[:e] = 5
+			end
+			assert_equal(4,@cache[:d],"Transaction failed.")
+			assert_equal(5,@cache[:e],"Transaction failed.")
+			@cache.transaction do
+				@cache[:f] = 6
+				@cache.rollback
+			end
+			assert(!@cache.include?(:f),"Transaction rollback failed.")
+			@cache.transaction do
+				@cache[:f] = 6
+				@cache.rollback
+				@cache[:g] = 6
+			end
+			assert(!@cache.include?(:f),"Transaction rollback failed.")
+			assert_equal(6,@cache[:g],"Transaction failed.")
+			@cache.transaction do
+				@cache[:h] = 7
+				@cache.transaction do
+					@cache[:i] = 8
+					@cache.transaction do
+						@cache[:j] = 9
+						@cache.rollback
+					end
+					@cache.transaction do
+						@cache[:k] = 9
+					end
+					@cache.rollback
+				end
+			end
+			assert(!@cache.include?(:j),"Nested transaction rollback(:j) failed.")
+			assert(!@cache.include?(:i),"Nexted transaction rollback(:i) failed.")
+			assert_equal(7,@cache[:h],"Transaction failed.")
+			assert_equal(9,@cache[:k],"Transaction failed.")
+		end
+	end
+	
+	def test_speed
+		n = 2000
+		test_dirs = []
+		test_dirs << ["In-Memory (/dev/shm) Test", make_shm_tmpdir_path] if FileTest.exist?('/dev/shm') and FileTest.writable?('/dev/shm')
+		test_dirs << ["On Disk Test (performance directly related to disk i/o performance)", make_tmpdir_path]
+		assert_nothing_raised("Something blew up during the DiskCache stress test.") do
+			test_dirs.each do |td|
+				puts "\n#{td[0]}"
+				cache = Iowa::Caches::DiskCache.new({:directory => td[1], :maxsize => 4294967296})
+				Benchmark.benchmark do |bm|
+					flush_disk
+					puts "Writing #{n} records to disk cache:"
+					bm.report('dc write') do
+						n.times {|x| cache[x] = x}
+					end
+					flush_disk
+					puts "Updating #{n} records in disk cache:"
+					bm.report('dc update') do
+						n.times {|x| cache[x] = x * 2}
+					end
+					flush_disk
+					puts "Reading #{n} records from disk cache:"
+					bm.report('dc read') do
+						(n).times {|x| cache[x]}
+					end
+					flush_disk
+					puts "Deleting #{n} records from disk cache:"
+					bm.report('dc delete') do
+						n.times {|x| cache.delete(x)}
+					end
+					flush_disk
+				end
+				FileUtils.rm_rf(td[1])
+			end
+		end
+
+		flush_disk
+		puts "\n\nNow Running speed comparison of DiskStore"
+		test_dirs = []
+		test_dirs << ["In-Memory (/dev/shm) Test", make_shm_tmpdir_path] if FileTest.exist?('/dev/shm') and FileTest.writable?('/dev/shm')
+		test_dirs << ["On Disk Test (performance directly related to disk i/o performance)", make_tmpdir_path]
+		assert_nothing_raised("Something blew up during the DiskCache stress test.") do
+			test_dirs.each do |td|
+				puts "\n#{td[0]}"
+				cache = Iowa::Caches::DiskStore.new({:directory => td[1]})
+				Benchmark.benchmark do |bm|
+					flush_disk
+					puts "Writing #{n} records to disk cache:"
+					bm.report('dc write') do
+						n.times {|x| cache[x] = x}
+					end
+					flush_disk
+					puts "Updating #{n} records in disk cache:"
+					bm.report('dc update') do
+						n.times {|x| cache[x] = x * 2}
+					end
+					flush_disk
+					puts "Reading #{n} records from disk cache:"
+					bm.report('dc read') do
+						(n).times {|x| cache[x]}
+					end
+					flush_disk
+					puts "Deleting #{n} records from disk cache:"
+					bm.report('dc delete') do
+						n.times {|x| cache.delete(x)}
+					end
+					flush_disk
+				end
+				FileUtils.rm_rf(td[1])
+			end
+		end
+	end
+
+	def flush_disk
+		if FileTest.exist?('/bin/sync')
+			system('/bin/sync')
+		end
+	end
+
+end