persistent-cache 0.0.8 → 0.1.2

data/README.md

@@ -1,11 +1,25 @@
 # Persistent::Cache
 
-Persistent cache behaves like a hash, with a sqlite3 storage back-end.
+Persistent cache behaves like a hash, with a pluggable back-end. Currently sqlite3 and file system directory back-ends are provided. The cache defaults to type STORAGE_SQLITE
 
-Values in the cache have a default freshness period of 15465600 ms. This can be configured in the cache initializer. Setting fresh = nil indicates that data remains fresh for-ever. Each user of the cache may have his own independent freshness value. If stale data is requested from the cache, nil is returned. Updates to the cache are written to the sqlite3 storage, with SQL driver timeout set to 30 seconds. Data is marshalled before storage. If a key is not found in the cache, nil is returned. Setting the value of a key in the cache to nil deletes the entry from the sqlite database.
+Values in the cache have a default freshness period of 15465600 ms. This can be configured in the cache initializer. Setting fresh = nil indicates that data remains fresh for-ever. Each user of the cache may have his own independent freshness value. If stale data is requested from the cache, nil is returned. Data is marshalled before storage. If a key is not found in the cache, nil is returned. Setting the value of a key in the cache to nil deletes the entry. If required, creation time of an entry can be specified using set(key, value, timestamp)
 
 This gem is sponsored by Hetzner (Pty) Ltd - http://hetzner.co.za
 
+## StorageSQLite
+
+Updates to the cache are written to the sqlite3 storage, with SQL driver timeout set to 30 seconds.
+
+## StorageDirectory
+
+Keys are required to be strings that are valid for use as directory names. The cache then stores from a storage root (configured in the StorageDirector constructor) with a subdirectory for each key, and a file called 'cache' for the value. The first line in the cache file is the timestamp of the entry.
+
+When a StorageDirectory is used, it can be asked whether a key is present and what the path to a cache value is using:
+
+    get_value_path(key)
+
+    key_cached?(key)
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -60,6 +74,10 @@ Or install it yourself as:
 
     cache = Persistent::Cache.new("/tmp/my-persistent-cache", nil) # for-ever fresh
 
+    cache = Persistent::Cache.new("/tmp/directory-cache", nil, STORAGE_DIRECTORY)
+
+    cache.set("mykey", "myvalue", Time.now) # explicitly set creation time
+
 Please send feedback and comments to the authors at:
 
 Wynand van Dyk <wynand.van.dyk@hetzner.co.za>

data/lib/persistent-cache.rb

@@ -1,19 +1,36 @@
 require "persistent-cache/version"
 require "sqlite3"
+require "persistent-cache/storage/storage_sq_lite"
 
 module Persistent
   class Cache
+    STORAGE_SQLITE = 'sqlite' unless defined? STORAGE_SQLITE
+    STORAGE_DIRECTORY = 'directory' unless defined? STORAGE_DIRECTORY
+
     # Fresh is 1 day less than the bacula default job retention time. If this is configured differently, FRESH should be updated as well.
     FRESH = 15465600; FRESH.freeze
-    DB_TABLE = "key_value"; DB_TABLE.freeze
 
-    attr_accessor :database_details
+    attr_accessor :storage_details
+    attr_accessor :storage
+    attr_accessor :fresh
+
+    def initialize(storage_details, fresh = FRESH, storage = STORAGE_SQLITE)
+      raise ArgumentError.new("No storage details provided") if storage_details.nil? or storage_details == ""
 
-    def initialize(database_details, fresh = FRESH)
-      @database_details = database_details
-      @db_handle = connect_to_database
-      @db_handle.busy_timeout = 30000
+      @storage = StorageSQLite.new(storage_details) if storage == STORAGE_SQLITE
+      @storage = StorageDirectory.new(storage_details) if storage == STORAGE_DIRECTORY
       @fresh = fresh
+      @storage_details = storage_details
+
+      raise ArgumentError.new("Unsupported storage type #{storage}}") if @storage.nil?
+    end
+
+    def set(key, value, timestamp)
+      if value.nil?
+        delete_entry(Marshal.dump(key))
+      else
+        save_key_value_pair(Marshal.dump(key), Marshal.dump(value), timestamp)
+      end
     end
 
     def []=(key, value)
@@ -35,29 +52,28 @@ module Persistent
     end
 
     def size
-      @db_handle.execute("SELECT value FROM #{DB_TABLE}").size
+      @storage.size
     end
 
     def keys
-      @db_handle.execute("SELECT key FROM #{DB_TABLE}").collect { |key|
+      @storage.keys.collect { |key|
         Marshal.load(key[0])
       }
     end
 
     def clear
-      @db_handle.execute("DELETE FROM #{DB_TABLE}")
+      @storage.clear
     end
 
-    # Methods not supported by this implementation
     private
 
-    def save_key_value_pair(serialized_key, serialized_value)
+    def save_key_value_pair(serialized_key, serialized_value, timestamp = nil)
       delete_entry(serialized_key)
-      @db_handle.execute("INSERT INTO #{DB_TABLE} (key, value, timestamp) VALUES(?, ?, ?)",serialized_key, serialized_value, Time.now.to_s)
+      @storage.save_key_value_pair(serialized_key, serialized_value, timestamp)
     end
 
     def lookup_key(serialized_key)
-      result = @db_handle.execute("SELECT value, timestamp FROM #{DB_TABLE} WHERE key=?", serialized_key)
+      result = @storage.lookup_key(serialized_key)
       return nil if nil_result?(result)
       return nil if stale_entry?(serialized_key, result)
 
@@ -66,6 +82,7 @@ module Persistent
 
     def stale_entry?(serialized_key, result)
       return false if @fresh.nil?
+
       timestamp = Time.parse(result[0][1])
       if ((Time.now - timestamp) > FRESH)
         delete_entry(serialized_key)
@@ -75,26 +92,7 @@ module Persistent
     end
 
     def delete_entry(serialized_key)
-      @db_handle.execute("DELETE FROM #{DB_TABLE} WHERE key=?", serialized_key)
-    end
-
-    def open_database
-      @handle = SQLite3::Database.open(@database_details)
-    end
-
-    def create_database
-      @handle = SQLite3::Database.new(@database_details)
-      create_table
-      @handle
-    end
-
-    def create_table
-      @handle.execute("CREATE TABLE #{DB_TABLE}(key TEXT PRIMARY KEY, value TEXT, timestamp TEXT)")
-      @handle
-    end
-
-    def connect_to_database
-      File.exists?(@database_details) ? open_database : create_database
+      @storage.delete_entry(serialized_key)
     end
 
     def nil_result?(result)

data/lib/persistent-cache/storage/storage_directory.rb

@@ -0,0 +1,165 @@
+require 'fileutils'
+
+module Persistent
+  class StorageDirectory
+    CACHE_FILE = "cache" unless defined? CACHE_FILE; CACHE_FILE.freeze
+
+    attr_accessor :storage_root
+
+    def initialize(storage_details)
+      raise ArgumentError.new("Storage details not provided") if storage_details.nil? or storage_details == ""
+      @storage_root = storage_details
+      connect_to_database
+    end
+
+    def connect_to_database
+      FileUtils.makedirs([@storage_root]) if not File.exists?(@storage_root)
+    end
+
+    def save_key_value_pair(key, value, timestamp = nil)
+      prepare_to_store_key_value(key, value, timestamp)
+      store_key_value(key, value, get_time(timestamp)) if not value.nil?
+    end
+
+    def lookup_key(key)
+      validate_key(key)
+      return [] if not File.exists? compile_value_path(key)
+      lookup_key_value_timestamp(key)
+    end
+
+    def delete_entry(key)
+      validate_key(key)
+      FileUtils.rm_rf(compile_key_path(key))
+    end
+
+    def size
+      count = Dir::glob("#{@storage_root}/**/").size
+      # if the directory does not exist, count == 0, which is what we want
+      return 0 if count == 0
+      # if the directory does exist, but is empty, count == 1, namely the directory itself, and we want to return 0 (i.e. count - 1)
+      # if the directory does exist and includes subdirectories, the directory itself is still counted as well, and we want to return count - 1
+      return count - 1
+    end
+
+    def keys
+      return [] if size == 0
+      list_keys_sorted
+    end
+
+    def clear
+      keys.each do |key|
+        delete_entry(key[0])
+      end
+    end
+
+    def get_value_path(key)
+      validate_key(key)
+
+      return nil if not key_cached?(key)
+
+      compile_value_path(key)
+    end
+
+    def key_cached?(key)
+      # don't read the value here, as it may be very large - rather look whether the key is present
+      File.exists? compile_key_path(key)
+    end
+
+    private
+
+    def list_keys_sorted
+      result = []
+      append_keys(result).sort
+    end
+
+    def append_keys(result)
+      get_key_directories.each do |dir|
+        result << extract_key_from_directory(dir)
+      end
+      result
+    end
+
+    def get_key_directories
+      subdirectories = Dir::glob("#{@storage_root}/**/")
+      #exclude the storage root directory itself
+      subdirectories[1..-1]
+    end
+
+    def extract_key_from_directory(dir)
+      key = dir.match(/#{@storage_root}\/(\w+)\//)[1]
+      [key]
+    end
+
+    def compile_key_path(key)
+      "#{@storage_root}/#{key}"
+    end
+
+    def compile_value_path(key)
+      "#{compile_key_path(key)}/#{CACHE_FILE}"
+    end
+
+    def validate_save_key_value_pair(key, value)
+      validate_key(key)
+      raise ArgumentError.new("Only string values allowed") if not value.is_a?(String)
+    end
+
+    def lookup_key_value_timestamp(key)
+      result = [[],[]]
+      data = File.read(compile_value_path(key))
+      format_value_timestamp(result, data)
+    end
+
+    def format_value_timestamp(result, data)
+      result[0][0] = data.lines.to_a[1..-1].join
+      result[0][1] = data.lines.to_a[0..0].join.split("\n")[0]
+      result
+    end
+
+    def validate_key(key)
+      raise ArgumentError.new("Only string keys allowed") if not key.is_a?(String)
+    end
+
+    def empty_key_value(key)
+      FileUtils.makedirs([compile_key_path(key)]) if not File.exists?(compile_key_path(key))
+      FileUtils.rm_f(compile_value_path(key))
+    end
+
+    def get_time(timestamp)
+      timestamp.nil? ? Time.now.to_s : timestamp.to_s
+    end
+
+    def prepare_to_store_key_value(key, value, timestamp)
+      validate_save_key_value_pair(key, value)
+      delete_entry(key)
+      empty_key_value(key)
+    end
+
+    def store_key_value(key, value, timestamp)
+      tempfile = Tempfile.new("store_key_value")
+      store_value_timestamp_in_file(value, timestamp, tempfile)
+      sync_cache_value(key, tempfile)
+    end
+
+    def sync_cache_value(key, tempfile)
+      target = compile_value_path(key)
+      FileUtils.rm_f(target)
+      FileUtils.move(tempfile.path, target)
+    end
+
+    def store_value_timestamp_in_file(value, timestamp, tempfile)
+      output = File.open(tempfile, 'a')
+      write_value_timestamp(output, value, timestamp)
+      output.close
+    end
+
+    def write_value_timestamp(output, value, timestamp)
+      # Have to be explicit about writing to the file. 'puts' collapses newline at the end of the file and
+      # so does not guarantee exact replication of 'value' on lookup
+      output.write(timestamp)
+      output.write("\n")
+      output.write(value)
+      output.flush
+    end
+
+  end
+end

data/lib/persistent-cache/storage/storage_sq_lite.rb

@@ -0,0 +1,84 @@
+require "sqlite3"
+require "eh/eh"
+
+module Persistent
+  class StorageSQLite
+    DB_TABLE = "key_value" unless defined? DB_TABLE; DB_TABLE.freeze
+    DB_TIMEOUT = 30000 unless defined? DB_TIMEOUT; DB_TIMEOUT.freeze
+
+    attr_accessor :storage_details
+    attr_accessor :storage_handler
+
+    def initialize(storage_details)
+      raise ArgumentError.new("Storage details not provided") if storage_details.nil? or storage_details == ""
+      @storage_details = storage_details
+      @storage_handler = connect_to_database
+      @storage_handler.busy_timeout = 30000
+    end
+
+    def save_key_value_pair(serialized_key, serialized_value, timestamp = nil)
+      delete_entry(serialized_key)
+      time_entry = timestamp.nil? ? Time.now.to_s : timestamp.to_s
+      EH::retry!(:args => [serialized_key, serialized_value, time_entry]) do
+        @storage_handler.execute("INSERT INTO #{DB_TABLE} (key, value, timestamp) VALUES(?, ?, ?)",serialized_key, serialized_value, time_entry)
+      end
+    end
+
+    def lookup_key(serialized_key)
+      EH::retry!(:args => [serialized_key]) do
+        @storage_handler.execute("SELECT value, timestamp FROM #{DB_TABLE} WHERE key=?", serialized_key)
+      end
+    end
+
+    def delete_entry(serialized_key)
+      EH::retry!(:args => [serialized_key]) do
+        @storage_handler.execute("DELETE FROM #{DB_TABLE} WHERE key=?", serialized_key)
+      end
+    end
+
+    def size
+      EH::retry!(:args => []) do
+        @storage_handler.execute("SELECT value FROM #{DB_TABLE}").size
+      end
+    end
+
+    def keys
+      EH::retry!(:args => []) do
+        @storage_handler.execute("SELECT key FROM #{DB_TABLE}")
+      end
+    end
+
+    def clear
+      EH::retry!(:args => []) do
+        @storage_handler.execute("DELETE FROM #{DB_TABLE}")
+      end
+    end
+
+    private
+
+    def connect_to_database
+      File.exists?(@storage_details) ? open_database : create_database
+    end
+
+    def open_database
+      EH::retry!(:args => []) do
+        SQLite3::Database.open(@storage_details)
+      end
+    end
+
+    def create_database
+      EH::retry!(:args => []) do
+        @storage_handler = SQLite3::Database.new(@storage_details)
+        create_table
+      end
+      @storage_handler
+    end
+
+    def create_table
+      EH::retry!(:args => []) do
+        @storage_handler.execute("CREATE TABLE #{DB_TABLE}(key TEXT PRIMARY KEY, value TEXT, timestamp TEXT)")
+      end
+      @storage_handler
+    end
+  end
+end

data/lib/persistent-cache/version.rb

@@ -1,5 +1,5 @@
 module Persistent
   class Cache
-    VERSION = "0.0.8"
+    VERSION = "0.1.2"
   end
 end

data/persistent-cache.gemspec

@@ -15,5 +15,8 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Persistent::Cache::VERSION
   gem.add_development_dependency 'rspec', '2.12.0'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'simplecov-rcov'
   gem.add_dependency 'sqlite3', '1.3.7'
+  gem.add_dependency 'eh'
 end

data/spec/persistent-cache_spec.rb

@@ -1,74 +1,126 @@
 require 'spec_helper'
-require 'persistent-cache'
-require 'tempfile'
 
-describe Persistent::Cache do
-  def get_database_name
-    path = Tempfile.new("persistent-cache-spec-testdb").path
-    FileUtils.rm_f(path)
-    path
-  end
+require "persistent-cache"
+require "persistent-cache/storage/storage_sq_lite"
 
+describe Persistent::Cache do
   before :each do
     @db_name = get_database_name
+    @mock_storage = double(Persistent::StorageSQLite)
+    @test_key = "testkey"
+    @test_value = "testvalue"
+    FileUtils.rm_f(@db_name)
   end
 
+  context "when constructing" do
+    it "should receive database connection details and create a StorageSQLite instance if specified" do
+      @pcache = Persistent::Cache.new(@db_name, Persistent::Cache::STORAGE_SQLITE)
+      @pcache.class.should == Persistent::Cache
+      @pcache.storage.is_a?(Persistent::StorageSQLite).should == true
+    end
+
+    it "should raise an ArgumentError if storage details have not been provided" do
+      expect {
+        Persistent::Cache.new(nil)
+      }.to raise_error(ArgumentError)
+    end
+
+    it "should remember the freshness interval if provided" do
+      @pcache = Persistent::Cache.new(@db_name, 123)
+      @pcache.fresh.should == 123
+    end
+
+    it "should remember the storage details provided" do
+      @pcache = Persistent::Cache.new(@db_name, 123)
+      @pcache.storage_details.should == @db_name
+    end
+
+    it "should default the freshness interval to FRESH if not provided" do
+      @pcache = Persistent::Cache.new(@db_name)
+      @pcache.fresh.should == Persistent::Cache::FRESH
+    end
+
+    it "should raise an ArgumentError if an unknown storage type has been provided" do
+      expect {
+        Persistent::Cache.new(@db_name, 100, "unknown")
+      }.to raise_error(ArgumentError)
+    end
+  end
 
   context "When assigning a value to a key" do
-    it "should store the key/value pair in the db, with a timestamp" do
-      start_time = Time.now - 1
+    it "should ask the storage handler to first delete, then save the key/value pair" do
+      Persistent::StorageSQLite.should_receive(:new).and_return @mock_storage
+      @mock_storage.should_receive(:delete_entry)
+      @mock_storage.should_receive(:save_key_value_pair).with(Marshal.dump(@test_key), Marshal.dump(@test_value), nil)
       @pcache = Persistent::Cache.new(@db_name)
-      @pcache["testkey"] = "testvalue"
-      handle = SQLite3::Database.open(@db_name)
-      result = handle.execute "select value, timestamp from #{Persistent::Cache::DB_TABLE} where key=?", Marshal.dump("testkey")
-      result.nil?.should == false
-      result[0].nil?.should == false
-      result[0][0].should == Marshal.dump("testvalue")
-      test_time = Time.parse(result[0][1])
-      test_time.should > start_time and test_time.should < start_time + 600
+      @pcache[@test_key] = @test_value
     end
 
-    it "should overwrite the existing key/value pair if they already exist" do
+    it "should ask the storage handler to delete if the value is nil using []" do
+      Persistent::StorageSQLite.should_receive(:new).and_return @mock_storage
+      @mock_storage.should_receive(:delete_entry).with(Marshal.dump(@test_key))
       @pcache = Persistent::Cache.new(@db_name)
-      @pcache["testkey"] = "testvalue"
-      @pcache["testkey"] = "testvalue2"
-      handle = SQLite3::Database.open(@db_name)
-      result = handle.execute "select value from #{Persistent::Cache::DB_TABLE} where key=?", Marshal.dump("testkey")
-      result.nil?.should == false
-      result[0].nil?.should == false
-      result.size.should == 1
-      result[0][0].should == Marshal.dump("testvalue2")
+      @pcache[@test_key] = nil
+    end
+
+    it "should ask the storage handler to delete if the value is nil using set()" do
+      Persistent::StorageSQLite.should_receive(:new).and_return @mock_storage
+      @mock_storage.should_receive(:delete_entry).with(Marshal.dump(@test_key))
+      @pcache = Persistent::Cache.new(@db_name)
+      @pcache.set(@test_key, nil, Time.now)
     end
 
     it "should serialize the key and value for persistence" do
+      Persistent::StorageSQLite.should_receive(:new).and_return @mock_storage
+      @mock_storage.should_receive(:delete_entry)
+      @mock_storage.should_receive(:save_key_value_pair).with(Marshal.dump(@test_key), Marshal.dump(@test_value), nil)
       @pcache = Persistent::Cache.new(@db_name)
-      @pcache.should_receive(:save_key_value_pair).with(Marshal.dump("testkey"), Marshal.dump("testvalue"))
-      @pcache["testkey"] = "testvalue"
+      @pcache[@test_key] = @test_value
+    end
+
+    it "should ask the storage handler to store the value, with a specific timestamp if specified" do
+      Persistent::StorageSQLite.should_receive(:new).and_return @mock_storage
+      @mock_storage.should_receive(:delete_entry)
+      timestamp = Time.now - 100
+      @mock_storage.should_receive(:save_key_value_pair).with(Marshal.dump(@test_key), Marshal.dump(@test_value), timestamp)
+      @pcache = Persistent::Cache.new(@db_name)
+      @pcache.set(@test_key, @test_value, timestamp)
     end
   end
   
   context "When looking up a value given its key" do
-    it "should retrieve the value from the database and deserialize the value" do
+    it "should retrieve the value from storage using lookup_key and deserialize the value" do
+      @mock_storage.should_receive(:delete_entry)
+      @mock_storage.should_receive(:save_key_value_pair).with(Marshal.dump(@test_key), Marshal.dump(@test_value), nil)
+      @mock_storage.should_receive(:lookup_key).with(Marshal.dump(@test_key)).and_return([[Marshal.dump(@test_value), Time.now.to_s]])
+      Persistent::StorageSQLite.should_receive(:new).and_return @mock_storage
       @pcache = Persistent::Cache.new(@db_name)
-      @pcache["testkey"] = "testvalue2"
-      result = @pcache["testkey"]
-      result.should == "testvalue2"
+      @pcache[@test_key] = @test_value
+      result = @pcache[@test_key]
+      result.should == @test_value
     end
 
     it "should return nil if a value exists but it not fresh" do
-      fetch_stale_entry
+      @mock_storage.should_receive(:delete_entry)
+      @mock_storage.should_receive(:lookup_key).with(Marshal.dump(@test_key)).and_return([[Marshal.dump(@test_value), (Time.now - Persistent::Cache::FRESH).to_s]])
+      Persistent::StorageSQLite.should_receive(:new).and_return @mock_storage
+
+      @pcache = Persistent::Cache.new(@db_name)
+      @pcache[@test_key].nil?.should == true
     end
 
     it "should remove from the cache an entry it encounters that is not fresh" do
-      fetch_stale_entry
-      result = @handle.execute("SELECT value FROM #{Persistent::Cache::DB_TABLE} WHERE key=?", Marshal.dump("oldkey"))
-      result.size.should == 0
+      @mock_storage.should_receive(:delete_entry)
+      @mock_storage.should_receive(:lookup_key).with(Marshal.dump(@test_key)).and_return([[Marshal.dump(@test_value), (Time.now - Persistent::Cache::FRESH).to_s]])
+      Persistent::StorageSQLite.should_receive(:new).and_return @mock_storage
+
+      @pcache = Persistent::Cache.new(@db_name)
+      @pcache[@test_key]
     end
 
     it "should return nil if a key is not in the database" do
       @pcache = Persistent::Cache.new(@db_name)
-      @pcache["testkey"] = "testvalue2"
-      result = @pcache["testkey2"]
+      result = @pcache["thiskeydoesnotexist"]
       result.nil?.should == true
     end
 
@@ -78,53 +130,6 @@ describe Persistent::Cache do
       @pcache.should_receive(:lookup_key).with(Marshal.dump("testkey"))
       @pcache["testkey"]
     end
-
-    def fetch_stale_entry
-      @pcache = Persistent::Cache.new(@db_name)
-      timestamp = Time.now - Persistent::Cache::FRESH
-      @handle = SQLite3::Database.open(@db_name)
-      @handle.execute("INSERT INTO #{Persistent::Cache::DB_TABLE} (key, value, timestamp) VALUES ('#{Marshal.dump("oldkey")}', '#{Marshal.dump("oldvalue")}', '#{timestamp}')")
-      @pcache["oldkey"].nil?.should == true
-    end
-  end
-
-  context "when constructing" do
-    it "should receive database connection details" do
-      @pcache = Persistent::Cache.new(@db_name)
-      @pcache.class.should == Persistent::Cache
-      @pcache.database_details.should == @db_name
-    end
-
-    it "should create the database if it does not exist" do
-      FileUtils.rm_f(@db_name)
-      Persistent::Cache.new(@db_name)
-      File.exists?(@db_name).should be_true
-    end
-
-    it "should create a key_value table with key (TEXT) and value (TEXT) and timestamp (TEXT) columns" do
-      FileUtils.rm_f(@db_name)
-      Persistent::Cache.new(@db_name)
-      handle = SQLite3::Database.open(@db_name)
-      result = handle.execute "PRAGMA table_info(#{Persistent::Cache::DB_TABLE})"
-      result[0][1].should == "key"
-      result[0][2].should == "TEXT"
-      result[1][1].should == "value"
-      result[1][2].should == "TEXT"
-      result[2][1].should == "timestamp"
-      result[2][2].should == "TEXT"
-    end
-
-
-    it "should use the existing database if it does exist" do
-      FileUtils.rm_f(@db_name)
-      handle = SQLite3::Database.new(@db_name)
-      handle.execute "create table test123 ( id int );"
-      handle.close
-      Persistent::Cache.new(@db_name)
-      handle = SQLite3::Database.open(@db_name)
-      result = handle.execute "select name from sqlite_master where type='table'"
-      result[0][0].should == "test123"
-    end
   end
 
   context "it should behave like a cache" do
@@ -162,7 +167,7 @@ describe Persistent::Cache do
       100.times do |i|
         threads << Thread.new do
           Thread.current['pcache'] = Persistent::Cache.new("multidb") 
-          Thread.current['pcache']["multi_test"] += 1
+          Thread.current['pcache']["multi_test"] += 1 if not Thread.current['pcache'].nil? and not Thread.current['pcache']["multi_test"].nil?
         end
       end
       threads.each { |t| t.join }

data/spec/spec_helper.rb

@@ -1,5 +1,8 @@
 require 'rspec'
 require 'rspec/mocks'
+require 'tempfile'
+require 'simplecov'
+require 'simplecov-rcov'
 
 # This file was generated by the `rspec --init` command. Conventionally, all
 # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
@@ -21,3 +24,15 @@ RSpec.configure do |config|
   #     --seed 1234
   config.order = 'random'
 end
+
+SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
+SimpleCov.start do
+  add_filter "/spec/"
+end
+
+def get_database_name
+  path = Tempfile.new("persistent-cache-spec-testdb").path
+  FileUtils.rm_f(path)
+  path
+end
+

data/spec/storage/storage_directory_spec.rb

@@ -0,0 +1,264 @@
+require 'spec_helper'
+require "persistent-cache/storage/storage_directory"
+
+describe Persistent::StorageDirectory do
+  before :each do
+    @test_key = "testkey"
+    @test_value = "testvalue"
+    @db_name = get_database_name
+    @test_dir = "#{@db_name}/#{@test_key}"
+    @test_file = "#{@test_dir}/#{Persistent::StorageDirectory::CACHE_FILE}"
+    @test_data = "some data\nmoredata\n\n"
+
+    delete_database
+    @iut = Persistent::StorageDirectory.new(@db_name)
+  end
+
+  context "when constructed" do
+    it "should create the database if it does not exist" do
+      File.exists?(@db_name).should be_true
+    end
+
+    it "should propagate errors that are raised when failing to create a database" do
+      delete_database
+      FileUtils.should_receive(:makedirs).and_raise RuntimeError.new("testing")
+      expect {
+        @iut = Persistent::StorageDirectory.new(@db_name)
+      }.to raise_error RuntimeError
+    end
+
+    it "should use the existing database if it does exist" do
+      delete_database
+      FileUtils.makedirs([@db_name])
+      test_file = "#{@db_name}/hello"
+      `touch #{test_file}`
+      Persistent::StorageDirectory.new(@db_name)
+      File.exist?(test_file).should == true
+    end
+
+    it "should have a database" do
+      @iut.is_a?(Persistent::StorageDirectory).should == true
+    end
+
+    it "should raise an ArgumentError if storage details have not been provided" do
+      expect {
+        Persistent::StorageDirectory.new(nil)
+      }.to raise_error(ArgumentError)
+    end
+  end
+
+  context "when asked to store a key value pair" do
+    it "should create a directory named the same as the key, in the storage root, if that directory does not exist, and store the value in a file called CACHE_FILE" do
+      setup_test_entry
+      read_file_content(@test_file).should == @test_data
+    end
+
+    it "should default to storing the current time as the first line of the catalogue" do
+      now = Time.now
+      FileUtils.rm_f(@test_dir)
+      @iut.save_key_value_pair(@test_key, @test_data)
+      read_file_timestamp(@test_file) == now.to_s
+    end
+
+    it "should store a time specified as the first line of the catalogue" do
+      time = Time.now - 2500
+      FileUtils.rm_f(@test_dir)
+      @iut.save_key_value_pair(@test_key, @test_data, time)
+      read_file_timestamp(@test_file).should == time.to_s
+    end
+
+    it "should overwrite the existing key/value pair if they already exist" do
+      FileUtils.rm_f(@test_dir)
+      @iut.save_key_value_pair(@test_key, "old data")
+      @iut.save_key_value_pair(@test_key, @test_data)
+      File.exists?(@test_dir).should == true
+      File.exists?(@test_file).should == true
+      read_file_content(@test_file).should == @test_data
+    end
+
+    it "should raise an ArgumentError if the key is not a string" do
+      expect {
+        @iut.save_key_value_pair(1234, @test_data)
+      }.to raise_error ArgumentError
+    end
+
+    it "should raise an ArgumentError if the value is not a string" do
+      expect {
+        @iut.save_key_value_pair(@test_key, 1234)
+      }.to raise_error ArgumentError
+    end
+
+    it "should store the value exactly as given, regardless of newlines" do
+      @iut.save_key_value_pair(@test_key, "some data")
+      @iut.lookup_key(@test_key)[0][0].should == "some data"
+
+      @iut.save_key_value_pair(@test_key, "some data\n")
+      @iut.lookup_key(@test_key)[0][0].should == "some data\n"
+
+      @iut.save_key_value_pair(@test_key, "some data\n\n")
+      @iut.lookup_key(@test_key)[0][0].should == "some data\n\n"
+
+      @iut.save_key_value_pair(@test_key, "\nline 1\n23456\n\n\nsome data\n")
+      @iut.lookup_key(@test_key)[0][0].should == "\nline 1\n23456\n\n\nsome data\n"
+    end
+  end
+
+  context "When looking up a value given its key" do
+    it "should retrieve the contents of the catalogue file from the database, excluding the timestamp" do
+      setup_test_entry
+      result = @iut.lookup_key(@test_key)
+      result[0][0].should == @test_data
+    end
+
+    it "should retrieve the timestamp of the revision from the database" do
+      now = Time.now
+      setup_test_entry
+      result = @iut.lookup_key(@test_key)
+      result[0][1].should == now.to_s
+    end
+
+    it "should return an empty array if a key is not in the database" do
+      setup_test_entry
+      result = @iut.lookup_key("thiskeyshouldnotexist")
+      result.should == []
+    end
+
+    it "should raise an ArgumentError if the key is not a string" do
+      expect {
+        @iut.lookup_key(1234)
+      }.to raise_error ArgumentError
+    end
+  end
+
+  context "when asked to delete an entry" do
+    it "should not raise an error if the directory that results from the hash is not present" do
+      @iut.delete_entry("thiskeyshouldnotexist")
+    end
+
+    it "should delete the directory that results from the hash if it is present" do
+      setup_test_entry
+      @iut.delete_entry(@test_key)
+      @iut.lookup_key(@test_key).should == []
+      File.exists?(@test_dir).should == false
+      File.exists?(@test_file).should == false
+    end
+
+    it "should raise an ArgumentError if the key is not a string" do
+      expect {
+        @iut.delete_entry(1234)
+      }.to raise_error ArgumentError
+    end
+  end
+
+  context "when asked the size of the database" do
+    it "should return 0 if the database has no entries" do
+      @iut.size.should == 0
+    end
+
+    it "should return the number of entries" do
+      populate_database
+      @iut.size.should == 3
+    end
+
+    it "should return 0 if the database does not exist" do
+      delete_database
+      @iut.size.should == 0
+    end
+  end
+
+  context "when asked for the keys in the database" do
+    it "should return an empty array if there are no entries in the database" do
+      @iut.keys.should == []
+    end
+
+    it "should return the keys (directories) in the database" do
+      populate_database
+      @iut.keys.should == [["one"], ["three"], ["two"]]
+    end
+
+    it "should return the keys in a sorted array" do
+      populate_database
+      @iut.keys.should == [["one"], ["three"], ["two"]]
+    end
+
+    it "should not return the storage root itself" do
+      populate_database
+      @iut.keys.each do |key|
+        (key == "").should == false
+        (key == "/").should == false
+      end
+    end
+
+    it "should return the keys in an array, with each key in its own sub-array" do
+      populate_database
+      @iut.keys.is_a?(Array).should == true
+      @iut.keys[0].is_a?(Array).should == true
+      @iut.keys[0][0].is_a?(String).should == true
+    end
+  end
+
+  context "when asked to clear the database" do
+    it "should not delete the database root directory" do
+      setup_test_entry
+      @iut.clear
+      File.exists?(@test_file).should == false
+      File.exists?(@test_dir).should == false
+      File.exist?(@iut.storage_root).should == true
+    end
+
+    it "should delete all directories in the database" do
+      populate_database
+      @iut.clear
+      @iut.size.should == 0
+    end
+  end
+
+  context "when asked about the path to a key's cache value file" do
+    it "should return nil if the key is not in the cache" do
+      @iut.get_value_path(@test_key).should == nil
+    end
+
+    it "should return the path to the key's cache value file if the key is in the cache" do
+      @iut.save_key_value_pair(@test_key, @test_data)
+      @iut.get_value_path(@test_key).should == @test_file
+    end
+
+    it "should raise an ArgumentError if the key is not a string" do
+      expect {
+        @iut.get_value_path(123)
+      }.to raise_error ArgumentError
+    end
+  end
+
+  def populate_database
+    @iut.save_key_value_pair("one", "one")
+    @iut.save_key_value_pair("two", "two")
+    @iut.save_key_value_pair("three", "three")
+  end
+
+  def setup_test_entry
+    FileUtils.rm_f(@test_dir)
+    @iut.save_key_value_pair(@test_key, @test_data)
+    File.exists?(@test_dir).should == true
+    File.exists?(@test_file).should == true
+  end
+
+  def delete_database
+    FileUtils.rm_rf(@db_name)
+    File.exists?(@db_name).should == false
+  end
+
+  def read_file_data(file)
+    File.read(file).split("\n")
+  end
+
+  def read_file_timestamp(file)
+    result = File.read(file)
+    result.lines.to_a[0..0].join.split("\n")[0]
+  end
+
+  def read_file_content(file)
+    result = File.read(file)
+    result.lines.to_a[1..-1].join
+  end
+end

data/spec/storage/storage_sqlite_spec.rb

@@ -0,0 +1,193 @@
+require 'spec_helper'
+require "persistent-cache/storage/storage_sq_lite"
+
+describe Persistent::StorageSQLite do
+  before :each do
+    @db_name = get_database_name
+    delete_database
+    @test_key = "testkey"
+    @test_value = "testvalue"
+    @iut = Persistent::StorageSQLite.new(@db_name)
+ end
+
+  context "when constructed" do
+    it "should create the database if it does not exist" do
+      File.exists?(@db_name).should be_true
+    end
+
+    it "should create a key_value table with key (TEXT) and value (TEXT) and timestamp (TEXT) columns" do
+      handle = SQLite3::Database.open(@db_name)
+      result = handle.execute "PRAGMA table_info(#{Persistent::StorageSQLite::DB_TABLE})"
+      result[0][1].should == "key"
+      result[0][2].should == "TEXT"
+      result[1][1].should == "value"
+      result[1][2].should == "TEXT"
+      result[2][1].should == "timestamp"
+      result[2][2].should == "TEXT"
+    end
+
+
+    it "should use the existing database if it does exist" do
+      delete_database
+      handle = SQLite3::Database.new(@db_name)
+      handle.execute "create table test123 ( id int );"
+      handle.close
+      Persistent::StorageSQLite.new(@db_name)
+      handle = SQLite3::Database.open(@db_name)
+      result = handle.execute "select name from sqlite_master where type='table'"
+      result[0][0].should == "test123"
+    end
+
+    it "should have a database handler" do
+      @iut.storage_handler.is_a?(SQLite3::Database).should == true
+    end
+
+    it "should set the SQLite busy timeout to DB_TIMEOUT" do
+      delete_database
+      mock_database = double(SQLite3::Database)
+      mock_database.should_receive(:execute).at_least(0)
+      mock_database.should_receive(:busy_timeout=).with(Persistent::StorageSQLite::DB_TIMEOUT)
+      SQLite3::Database.should_receive(:new).and_return(mock_database)
+      Persistent::StorageSQLite.new(@db_name)
+    end
+
+    it "should raise an ArgumentError if storage details have not been provided" do
+      expect {
+        Persistent::StorageSQLite.new(nil)
+      }.to raise_error(ArgumentError)
+    end
+  end
+
+  context "when asked to store a key value pair" do
+    it "should store the key/value pair in the db, with the current time as timestamp" do
+      start_time = Time.now - 1
+      @iut.save_key_value_pair(Marshal.dump(@test_key), Marshal.dump(@test_value))
+      handle = SQLite3::Database.open(@db_name)
+      result = handle.execute "select value, timestamp from #{Persistent::StorageSQLite::DB_TABLE} where key=?", Marshal.dump(@test_key)
+      result.nil?.should == false
+      result[0].nil?.should == false
+      result[0][0].should == Marshal.dump(@test_value)
+      test_time = Time.parse(result[0][1])
+      test_time.should > start_time and test_time.should < start_time + 600
+    end
+
+    it "should store the key/value pair in the db, with a timestamp specified" do
+      test_time = (Time.now - 2500)
+      @iut.save_key_value_pair(Marshal.dump(@test_key), Marshal.dump(@test_value), test_time)
+      handle = SQLite3::Database.open(@db_name)
+      result = handle.execute "select value, timestamp from #{Persistent::StorageSQLite::DB_TABLE} where key=?", Marshal.dump(@test_key)
+      result.nil?.should == false
+      result[0].nil?.should == false
+      result[0][0].should == Marshal.dump(@test_value)
+      time_retrieved = Time.parse(result[0][1])
+      time_retrieved.to_s.should == test_time.to_s
+    end
+
+    it "should overwrite the existing key/value pair if they already exist" do
+      @iut.save_key_value_pair(Marshal.dump(@test_key), Marshal.dump(@test_value))
+      @iut.save_key_value_pair(Marshal.dump(@test_key), Marshal.dump("testvalue2"))
+      handle = SQLite3::Database.open(@db_name)
+      result = handle.execute "select value from #{Persistent::StorageSQLite::DB_TABLE} where key=?", Marshal.dump(@test_key)
+      result.nil?.should == false
+      result[0].nil?.should == false
+      result.size.should == 1
+      result[0][0].should == Marshal.dump("testvalue2")
+    end
+  end
+
+  context "When looking up a value given its key" do
+    it "should retrieve the value from the database" do
+      @iut.save_key_value_pair(Marshal.dump(@test_key), Marshal.dump(@test_value))
+      result = @iut.lookup_key(Marshal.dump(@test_key))
+      result[0][0].should == Marshal.dump(@test_value)
+    end
+
+    it "should retrieve the timestamp when the value was stored from the database" do
+      now = Time.now.to_s
+      @iut.save_key_value_pair(Marshal.dump(@test_key), Marshal.dump(@test_value))
+      sleep 1
+      result = @iut.lookup_key(Marshal.dump(@test_key))
+      result[0][1].should == now
+    end
+
+    it "should return an empty array if a key is not in the database" do
+      @iut.delete_entry(Marshal.dump(@test_key))
+      result = @iut.lookup_key(Marshal.dump(@test_key))
+      result.should == []
+      result[0].should == nil
+    end
+  end
+
+  context "when asked to delete an entry" do
+    it "should not raise an error if the entry is not present" do
+      @iut.delete_entry(Marshal.dump("shouldnotbepresent"))
+    end
+
+    it "should delete the entry if it is present" do
+      @iut.save_key_value_pair(Marshal.dump(@test_key), Marshal.dump(@test_value))
+      result = @iut.lookup_key(Marshal.dump(@test_key))
+      result[0][0].should == Marshal.dump(@test_value)
+      @iut.delete_entry(Marshal.dump(@test_key))
+      result = @iut.lookup_key(Marshal.dump(@test_key))
+      result.should == []
+    end
+  end
+
+  context "when asked the size of the database" do
+    it "should return 0 if the database has no entries" do
+      @iut.size.should == 0
+    end
+
+    it "should return the number of entries" do
+      populate_database(@iut)
+      @iut.size.should == 3
+    end
+  end
+
+  context "when asked for the keys in the database" do
+    it "should return an empty array if there are no entries in the database" do
+      @iut.keys.should == []
+    end
+
+    it "should return the keys in the database" do
+      populate_database(@iut)
+      keys = @iut.keys.flatten
+      keys.include?(Marshal.dump("one")).should == true
+      keys.include?(Marshal.dump("two")).should == true
+      keys.include?(Marshal.dump("three")).should == true
+      @iut.size.should == 3
+    end
+
+    it "should return the keys in an array, with each key in its own sub-array" do
+      populate_database(@iut)
+      found = false
+      test = Marshal.dump("one")
+      found = true if (@iut.keys[0][0] == test or @iut.keys[0][1] == test or @iut.keys[0][2] == test)
+      found.should == true
+    end
+  end
+
+  context "when asked to clear the database" do
+    it "should not delete the database file" do
+      populate_database(@iut)
+      @iut.clear
+      File.exists?(@db_name).should be_true
+    end
+
+    it "should delete all entries in the database" do
+      populate_database(@iut)
+      @iut.clear
+      @iut.size.should == 0
+    end
+  end
+
+  def populate_database(iut)
+    iut.save_key_value_pair(Marshal.dump("one"), Marshal.dump("one"))
+    iut.save_key_value_pair(Marshal.dump("two"), Marshal.dump("two"))
+    iut.save_key_value_pair(Marshal.dump("three"), Marshal.dump("three"))
+  end
+
+  def delete_database
+    FileUtils.rm_f(@db_name)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: persistent-cache
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-08 00:00:00.000000000 Z
+date: 2013-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -28,6 +28,38 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.12.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-rcov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -44,6 +76,22 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.3.7
+- !ruby/object:Gem::Dependency
+  name: eh
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Persistent Cache using SQLite
 email:
 - wvd@hetzner.co.za
@@ -60,11 +108,15 @@ files:
 - README.md
 - Rakefile
 - lib/persistent-cache.rb
+- lib/persistent-cache/storage/storage_directory.rb
+- lib/persistent-cache/storage/storage_sq_lite.rb
 - lib/persistent-cache/version.rb
 - multidb
 - persistent-cache.gemspec
 - spec/persistent-cache_spec.rb
 - spec/spec_helper.rb
+- spec/storage/storage_directory_spec.rb
+- spec/storage/storage_sqlite_spec.rb
 homepage: ''
 licenses: []
 post_install_message: 
@@ -93,3 +145,5 @@ summary: Persistent Cache has a default freshness threshold of 179 days after wh
 test_files:
 - spec/persistent-cache_spec.rb
 - spec/spec_helper.rb
+- spec/storage/storage_directory_spec.rb
+- spec/storage/storage_sqlite_spec.rb