sutch-anemone 0.7.2

data/lib/anemone/storage/base.rb

@@ -0,0 +1,75 @@
+require 'anemone/storage/exceptions'
+
+module Anemone
+  module Storage
+    class Base
+
+      def initialize(adapter)
+        @adap = adapter
+
+        # verify adapter conforms to this class's methods
+        methods.each do |method|
+          if !@adap.respond_to?(method.to_sym)
+            raise "Storage adapter must support method #{method}"
+          end
+        end
+      end
+
+      def [](key)
+        @adap[key]
+        rescue
+          puts key
+          raise RetrievalError, $! 
+      end
+
+      def []=(key, value)
+        @adap[key] = value
+        rescue
+          raise InsertionError, $!
+      end
+
+      def delete(key)
+        @adap.delete(key)
+        rescue
+          raise DeletionError, $!
+      end
+
+      def each
+        @adap.each { |k, v| yield k, v }
+        rescue
+          raise GenericError, $!
+      end
+
+      def merge!(hash)
+        @adap.merge!(hash)
+        rescue
+          raise GenericError, $!
+      end
+
+      def close
+        @adap.close
+        rescue
+          raise CloseError, $!
+      end
+
+      def size
+        @adap.size
+        rescue
+          raise GenericError, $!
+      end
+
+      def keys
+        @adap.keys
+        rescue
+          raise GenericError, $!
+      end
+
+      def has_key?(key)
+        @adap.has_key?(key)
+        rescue
+          raise GenericError, $!
+      end
+
+    end
+  end
+end

data/lib/anemone/storage/exceptions.rb

@@ -0,0 +1,15 @@
+module Anemone
+  module Storage
+
+    class GenericError < Error; end;
+
+    class ConnectionError < Error; end
+
+    class RetrievalError < Error; end
+
+    class InsertionError < Error; end
+
+    class CloseError < Error; end
+
+  end
+end

data/lib/anemone/storage/kyoto_cabinet.rb

@@ -0,0 +1,72 @@
+begin
+  require 'kyotocabinet'
+rescue LoadError
+  puts $!
+  puts "You need the kyotocabinet-ruby gem to use Anemone::Storage::KyotoCabinet"
+  exit
+end
+
+require 'forwardable'
+
+module Anemone
+  module Storage
+    class KyotoCabinet
+      extend Forwardable
+
+      def_delegators :@db, :close, :size, :each
+
+      def initialize(file)
+        raise "KyotoCabinet filename must have .kch extension" if File.extname(file) != '.kch'
+        @db = ::KyotoCabinet::DB::new
+        @db.open(file, ::KyotoCabinet::DB::OWRITER | ::KyotoCabinet::DB::OCREATE)
+        @db.clear
+      end
+
+      def [](key)
+        if value = @db[key]
+          load_value(value)
+        end
+      end
+
+      def []=(key, value)
+        @db[key] = [Marshal.dump(value)].pack("m")
+      end
+
+      def each
+        @db.each do |k, v|
+          yield(k, load_value(v))
+        end
+      end
+
+      def has_key?(key)
+        # Kyoto Cabinet doesn't have a way to query whether a key exists, so hack it
+        keys = @db.match_prefix(key)
+        !!keys && keys.include?(key)
+      end
+
+      def keys
+        acc = []
+        @db.each_key { |key| acc << key.first }
+        acc
+      end
+
+      def delete(key)
+        value = self[key]
+        @db.delete(key)
+        value
+      end
+
+      def merge!(hash)
+        hash.each { |key, value| self[key] = value }
+        self
+      end
+
+      private
+
+      def load_value(value)
+        Marshal.load(value.unpack("m")[0])
+      end
+
+    end
+  end
+end

data/lib/anemone/storage/mongodb.rb

@@ -0,0 +1,89 @@
+begin
+  require 'mongo'
+rescue LoadError
+  puts "You need the mongo gem to use Anemone::Storage::MongoDB"
+  exit
+end
+
+module Anemone
+  module Storage
+    class MongoDB 
+
+      BINARY_FIELDS = %w(body headers data)
+
+      def initialize(mongo_db, collection_name)
+        @db = mongo_db
+        @collection = @db[collection_name]
+        @collection.remove
+        @collection.create_index 'url'
+      end
+
+      def [](url)
+        if value = @collection.find_one('url' => url.to_s)
+          load_page(value)
+        end
+      end
+
+      def []=(url, page)
+        hash = page.to_hash
+        BINARY_FIELDS.each do |field|
+          hash[field] = BSON::Binary.new(hash[field]) unless hash[field].nil?
+        end
+        @collection.update(
+          {'url' => page.url.to_s},
+          hash,
+          :upsert => true
+        )
+      end
+
+      def delete(url)
+        page = self[url]
+        @collection.remove('url' => url.to_s)
+        page
+      end
+
+      def each
+        @collection.find do |cursor|
+          cursor.each do |doc|
+            page = load_page(doc)
+            yield page.url.to_s, page 
+          end
+        end
+      end
+
+      def merge!(hash)
+        hash.each { |key, value| self[key] = value }
+        self
+      end
+
+      def size
+        @collection.count
+      end
+
+      def keys
+        keys = []
+        self.each { |k, v| keys << k.to_s }
+        keys
+      end
+
+      def has_key?(url)
+        !!@collection.find_one('url' => url.to_s)
+      end
+
+      def close
+        @db.connection.close
+      end
+
+      private
+
+      def load_page(hash)
+        BINARY_FIELDS.each do |field|
+          hash[field] = hash[field].to_s
+        end
+        Page.from_hash(hash)
+      end
+
+    end
+  end
+end
+

data/lib/anemone/storage/pstore.rb

@@ -0,0 +1,50 @@
+require 'pstore'
+require 'forwardable'
+
+module Anemone
+  module Storage
+    class PStore
+      extend Forwardable
+
+      def_delegators :@keys, :has_key?, :keys, :size
+
+      def initialize(file)
+        File.delete(file) if File.exists?(file)
+        @store = ::PStore.new(file)
+        @keys = {}
+      end
+
+      def [](key)
+        @store.transaction { |s| s[key] }
+      end
+
+      def []=(key,value)
+        @keys[key] = nil
+        @store.transaction { |s| s[key] = value }
+      end
+
+      def delete(key)
+        @keys.delete(key)
+        @store.transaction { |s| s.delete key}
+      end
+
+      def each
+        @keys.each_key do |key|
+          value = nil
+          @store.transaction { |s| value = s[key] }
+          yield key, value
+        end
+      end
+
+      def merge!(hash)
+        @store.transaction do |s|
+          hash.each { |key, value| s[key] = value; @keys[key] = nil }
+        end
+        self
+      end
+
+      def close; end
+
+    end
+  end
+end

data/lib/anemone/storage/redis.rb

@@ -0,0 +1,90 @@
+require 'redis'
+
+module Anemone
+  module Storage
+    class Redis
+
+      MARSHAL_FIELDS = %w(links visited fetched)
+
+      def initialize(opts = {})
+        @redis = ::Redis.new(opts)
+        @key_prefix = opts[:key_prefix] || 'anemone'
+        keys.each { |key| delete(key) }
+      end
+
+      def [](key)
+        rkey = "#{@key_prefix}:pages:#{key.to_s}"
+        rget(rkey)
+      end
+
+      def []=(key, value)
+        rkey = "#{@key_prefix}:pages:#{key.to_s}"
+        hash = value.to_hash
+        MARSHAL_FIELDS.each do |field|
+          hash[field] = Marshal.dump(hash[field])
+        end
+        hash.each do |field, value|
+          @redis.hset(rkey, field, value)
+        end
+      end
+
+      def delete(key)
+        rkey = "#{@key_prefix}:pages:#{key.to_s}"
+        page = self[key]
+        @redis.del(rkey)
+        page
+      end
+
+      def each
+        rkeys = @redis.keys("#{@key_prefix}:pages:*")
+        rkeys.each do |rkey|
+          page = rget(rkey)
+          yield page.url.to_s, page
+        end
+      end
+
+      def merge!(hash)
+        hash.each { |key, value| self[key] = value }
+        self
+      end
+
+      def size
+        @redis.keys("#{@key_prefix}:pages:*").size
+      end
+
+      def keys
+        keys = []
+        self.each { |k, v| keys << k.to_s }
+        keys
+      end
+
+      def has_key?(key)
+        rkey = "#{@key_prefix}:pages:#{key.to_s}"
+        @redis.exists(rkey)
+      end
+
+      def close
+        @redis.quit
+      end
+
+      private
+
+      def load_value(hash)
+        MARSHAL_FIELDS.each do |field|
+          unless hash[field].nil? || hash[field] == ''
+            hash[field] = Marshal.load(hash[field]) 
+          end
+        end
+        Page.from_hash(hash)
+      end
+
+      def rget(rkey)
+        hash = @redis.hgetall(rkey)
+        if !!hash
+          load_value(hash)
+        end
+      end
+
+    end
+  end
+end

data/lib/anemone/storage/sqlite3.rb

@@ -0,0 +1,90 @@
+begin
+  require 'sqlite3'
+rescue LoadError
+  puts "You need the sqlite3 gem to use Anemone::Storage::SQLite3"
+  exit
+end
+
+module Anemone
+  module Storage
+    class SQLite3 
+
+      def initialize(file)
+        @db = ::SQLite3::Database.new(file)
+        create_schema
+      end
+
+      def [](url)
+        value = @db.get_first_value('SELECT data FROM anemone_storage WHERE key = ?', url.to_s)
+        if value
+          Marshal.load(value)
+        end
+      end
+
+      def []=(url, value)
+        data = Marshal.dump(value)
+        if has_key?(url)
+          @db.execute('UPDATE anemone_storage SET data = ? WHERE key = ?', data, url.to_s)
+        else
+          @db.execute('INSERT INTO anemone_storage (data, key) VALUES(?, ?)', data, url.to_s)
+        end
+      end
+
+      def delete(url)
+        page = self[url]
+        @db.execute('DELETE FROM anemone_storage WHERE key = ?', url.to_s)
+        page
+      end
+
+      def each
+        @db.execute("SELECT key, data FROM anemone_storage ORDER BY id") do |row|
+          value = Marshal.load(row[1])
+          yield row[0], value
+        end
+      end
+
+      def merge!(hash)
+        hash.each { |key, value| self[key] = value }
+        self
+      end
+
+      def size
+        @db.get_first_value('SELECT COUNT(id) FROM anemone_storage')
+      end
+
+      def keys
+        @db.execute("SELECT key FROM anemone_storage ORDER BY id").map{|t| t[0]}
+      end
+
+      def has_key?(url)
+        !!@db.get_first_value('SELECT id FROM anemone_storage WHERE key = ?', url.to_s)
+      end
+
+      def close
+        @db.close
+      end
+
+      private
+      
+      def create_schema
+        @db.execute_batch <<SQL
+          create table if not exists anemone_storage (
+            id INTEGER PRIMARY KEY ASC,
+            key TEXT,
+            data BLOB
+          );
+          create index  if not exists anemone_key_idx on anemone_storage (key);
+SQL
+      end
+
+      def load_page(hash)
+        BINARY_FIELDS.each do |field|
+          hash[field] = hash[field].to_s
+        end
+        Page.from_hash(hash)
+      end
+
+    end
+  end
+end
+