monkeyshines 0.0.2

data/lib/monkeyshines/store/conditional_store.rb

@@ -0,0 +1,57 @@
+module Monkeyshines
+  module Store
+    class ConditionalStore < Monkeyshines::Store::Base
+      attr_accessor :options, :cache, :store, :misses
+
+      DEFAULT_OPTIONS = {
+        :cache => { :type => :tyrant_rdb_key_store    },
+        :store => { :type => :chunked_flat_file_store },
+      }
+
+      #
+      #
+      # +cache+ must behave like a hash (Hash and
+      #  Monkeyshines::Store::TyrantRdbKeyStore are both cromulent
+      #  choices).
+      #
+      #
+      #
+      def initialize _options
+        self.options = DEFAULT_OPTIONS.deep_merge(_options)
+        self.cache  = Monkeyshines::Store.create(options[:cache])
+        self.store  = Monkeyshines::Store.create(options[:store])
+        self.misses = 0
+      end
+
+      #
+      # If key is absent, save the result of calling the block.
+      # If key is present, block is never called.
+      #
+      # Ex:
+      #   rt_store.set(url) do
+      #     fetcher.get url # will only be called if url isn't in rt_store
+      #   end
+      #
+      def set key, force=nil, &block
+        return if (!force) && cache.include?(key)
+        cache_val, store_val = block.call()
+        return unless cache_val
+        cache.set_nr key, cache_val # update cache
+        store << store_val          # save value
+        self.misses += 1            # track the cache miss
+        store_val
+      end
+
+      def size() cache.size  end
+
+      def log_line
+        [size, "%8d misses"%misses]
+      end
+
+      def close()
+        cache.close
+        store.close
+      end
+    end
+  end
+end

data/lib/monkeyshines/store/factory.rb

@@ -0,0 +1,8 @@
+module Monkeyshines
+  module Store
+    class Factory
+      def self.generate type, opts
+      end
+    end
+  end
+end

data/lib/monkeyshines/store/flat_file_store.rb

@@ -0,0 +1,84 @@
+require 'fileutils'; include FileUtils
+
+module Monkeyshines
+  module Store
+    #
+    class FlatFileStore < Store::Base
+      attr_accessor :filename, :filemode
+
+      #
+      # +filename_root+  : first part of name for files
+      #
+      def initialize options={}
+        Log.debug "New #{self.class} as #{options.inspect}"
+        self.filename = options[:filename] or raise "Missing filename in #{self.class}"
+        self.filemode = options[:filemode] || 'r'
+        skip!(options[:skip]) if options[:skip]
+      end
+
+      #
+      #
+      #
+      def each &block
+        file.each do |line|
+          next if line[0..0] == '#'
+          attrs = line.chomp.split("\t")
+          next if attrs.blank?
+          yield *attrs
+        end
+      end
+
+      #
+      # Read ahead n_lines lines in the file
+      #
+      def skip! n_lines
+        Log.info "Skipping #{n_lines} in #{self.class}:#{filename}"
+        n_lines.times do
+          file.readline
+        end
+      end
+
+      #
+      # Open the timestamped file,
+      # ensuring its directory exists
+      #
+      def file
+        return @file if @file
+        Log.info "Opening file #{filename} with mode #{filemode}"
+        @file = File.open(filename, filemode)
+      end
+
+      # Close the dump file
+      def close
+        @file.close if @file
+        @file = nil
+      end
+
+      # Ensure the file's directory exists
+      def mkdir!
+        dir = File.dirname(filename)
+        return if File.directory?(dir)
+        Log.info "Making directory #{dir}"
+        FileUtils.mkdir_p dir
+      end
+
+      # write to the file
+      def save obj
+        file << obj.to_flat.join("\t")+"\n"
+        obj
+      end
+
+      def set key, *args, &block
+        tok, obj = block.call
+        save obj
+      end
+
+      # delegates to +#save+ -- writes the object to the file
+      def <<(obj)
+        save obj
+      end
+
+    end
+  end
+end
+

data/lib/monkeyshines/store/key_store.rb

@@ -0,0 +1,51 @@
+module Monkeyshines
+  module Store
+    class KeyStore < Monkeyshines::Store::Base
+      # The actual backing store; should respond to #set and #get methods
+      attr_accessor :db
+
+      #
+      # Executes block once for each element in the whole DB, in whatever order
+      # the DB thinks you should see it.
+      #
+      # Your block will see |key, val|
+      #
+      # key_store.each do |key, val|
+      #   # ... stuff ...
+      # end
+      #
+      def each &block
+        db.iterinit
+        loop do
+          key = db.iternext or break
+          val = db[key]
+          yield key, val
+        end
+      end
+
+
+      # Save the value into the database
+      def set(key, val)
+        return unless val
+        db[key] = val
+      end
+
+      alias_method :save, :set
+      def get(key)      db[key]  end
+      def [](key)       db[key]  end
+      def close()       db.close end
+      def size()        db.size  end
+
+      #
+      # Load from standard command-line options
+      #
+      # obvs only works when there's just one store
+      #
+      def self.new_from_command_line cmdline_opts, default_opts={}
+        options = default_opts.merge(cmdline_opts)
+        store = self.new(options[:store_db])
+        store
+      end
+    end
+  end
+end

data/lib/monkeyshines/store/null_store.rb

@@ -0,0 +1,15 @@
+module Monkeyshines
+  module Store
+    class NullStore < Monkeyshines::Store::Base
+
+      def each *args, &block
+      end
+
+
+      # Does nothing!
+      def set *args
+      end
+
+    end
+  end
+end

data/lib/monkeyshines/store/read_thru_store.rb

@@ -0,0 +1,22 @@
+module Monkeyshines
+  module Store
+    class ReadThruStore < Monkeyshines::Store::TyrantTdbKeyStore
+
+      #
+      # If key is absent, save the result of calling the block.
+      # If key is present, block is never called.
+      #
+      # Ex:
+      #   rt_store.set(url) do
+      #     fetcher.get url # will only be called if url isn't in rt_store
+      #   end
+      #
+      def set key, force=nil, &block
+        return if !force && db.has_key?(key)
+        result = block.call() or return
+        super(key, result)
+      end
+
+    end
+  end
+end

data/lib/monkeyshines/store/tokyo_tdb_key_store.rb

@@ -0,0 +1,33 @@
+require 'tokyocabinet'
+module Monkeyshines
+  module Store
+    #
+    # Implementation of KeyStore with a Local TokyoCabinet table database (TDB)
+    #
+    class TokyoTdbKeyStore < Monkeyshines::Store::KeyStore
+
+      # pass in the filename or URI of a tokyo cabinet table-style DB
+      # set create_db = true if you want to create a missing DB file
+      def initialize db_uri, *args
+        self.db = TokyoCabinet::TDB.new
+        db.open(db_uri, TokyoCabinet::TDB::OWRITER) or raise "#{self.class.to_s}: Can't open TokyoCabinet TDB #{db_uri}"
+        super *args
+      end
+
+
+      def each_as klass, &block
+        self.each do |key, hsh|
+          yield klass.from_hash hsh
+        end
+      end
+      # Delegate to store
+      def set(key, val)
+        return unless val
+        db.put key, val.to_hash.compact
+      end
+
+      def size()        db.rnum  end
+
+    end #class
+  end
+end

data/lib/monkeyshines/store/tyrant_rdb_key_store.rb

@@ -0,0 +1,56 @@
+require 'tokyotyrant'
+module Monkeyshines
+  module Store
+
+    #
+    # Implementation of KeyStore with a Local TokyoCabinet hash database (RDB)
+    #
+    class TyrantRdbKeyStore < Monkeyshines::Store::KeyStore
+      attr_accessor :db_host, :db_port
+
+      # pass in the host:port uri of the key store.
+      def initialize options
+        raise "URI for #{self.class} is required" if options[:uri].blank?
+        self.db_host, self.db_port = options[:uri].to_s.split(':')
+        super options
+      end
+
+      def db
+        return @db if @db
+        @db ||= TokyoTyrant::RDB.new
+        @db.open(db_host, db_port) or raise("Can't open DB #{db_host}:#{db_port}. Pass in host:port' #{@db.ecode}: #{@db.errmsg(@db.ecode)}")
+        @db
+      end
+
+      def close
+        @db.close if @db
+        @db = nil
+      end
+
+      # Save the value into the database without waiting for a response.
+      def set_nr(key, val)
+        db.putnr key, val if val
+      end
+
+      def size()        db.rnum  end
+      def include? *args
+        db.has_key? *args
+      end
+
+      # require 'memcache'
+      # def initialize db_uri=nil, *args
+      #   # db_uri ||= ':1978'
+      #   # self.db_host, self.db_port = db_uri.split(':')
+      #   self.db = MemCache.new(db_uri, :no_reply => true)
+      #   if !self.db then raise("Can't open DB #{db_uri}. Pass in host:port, default is ':1978' #{db.ecode}: #{db.errmsg(db.ecode)}") end
+      #   super *args
+      # end
+      #
+      # def size
+      #   db.stats
+      # end
+
+    end #class
+  end
+end
+

data/lib/monkeyshines/store/tyrant_tdb_key_store.rb

@@ -0,0 +1,20 @@
+require 'tokyotyrant'
+require 'tyrant_rdb_key_store'
+module Monkeyshines
+  module Store
+    #
+    # Implementation of KeyStore with a Local TokyoCabinet Table database (RDBTBL)
+    #
+    class TyrantRdbKeyStore < TyrantRdbKeyStore Monkeyshines::Store::KeyStore
+
+      def db
+        return @db if @db
+        @db ||= TokyoTyrant::RDBTBL.new
+        @db.open(db_host, db_port) or raise("Can't open DB #{db_host}:#{db_port}. Pass in host:port' #{@db.ecode}: #{@db.errmsg(@db.ecode)}")
+        @db
+      end
+
+    end #class
+  end
+end
+

data/lib/monkeyshines/utils/factory_module.rb

@@ -0,0 +1,106 @@
+#
+# Makes a module behave as a factory.
+#
+# A module that extends FactoryModule gets a method #new(klass_name, *args):
+# this finds the class corresponding to klass_name and creates it with *args as
+# arguments.
+#
+# if +klass_name+ is a class, it's used directly.  Otherwise, it's converted to
+# a class, and can be in underscored form (mysql_doc_source) or namespace form
+# (FileSources::WordDoc); the name is interpreted relative to the extending
+# module's namespace. (So, in the example below, :file_doc_source,
+# FileDocSource, DocSource::
+#
+# Example. Given:
+#
+#     module DocSource
+#       extend FactoryModule
+#     end
+#
+#     # ... elsewhere ...
+#     module DocSource
+#       # load docs from file
+#       class FileDocSource
+#         def initialize filename
+#           #...
+#         end
+#       end
+#
+#       # load docs from web
+#       class MySqlDocSource
+#         def initialize host, port, user, password
+#           # ...
+#         end
+#       end
+#     end
+#
+# Then:
+#     DocSource.new :file_doc_source, '/tmp/foo.doc'    # => returns DocSource::FileDocSource
+#     DocSource.new :MySqlDocSource,  'localhost', 6666 # => returns DocSource::MySqlDocSource
+#
+#
+module FactoryModule
+  def self.extended base
+    base.class_eval do
+
+      def self.new klass_name, *args
+        FactoryModule.get_class(self, klass_name).new(*args)
+      end
+
+      def self.from_hash plan
+        return plan unless plan.is_a?(Hash)
+        klass_name = (plan[:type] || plan['type']) or raise "Fat, drunk, and stupid is no way to go through life, son. You need a plan: #{plan.inspect}"
+        FactoryModule.get_class(self, klass_name).from_hash(plan)
+      end
+
+      def self.create plan
+        case
+        # when plan.class.ancestors.include? self
+        when plan.is_a?(Hash)
+          klass_name = plan[:type] || plan['type']
+          FactoryModule.get_class(self, klass_name).new(plan)
+        when plan.is_a?(Symbol)
+          klass_name = plan
+          FactoryModule.get_class(self, klass_name).new()
+        else plan
+        end
+      end
+
+    end
+  end
+
+  def get_class klass_name
+    FactoryModule.get_class self, klass_name
+  end
+
+  FACTORY_CLASSES = {}
+  def self.get_class scope, klass_name
+    return FACTORY_CLASSES[ [scope, klass_name] ] if FACTORY_CLASSES[ [scope, klass_name] ]
+    if klass_name.is_a? Class
+      klass = klass_name
+    else
+      begin
+        klass = scope.find_const(klass_name.to_s.camelize)
+      rescue NameError => e
+        raise "Can't find #{klass_name.inspect} in #{scope}"
+      end
+    end
+    # find_const from wukong/extensions/module via extlib
+    FACTORY_CLASSES[ [scope, klass_name] ] = klass
+  end
+
+
+  #
+  #
+  #   FactoryModule.list_of_classes(Wuclan::Twitter::Scrape, 'followers_ids,friends_ids', 'request')
+  #   # => [Wuclan::Twitter::Scrape::FollowersIdsRequest, Wuclan::Twitter::Scrape::FriendsIdsRequest]
+  #
+  def self.list_of_classes scope, klass_names, prefix=nil, suffix=nil
+    klass_names = klass_names.split(',') if klass_names.is_a?(String)
+    klass_names.map do |klass_name|
+      klass_name = [prefix, klass_name, suffix].compact.join('_') if klass_name.is_a?(String)
+      self.get_class(scope, klass_name)
+    end
+  end
+
+end