monkeyshines 0.2.1 → 0.2.2

data/examples/shorturls/scrape_shorturls.rb

@@ -1,16 +1,19 @@
 #!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../../lib'; $: << File.dirname(__FILE__)
+$: << File.dirname(__FILE__)+'/../../lib'; $: << File.dirname(__FILE__); $: << File.dirname(__FILE__)+'/../../../graphiterb/lib'
 require 'rubygems'
 require 'wukong'
 require 'monkeyshines'
+require 'configliere'
 #
 require 'shorturl_request'
 require 'shorturl_sequence'
+require 'shorturl_stats'
 require 'monkeyshines/utils/uri'
 require 'monkeyshines/utils/filename_pattern'
 require 'monkeyshines/store/conditional_store'
 require 'monkeyshines/fetcher/http_head_fetcher'
-require 'trollop' # gem install trollop
+require 'graphiterb' # needs graphiterb - simple ruby interface for graphite
+# require 'trollop' # gem install trollop
 
 # ===========================================================================
 #
@@ -26,39 +29,74 @@ require 'trollop' # gem install trollop
 #       --base-url="http://tinyurl.com" --min-limit= --max-limit= --encoding_radix=
 #
 #
-opts = Trollop::options do
-  opt :base_url,       "Host part of URL: eg tinyurl.com",             :type => String, :required => true
-  opt :log,            "Log file name; leave blank to use STDERR",     :type => String
-  # input from file
-  opt :from,           "Location of URLs to scrape",                   :type => String
-  opt :skip,           "Initial lines to skip",                        :type => Integer
-  # OR do a random walk
-  opt :random,         "Generate and visit random URL suffixes"
-  opt :min_limit,      "Smallest sequential URL to randomly visit",    :type => Integer # default in shorturl_sequence.rb
-  opt :max_limit,      "Largest sequential URL to randomly visit",     :type => Integer # default in shorturl_sequence.rb
-  opt :encoding_radix, "36 for most, 62 if URLs are case-sensitive",   :type => Integer, :default => 36
-  # output storage
-  opt :cache_loc,      "URI for cache server",                         :type => String
-  opt :chunk_time,     "Frequency to rotate chunk files (in seconds)", :type => Integer, :default => 60*60*4
-  opt :dest_dir,       "Filename base for output, def /data/ripd",     :type => String,  :default => '/data/ripd'
-  opt :dest_pattern,   "Pattern for dump file output",                 :default => ":dest_dir/:handle_prefix/:handle/:date/:handle+:timestamp-:pid.tsv"
-end
-handle = opts[:base_url].gsub(/\.com$/,'').gsub(/\W+/,'')
 
+Configliere.use :commandline, :config_file, :define
+Settings.read 'shorturls.yaml' #~/.configliere/shorturls.yaml
+Settings.define :base_url,     :description => "Host part of URL: eg tinyurl.com", :type => String, :required => true
+# Settings.define :log,          :description => "Log file name; leave blank to use STDERR", :type => String
+Settings.define :log_time,     :description => "Log time interval, in seconds, for periodic logger and Graphite logger", :type => Integer, :default => 60
+Settings.define :log_iters,    :description => "Log iteration interval for periodic logger and Graphite logger", :type => Integer, :default => 10000
+# input from file
+Settings.define :file_from,    :description => "Location of URLs to scrape", :type => String
+Settings.define :file_skip,    :description => "Initial lines to skip", :type => Integer
+# OR do a random walk
+Settings.define :random,       :description => "Generate and visit random URL suffixes"
+Settings.define :random_min,   :description => "Smallest sequential URL to randomly visit",    :type => Integer # default in shorturl_sequence.rb
+Settings.define :random_max,   :description => "Largest sequential URL to randomly visit",     :type => Integer # default in shorturl_sequence.rb
+Settings.define :random_radix, :description => "36 for most, 62 if URLs are case-sensitive",   :type => Integer, :default => 36
+# output storage
+Settings.define :cache_loc,      :description => "URI for cache server",                         :type => String
+Settings.define :chunk_time,     :description => "Frequency to rotate chunk files (in seconds)", :type => Integer, :default => 60*60*4
+Settings.define :rootdir,        :description => "Filename base for output, def /data/ripd",     :type => String,  :default => '/data/ripd/shorturls'
+Settings.define :dest_pattern,   :description => "Pattern for dump file output",                 :default => ":rootdir/:date/:handle+:timestamp-:pid.tsv"
+Settings.resolve!
+Log = Logger.new($stderr) unless defined?(Log)
+
+# Removed trollop optioning, added in configliere instead
+# opts = Trollop::options do
+#   opt :base_url,       "Host part of URL: eg tinyurl.com",             :type => String, :required => true
+#   opt :log,            "Log file name; leave blank to use STDERR",     :type => String
+#   # input from file
+#   opt :from,           "Location of URLs to scrape",                   :type => String
+#   opt :skip,           "Initial lines to skip",                        :type => Integer
+#   # OR do a random walk
+#   opt :random,         "Generate and visit random URL suffixes"
+#   opt :min_limit,      "Smallest sequential URL to randomly visit",    :type => Integer # default in shorturl_sequence.rb
+#   opt :max_limit,      "Largest sequential URL to randomly visit",     :type => Integer # default in shorturl_sequence.rb
+#   opt :encoding_radix, "36 for most, 62 if URLs are case-sensitive",   :type => Integer, :default => 36
+#   # output storage
+#   opt :cache_loc,      "URI for cache server",                         :type => String
+#   opt :chunk_time,     "Frequency to rotate chunk files (in seconds)", :type => Integer, :default => 60*60*4
+#   opt :rootdir,       "Filename base for output, def /data/ripd",     :type => String,  :default => '/data/ripd'
+#   opt :dest_pattern,   "Pattern for dump file output",                 :default => ":rootdir/:handle_prefix/:handle/:date/:handle+:timestamp-:pid.tsv"
+# end
+handle = Settings.base_url.gsub(/\.com$/,'').gsub(/\W+/,'')
+hostname ||= `hostname`.chomp.gsub(".","_")
+
+
+#
 # ******************** Log ********************
-opts[:log] = (WORK_DIR+"/log/shorturls_#{handle}-#{Time.now.to_flat}.log") if (opts[:log]=='')
-periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:iters => 10000, :time => 30)
+#
+# (I don't think the log file name ever gets used)
+# Settings.log = (WORK_DIR+"/log/shorturls_#{handle}-#{Time.now.to_flat}.log") if (Settings.log=='')
+periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:iters => Settings.log_iters, :time => Settings.log_time)
+
+#
+# ******************** Graphite Sender ***********************
+#
+graphite_sender = Graphiterb::GraphiteLogger.new(:iters => Settings.log_iters, :time => Settings.log_time)
 
 #
 # ******************** Load from store or random walk ********************
 #
-if opts[:from]
-  src_store = Monkeyshines::Store::FlatFileStore.new_from_command_line(opts, :filemode => 'r')
-  src_store.skip!(opts[:skip].to_i) if opts[:skip]
-elsif opts[:random]
+if Settings.file_from
+  # Settings.filename = Settings.file_from
+  src_store = Monkeyshines::Store::FlatFileStore.new(:filename => Settings.file_from, :skip => Settings.file_skip.to_i) # + {:filemode => 'r'}
+  # src_store.skip!(Settings.file_skip.to_i) if Settings.file_skip
+elsif Settings.random
   src_store = Monkeyshines::Store::RandomUrlStream.new_from_command_line(opts)
 else
-  Trollop::die "Need to either say --random or --from=filename"
+  Settings.die "Need to either say --random or --file_from=filename"
 end
 
 #
@@ -67,30 +105,37 @@ end
 # Track visited URLs with key-value database
 #
 RDB_PORTS  = { 'tinyurl' => "localhost:10042", 'bitly' => "localhost:10043", 'other' => "localhost:10044" }
-cache_loc  = opts[:cache_loc] || RDB_PORTS[handle] or raise "Need a handle (bitly, tinyurl or other)."
-dest_cache = Monkeyshines::Store::TyrantRdbKeyStore.new(cache_loc)
+cache_loc  = Settings.cache_loc || RDB_PORTS[handle] or raise "Need a handle (bitly, tinyurl or other)."
+dest_cache = Monkeyshines::Store::TyrantRdbKeyStore.new(:uri => cache_loc)
+
+
 # dest_cache = Monkeyshines::Store::MultiplexShorturlCache.new(RDB_PORTS)
 
 #
 # Store the data into flat files
 #
-dest_pattern = Monkeyshines::Utils::FilenamePattern.new(opts[:dest_pattern],
-  :handle => 'shorturl-'+handle, :dest_dir => opts[:dest_dir])
-dest_files   = Monkeyshines::Store::ChunkedFlatFileStore.new(dest_pattern,
-  opts[:chunk_time].to_i, opts)
+dest_pattern = Monkeyshines::Utils::FilenamePattern.new(Settings.dest_pattern,
+  :handle => 'shorturl-'+handle, :rootdir => Settings.rootdir)
+dest_files   = Monkeyshines::Store::ChunkedFlatFileStore.new(:pattern => Settings.dest_pattern,
+  :chunk_time => Settings.chunk_time.to_i, :handle => 'shorturl-'+handle, :rootdir => Settings.rootdir)
 
 #
 # Conditional store uses the key-value DB to boss around the flat files --
 # requests are only made (and thus data is only output) if the url is missing
 # from the key-value store.
 #
-dest_store = Monkeyshines::Store::ConditionalStore.new(dest_cache, dest_files)
+dest_store = Monkeyshines::Store::ConditionalStore.new(:cache => dest_cache, :store => dest_files)
 
 #
 # ******************** Fetcher ********************
 #
 fetcher = Monkeyshines::Fetcher::HttpHeadFetcher.new
 
+#
+# ******************** Success/Fail stats ********************
+#
+stats = ShorturlStats.new(0,0,0,0)
+
 #
 # ******************** Do this thing ********************
 #
@@ -104,9 +149,18 @@ src_store.each do |bareurl, *args|
   result = dest_store.set( req.url ) do
     response = fetcher.get(req)                             # do the url fetch
     next unless response.response_code || response.contents # don't store bad fetches
+    stats.code_sort(response.response_code)                 # count successes (301) and failures (404)
     [response.scraped_at, response]                         # timestamp into cache, result into flat file
   end
-  periodic_log.periodically{ ["%7d"%dest_store.misses, 'misses', dest_store.size, req.response_code, result, req.url] }
+  periodic_log.periodically{ ["%7d"%stats.success_tot, 'successes', "%7d"%stats.failure_tot, 'failures', dest_store.size, req.response_code, result, req.url] }
+  graphite_sender.periodically do |metrics, iter, since|
+    rates = stats.rates_inst
+    metrics << ["scraper.#{hostname}.shorturl.#{handle}.success_rate", rates[0]]
+    metrics << ["scraper.#{hostname}.shorturl.#{handle}.failure_rate", rates[1]]
+    metrics << ["scraper.#{hostname}.shorturl.#{handle}.success_tot_rate", stats.rates_tot[0]]
+    metrics << ["scraper.#{hostname}.shorturl.#{handle}.failure_tot_rate", stats.rates_tot[1]]
+    metrics << ["scraper.#{hostname}.shorturl.#{handle}.current_file_size", dest_files.size]
+  end
 end
 dest_store.close
 fetcher.close

data/examples/shorturls/shorturl_stats.rb

@@ -0,0 +1,37 @@
+class ShorturlStats < Struct.new(
+    :success_tot,
+    :failure_tot,
+    :success_last,
+    :fail_last
+    )
+    
+  def code_sort code
+    case code.to_s
+    when /4\d{2}/
+      self.failure_tot += 1
+      self.fail_last += 1
+    when /3\d{2}/
+      self.success_tot += 1
+      self.success_last += 1
+    else 
+      Log.warn "Code #{code} not included in stats."
+    end
+  end
+  
+  def rates_inst
+    return [0,0] if (self.success_last.to_f + self.fail_last.to_f) == 0
+    s_rate = (self.success_last.to_f)/(self.success_last.to_f + self.fail_last.to_f)
+    f_rate = (self.fail_last.to_f)/(self.success_last.to_f + self.fail_last.to_f)
+    self.success_last = 0
+    self.fail_last = 0
+    [s_rate,f_rate]
+  end
+  
+  def rates_tot
+    return [0,0] if (self.success_tot.to_f + self.failure_tot.to_f) == 0
+    st_rate = (self.success_tot.to_f)/(self.success_tot.to_f + self.failure_tot.to_f)
+    ft_rate = (self.failure_tot.to_f)/(self.success_tot.to_f + self.failure_tot.to_f)
+    [st_rate,ft_rate]
+  end
+    
+end

data/examples/shorturls/split_short_urls.rb

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+WORK_DIR = '/data/rawd/social/networks/twitter_friends/tokens_by_month/'
+
+OTHER_SHORTURL_RE = 
+%r{.*(http://(?:1link.in|4url.cc|6url.com|adjix.com|ad.vu|bellypath.com|bkite.com|budurl.com|canurl.com|chod.sk|cli.gs|decenturl.com|dn.vc|doiop.com|dwarfurl.com|easyuri.com|easyurl.net|ff.im|go2cut.com|gonext.org|hulu.com|hypem.com|ifood.tv|ilix.in|is.gd|ix.it|jdem.cz|jijr.com|kissa.be|kurl.us|litturl.com|lnkurl.com|memurl.com|metamark.net|miklos.dk|minilien.com|minurl.org|muhlink.com|myurl.in|myurl.us|notlong.com|ow.ly|plexp.com|poprl.com|qurlyq.com|redirx.com|s3nt.com|shorterlink.com|shortlinks.co.uk|short.to|shorturl.com|shrinklink.co.uk|shrinkurl.us|shrt.st|shurl.net|simurl.com|shorl.com|smarturl.eu|snipr.com|snipurl.com|snurl.com|sn.vc|starturl.com|surl.co.uk|tighturl.com|timesurl.at|tiny123.com|tiny.cc|tinylink.com|tobtr.com|traceurl.com|tr.im|tweetburner.com|twitpwr.com|twitthis.com|twurl.nl|u.mavrev.com|ur1.ca|url9.com|urlborg.com|urlbrief.com|urlcover.com|urlcut.com|urlhawk.com|url-press.com|urlsmash.com|urltea.com|urlvi.be|vimeo.com|wlink.us|xaddr.com|xil.in|xrl.us|x.se|xs.md|yatuc.com|yep.it|yweb.com|zi.ma|w3t.org)/.+)}
+
+bitly_file = File.open('/home/doncarlo/shorturls/shorturls_bitly','w')
+tinyurl_file = File.open('/home/doncarlo/shorturls/shorturls_tinyurl','w')
+otherurl_file = File.open('/home/doncarlo/shorturls/shorturls_other','w')
+
+
+
+File.open(WORK_DIR + 'urls_by_month-20091111.tsv').each do |line|
+  line.chomp!
+  url = line.split("\t")[-1]
+  bitly_file << $1 + "\n" if url =~ %r{.*(http://bit.ly/.+)}
+  tinyurl_file << $1 + "\n" if url =~ %r{.*(http://tinyurl.com/.+)}
+  otherurl_file << $1 + "\n" if url =~ OTHER_SHORTURL_RE
+end
+

data/lib/monkeyshines/fetcher/authed_http_fetcher.rb

@@ -1,25 +1,137 @@
 require 'net/http'
+require 'oauth'
 Net::HTTP.version_1_2
 module Monkeyshines
   module Fetcher
 
     #
-    class AuthedHttpFetcher
-      cattr_accessor :auth_params
+    class AuthedHttpFetcher < HttpFetcher
+      attr_accessor :auth_params, :oauth_token, :oauth_secret, :consumer_key, :consumer_secret, :site, :authorize_path
+      # 
+      # All the stuff below was copied from http://github.com/moomerman/twitter_oauth in the client.rb file
+      # 
+      # def initialize(options = {})
+      #   @consumer_key = options[:consumer_key]
+      #   @consumer_secret = options[:consumer_secret]
+      #   @token = options[:token]
+      #   @secret = options[:secret]
+      # end
+      # 
+      # def authorize(token, secret, options = {})
+      #   request_token = OAuth::RequestToken.new(
+      #     consumer, token, secret
+      #   )
+      #   @access_token = request_token.get_access_token(options)
+      #   @token = @access_token.token
+      #   @secret = @access_token.secret
+      #   @access_token
+      # end
+      # 
+      # def show(username)
+      #   get("/users/show/#{username}.json")
+      # end
+      # 
+      # # Returns the string "ok" in the requested format with a 200 OK HTTP status code.
+      # def test
+      #   get("/help/test.json")
+      # end
+      # 
+      # def request_token(options={})
+      #   consumer.get_request_token(options)
+      # end
+      # 
+      # def authentication_request_token(options={})
+      #   consumer.options[:authorize_path] = '/oauth/authenticate'
+      #   request_token(options)
+      # end
+      # 
+      # private
+      # 
+      #   def consumer
+      #     @consumer ||= OAuth::Consumer.new(
+      #       @consumer_key,
+      #       @consumer_secret,
+      #       { :site => "http://api.twitter.com" }
+      #     )
+      #   end
+      # 
+      #   def access_token
+      #     @access_token ||= OAuth::AccessToken.new(consumer, @token, @secret)
+      #   end
+      # 
+      #   def get(path, headers={})
+      #     headers.merge!("User-Agent" => "twitter_oauth gem v#{TwitterOAuth::VERSION}")
+      #     oauth_response = access_token.get("/1#{path}", headers)
+      #     JSON.parse(oauth_response.body)
+      #   end
+      # 
+      #   def post(path, body='', headers={})
+      #     headers.merge!("User-Agent" => "twitter_oauth gem v#{TwitterOAuth::VERSION}")
+      #     oauth_response = access_token.post("/1#{path}", body, headers)
+      #     JSON.parse(oauth_response.body)
+      #   end
+      # 
+      #   def delete(path, headers={})
+      #     headers.merge!("User-Agent" => "twitter_oauth gem v#{TwitterOAuth::VERSION}")
+      #     oauth_response = access_token.delete("/1#{path}", headers)
+      #     JSON.parse(oauth_response.body)
+      #   end
 
-      def get_request_token
+
+      def initialize _options={}
+        super _options
+        # These should get called by calling super, right?
+        # self.username = options[:username]
+        # self.password = options[:password]
+        # self.http_req_options = {}
+        # self.http_req_options["User-Agent"] = options[:user_agent] || USER_AGENT
+        # self.http_req_options["Connection"] = "keep-alive"
+        self.oauth_token = options[:oauth_token]
+        self.oauth_secret = options[:oauth_token_secret]
+        self.consumer_key = options[:consumer_key]
+        self.consumer_secret = options[:consumer_secret]
+        self.site = options[:site]
+        self.authorize_path = options[:authorize_path]
+      end
+
+      def request_token(options={})
+        consumer.options[:authorize_path] = @authorize_path
+        consumer.get_request_token(options)
       end
 
-      def authorize
+      def authorize(token, secret, options = {})
+        request_token = OAuth::RequestToken.new(
+          consumer, token, secret
+        )
+        @access_token = request_token.get_access_token(options)
+        @token = @access_token.token
+        @secret = @access_token.secret
+        @access_token
       end
 
       def get_access_token
       end
 
-      def api_key
+      def oauth_token
+        @oauth_token
+      end
+      
+      def oauth_secret
+        @oauth_secret
+      end
+      
+      def consumer
+        @consumer ||= OAuth::Consumer.new(
+          @consumer_key,
+          @consumer_secret,
+          { :site => @site }
+        )
       end
-      def api_secret
+    
+      def access_token
+        @access_token ||= OAuth::AccessToken.new(consumer, @token, @secret)
       end
+
       def session_key
       end
       

data/lib/monkeyshines/fetcher/http_fetcher.rb

@@ -82,17 +82,18 @@ module Monkeyshines
         # Response-based sleep time
         sleep_time = 0
         case response
-        when Net::HTTPSuccess             then return          # 2xx
-        when Net::HTTPRedirection         then return          # 3xx
-        when Net::HTTPBadRequest          then sleep_time =  5 # 400 (rate limit, probably)
-        when Net::HTTPUnauthorized        then sleep_time =  0 # 401 (protected user, probably)
-        when Net::HTTPForbidden           then sleep_time =  4 # 403 update limit
-        when Net::HTTPNotFound            then sleep_time =  0 # 404 deleted
-        when Net::HTTPServiceUnavailable  then sleep_time = 15 # 503 Fail Whale
-        when Net::HTTPServerError         then sleep_time =  2 # 5xx All other server errors
+        when Net::HTTPSuccess             then return           # 2xx
+        when Net::HTTPRedirection         then return           # 3xx
+        when Net::HTTPBadRequest          then sleep_time =  10 # 400 (rate limit, probably)
+        when Net::HTTPUnauthorized        then sleep_time =  0  # 401 (protected user, probably)
+        when Net::HTTPForbidden           then sleep_time =  10 # 403 update limit
+        when Net::HTTPNotFound            then sleep_time =  0  # 404 deleted or suspended
+        when Net::HTTPServiceUnavailable  then sleep_time =  10 # 503 Fail Whale
+        when Net::HTTPServerError         then sleep_time =  2  # 5xx All other server errors
         else                              sleep_time = 1
         end
-        Log.warn "Received #{response.code}, sleeping #{sleep_time} ('#{response.message[0..200].gsub(%r{[\r\n\t]}, " ")}' from #{@host}+#{@connection_opened_at})"
+        sleep_time += response['retry-after'].to_i rescue 0
+        Log.warn "Received #{response.code} and retry-after #{response['retry-after']}, sleeping #{sleep_time} ('#{response.message[0..200].gsub(%r{[\r\n\t]}, " ")}' from #{@host}+#{@connection_opened_at}): '#{response.body[0..200].gsub(%r{[\r\n\t]}, " ")}'"
         sleep sleep_time
       end
 

data/lib/monkeyshines/monitor/periodic_logger.rb

@@ -24,7 +24,8 @@ module Monkeyshines
       #
       def periodically &block
         super do
-          result = [ "%10d"%iter, "%7.1f"%since, "%7.1f"%rate, (block ? block.call : nil) ].flatten.compact
+          now = Time.now.utc.to_f
+          result = [ "%10d"%iter, "%7.1f"%since, "%7.1f"%inst_rate(now), (block ? block.call : nil) ].flatten.compact
           Log.info result.join("\t")
         end
       end

data/lib/monkeyshines/monitor/periodic_monitor.rb

@@ -18,12 +18,13 @@ module Monkeyshines
     #
     class PeriodicMonitor
       attr_accessor :time_interval, :iter_interval
-      attr_accessor :last_time, :iter, :started_at
+      attr_accessor :last_time, :current_iter, :iter, :started_at
 
       def initialize options={}
         self.started_at    = Time.now.utc.to_f
         self.last_time     = started_at
         self.iter          = 0
+        self.current_iter  = 0
         self.time_interval = options[:time]
         self.iter_interval = options[:iters]
       end
@@ -42,10 +43,14 @@ module Monkeyshines
       def since
         Time.now.utc.to_f - started_at
       end
-      # Iterations per second
+      # Overall iterations per second
       def rate
         iter.to_f / since.to_f
       end
+      # "Instantaneous" iterations per second
+      def inst_rate now
+        current_iter.to_f / (now-last_time).to_f
+      end
 
       #
       # if the interval conditions are met, executes block; otherwise just does
@@ -53,10 +58,12 @@ module Monkeyshines
       #
       def periodically &block
         self.iter += 1
+        self.current_iter += 1
         now       = Time.now.utc.to_f
         if enough_iterations? || enough_time?(now)
           block.call(iter, (now-last_time))
           self.last_time = now
+          self.current_iter = 0
         end
       end
     end

data/lib/monkeyshines/options.rb

@@ -1,3 +1,5 @@
+require 'yaml'
+require 'monkeyshines/runner_core/options'
 require 'monkeyshines/utils/trollop'
 module Monkeyshines
 

data/lib/monkeyshines/runner.rb

@@ -142,6 +142,7 @@ module Monkeyshines
     def setup_main_log
       unless options[:log][:dest].blank?
         log_file = "%s/log/%s" % [WORK_DIR, options[:log][:dest]]
+        require 'fileutils'
         FileUtils.mkdir_p(File.dirname(log_file))
         $stdout = $stderr = File.open( log_file+"-console.log", "a" )
       end

data/lib/monkeyshines/store/flat_file_store.rb

@@ -67,6 +67,12 @@ module Monkeyshines
         file << obj.to_flat.join("\t")+"\n"
         obj
       end
+      
+      # returns the size of the current file
+      def size
+        return 0 if !@file
+        File.size(filename)
+      end
 
       def set key, *args, &block
         tok, obj = block.call

data/lib/monkeyshines/store/tyrant_rdb_key_store.rb

@@ -12,13 +12,14 @@ module Monkeyshines
       def initialize options
         raise "URI for #{self.class} is required" if options[:uri].blank?
         self.db_host, self.db_port = options[:uri].to_s.split(':')
+        self.db_host.gsub!(/^(localhost|127\.0\.0\.1)$/,'')
         super options
       end
 
       def db
         return @db if @db
         @db ||= TokyoTyrant::RDB.new
-        @db.open(db_host, db_port) or raise("Can't open DB #{db_host}:#{db_port}. Pass in host:port' #{@db.ecode}: #{@db.errmsg(@db.ecode)}")
+        @db.open(db_host, db_port) or raise("Can't open DB at host #{db_host} port #{db_port}. Pass in host:port' #{@db.ecode}: #{@db.errmsg(@db.ecode)}")
         @db
       end
 

data/lib/monkeyshines/utils/filename_pattern.rb

@@ -75,7 +75,7 @@ module Monkeyshines
 
       # Memoized: the hostname for the machine running this script.
       def hostname
-        @hostname ||= ENV['HOSTNAME'] || `hostname`
+        @hostname ||= ENV['HOSTNAME'] || `hostname`.delete("\n")
       end
       # Memoized: the Process ID for this invocation.
       def pid

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: monkeyshines
 version: !ruby/object:Gem::Version 
-  version: 0.2.1
+  hash: 19
+  prerelease: false
+  segments: 
+  - 0
+  - 2
+  - 2
+  version: 0.2.2
 platform: ruby
 authors: 
 - Philip (flip) Kromer
@@ -9,39 +15,51 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-02 00:00:00 -06:00
+date: 2010-07-15 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: addressable
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: uuid
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: wukong
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id003
 description: A simple scraper for directed scrapes of APIs, feed or structured HTML. Plays nicely with wuclan and wukong.
 email: flip@infochimps.org
 executables: []
@@ -64,12 +82,12 @@ files:
 - examples/shorturls/bulkload_shorturls.rb
 - examples/shorturls/extract_urls.rb
 - examples/shorturls/multiplex_shorturl_cache.rb
-- examples/shorturls/old/multidump_and_fix_shorturls.rb
-- examples/shorturls/old/shorturl_stats.rb
 - examples/shorturls/scrape_shorturls.rb
 - examples/shorturls/shorturl_request.rb
 - examples/shorturls/shorturl_sequence.rb
 - examples/shorturls/shorturl_start_tyrant.sh
+- examples/shorturls/shorturl_stats.rb
+- examples/shorturls/split_short_urls.rb
 - examples/shorturls/start_shorturl_cache.sh
 - lib/monkeyshines.rb
 - lib/monkeyshines/extensions.rb
@@ -139,37 +157,43 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: A simple scraper for directed scrapes of APIs, feed or structured HTML.
 test_files: 
 - spec/monkeyshines_spec.rb
 - spec/spec_helper.rb
-- examples/bulk_urls/scrape_bulk_urls.rb
-- examples/rename_tree/rename_hdp_tree.rb
-- examples/rename_tree/rename_ripd_tree.rb
-- examples/rss_feeds/scrape_rss_feeds.rb
-- examples/shorturls/bulkdump_shorturls.rb
-- examples/shorturls/bulkload_shorturls.rb
-- examples/shorturls/extract_urls.rb
+- examples/shorturls/shorturl_stats.rb
+- examples/shorturls/shorturl_request.rb
 - examples/shorturls/multiplex_shorturl_cache.rb
-- examples/shorturls/old/multidump_and_fix_shorturls.rb
-- examples/shorturls/old/shorturl_stats.rb
+- examples/shorturls/bulkload_shorturls.rb
 - examples/shorturls/scrape_shorturls.rb
-- examples/shorturls/shorturl_request.rb
+- examples/shorturls/extract_urls.rb
+- examples/shorturls/bulkdump_shorturls.rb
 - examples/shorturls/shorturl_sequence.rb
+- examples/shorturls/split_short_urls.rb
+- examples/rename_tree/rename_hdp_tree.rb
+- examples/rename_tree/rename_ripd_tree.rb
+- examples/rss_feeds/scrape_rss_feeds.rb
+- examples/bulk_urls/scrape_bulk_urls.rb

data/examples/shorturls/old/multidump_and_fix_shorturls.rb

@@ -1,66 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../../lib'; $: << File.dirname(__FILE__)
-require 'rubygems'
-require 'trollop'
-require 'wukong'
-require 'monkeyshines'
-require 'shorturl_request'
-require 'shorturl_sequence'
-require 'monkeyshines/utils/uri'
-
-#
-# Command line options
-#
-opts = Trollop::options do
-  opt :from_type,    'Class name for scrape store to load from',  :type => String
-  opt :from,         'URI for scrape store to load from',  :type => String
-  opt :into,         'Filename for flat TSV dump', :type => String
-  opt :log,          'File to store log', :type => String
-end
-Trollop::die :from_type unless opts[:from_type]
-
-# ******************** Read From ********************
-src_store_klass = Wukong.class_from_resource('Monkeyshines::Store::'+opts[:from_type])
-src_store = src_store_klass.new(opts[:from])
-Log.info "Loaded store with #{src_store.size}"
-
-# ******************** Write into ********************
-DUMPFILE_BASE = opts[:into]
-def make_store uri
-  Monkeyshines::Store::FlatFileStore.new "#{DUMPFILE_BASE+"-"+uri}.tsv", :filemode => 'w'
-end
-dests = { }
-[ 'tinyurl', 'bitly', 'other'
-].each do |handle|
-  dests[handle] = make_store handle
-end
-
-# ******************** Log ********************
-periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:iters => 20_000, :time => 30)
-
-# ******************** Cross Load ********************
-# Read , process, dump
-iter = 0
-src_store.each do |key, hsh|
-  hsh['contents']             ||= hsh.delete 'expanded_url'
-  hsh['response_code']          = nil if hsh['response_code']    == 'nil'
-  hsh['contents']               = nil if hsh['contents']         == 'nil'
-  unless hsh['contents'] || hsh['response_code']
-    # Log.info "removing #{hsh.inspect}"
-    src_store.db.out(key)
-    next
-  end
-  hsh['response_message']       = nil if hsh['response_message'] == 'nil'
-  hsh['url']                  ||= hsh.delete 'short_url'
-  req = ShorturlRequest.from_hash hsh
-  periodic_log.periodically{ [src_store.size, req.to_flat] }
-
-  req.contents = Addressable::URI.scrub_url req.contents if req.contents
-
-  case
-  when (key =~ %r{^http://tinyurl.com/(.*)}) then dests['tinyurl'].save req
-  when (key =~ %r{^http://bit.ly/(.*)})      then dests['bitly'  ].save req
-  else                                            dests['other'  ].save req
-  end
-  # src_store.save(key, req.to_hash.compact)
-end

data/examples/shorturls/old/shorturl_stats.rb

@@ -1,81 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../../lib'; $: << File.dirname(__FILE__)
-#require 'rubygems'
-# require 'wukong'
-require 'monkeyshines'
-# require 'monkeyshines/utils/uri'
-# require 'monkeyshines/utils/filename_pattern'
-# require 'monkeyshines/store/conditional_store'
-# require 'monkeyshines/fetcher/http_head_fetcher'
-# require 'trollop' # gem install trollop
-# require 'shorturl_request'
-require 'shorturl_sequence'
-
-digits = { } ; (('0'..'9').to_a+('a'..'z').to_a).each do |ch| digits[ch] = 0 end
-
-# (1..10000).each do |idx|
-#   s = ShorturlSequence.encode_integer idx, 36
-#   digits[s[0..0]] += 1
-# end
-# p digits
-# puts digits.sort.map{|ch,ct| "%-7s\t%10d"%[ch,ct]}
-
-class Histo
-  attr_accessor :buckets
-  def initialize
-    self.buckets = { }
-  end
-  def << val
-    buckets[val] ||= 0
-    buckets[val]  += 1
-  end
-  def dump
-    buckets.sort.each do |val, count|
-      puts "%10d\t%s"%[count,val]
-    end
-  end
-end
-
-len_histo = Histo.new
-num_histo = Histo.new
-ltr_histo = Histo.new
-iter = 0
-
-# 123456789-123456789-
-# http://bit.ly/
-# http://tinyurl.com/
-BASE_URL     = "http://is.gd/"
-RADIX        = 62
-HANDLE       = BASE_URL.gsub(%r{^http://},'').gsub(/\.com$/,'').gsub(/\W+/,'')
-BASE_URL_LEN = BASE_URL.length
-MAX_TAIL_LEN = BASE_URL_LEN + 2 + 6
-SIX_CHARS    = RADIX**6
-File.open("rawd/req/shorturl_requests-20090710-#{HANDLE}.tsv"
-  ) do |reqfile|
-  reqfile.each do |url|
-    #decode
-    next unless url.length <= MAX_TAIL_LEN
-    tail = url.chomp.strip[BASE_URL_LEN..-1] || ''
-    # tail.downcase!
-    asnum = ShorturlSequence.decode_str tail, RADIX rescue nil  # tail.to_i(36) rescue -1
-    next unless asnum && asnum < SIX_CHARS
-    size = (asnum / 1_000_000)
-    len  = tail.length
-    # track stats
-    len_histo << len
-    num_histo << size
-    ltr_histo << "%s-%s" % [len, tail[0..0]] #  + (len > 1 ? '.'* (len-1) : '')
-    puts iter if ((iter += 1) % 1_000_000 == 0)
-
-  end
-end
-puts "Integer magnitude of decoded (M):"
-num_histo.dump
-puts "Length of encoded:"
-len_histo.dump
-puts "First Letter:"
-ltr_histo.dump
-
-
-# puts tail.length # [tail.length, tail, tail[-1].to_i].join("\t")
-# puts [asnum, tail, url].inspect