monkeyshines 0.0.2

data/examples/shorturls/old/shorturl_stats.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'; $: << File.dirname(__FILE__)
+#require 'rubygems'
+# require 'wukong'
+require 'monkeyshines'
+# require 'monkeyshines/utils/uri'
+# require 'monkeyshines/utils/filename_pattern'
+# require 'monkeyshines/store/conditional_store'
+# require 'monkeyshines/fetcher/http_head_fetcher'
+# require 'trollop' # gem install trollop
+# require 'shorturl_request'
+require 'shorturl_sequence'
+
+digits = { } ; (('0'..'9').to_a+('a'..'z').to_a).each do |ch| digits[ch] = 0 end
+
+# (1..10000).each do |idx|
+#   s = ShorturlSequence.encode_integer idx, 36
+#   digits[s[0..0]] += 1
+# end
+# p digits
+# puts digits.sort.map{|ch,ct| "%-7s\t%10d"%[ch,ct]}
+
+class Histo
+  attr_accessor :buckets
+  def initialize
+    self.buckets = { }
+  end
+  def << val
+    buckets[val] ||= 0
+    buckets[val]  += 1
+  end
+  def dump
+    buckets.sort.each do |val, count|
+      puts "%10d\t%s"%[count,val]
+    end
+  end
+end
+
+len_histo = Histo.new
+num_histo = Histo.new
+ltr_histo = Histo.new
+iter = 0
+
+# 123456789-123456789-
+# http://bit.ly/
+# http://tinyurl.com/
+BASE_URL     = "http://is.gd/"
+RADIX        = 62
+HANDLE       = BASE_URL.gsub(%r{^http://},'').gsub(/\.com$/,'').gsub(/\W+/,'')
+BASE_URL_LEN = BASE_URL.length
+MAX_TAIL_LEN = BASE_URL_LEN + 2 + 6
+SIX_CHARS    = RADIX**6
+File.open("rawd/req/shorturl_requests-20090710-#{HANDLE}.tsv"
+  ) do |reqfile|
+  reqfile.each do |url|
+    #decode
+    next unless url.length <= MAX_TAIL_LEN
+    tail = url.chomp.strip[BASE_URL_LEN..-1] || ''
+    # tail.downcase!
+    asnum = ShorturlSequence.decode_str tail, RADIX rescue nil  # tail.to_i(36) rescue -1
+    next unless asnum && asnum < SIX_CHARS
+    size = (asnum / 1_000_000)
+    len  = tail.length
+    # track stats
+    len_histo << len
+    num_histo << size
+    ltr_histo << "%s-%s" % [len, tail[0..0]] #  + (len > 1 ? '.'* (len-1) : '')
+    puts iter if ((iter += 1) % 1_000_000 == 0)
+
+  end
+end
+puts "Integer magnitude of decoded (M):"
+num_histo.dump
+puts "Length of encoded:"
+len_histo.dump
+puts "First Letter:"
+ltr_histo.dump
+
+
+# puts tail.length # [tail.length, tail, tail[-1].to_i].join("\t")
+# puts [asnum, tail, url].inspect

data/examples/shorturls/scrape_shorturls.rb

@@ -0,0 +1,112 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'; $: << File.dirname(__FILE__)
+require 'rubygems'
+require 'wukong'
+require 'monkeyshines'
+#
+require 'shorturl_request'
+require 'shorturl_sequence'
+require 'monkeyshines/utils/uri'
+require 'monkeyshines/utils/filename_pattern'
+require 'monkeyshines/store/conditional_store'
+require 'monkeyshines/fetcher/http_head_fetcher'
+require 'trollop' # gem install trollop
+
+# ===========================================================================
+#
+# scrape_shorturls.rb --
+#
+# To scrape from a list of shortened urls:
+#
+#    ./shorturl_random_scrape.rb --from-type=FlatFileStore --from=request_urls.tsv
+#
+# To do a random scrape:
+#
+#    ./shorturl_random_scrape.rb --from-type=RandomUrlStream --base-url=tinyurl.com
+#       --base-url="http://tinyurl.com" --min-limit= --max-limit= --encoding_radix=
+#
+#
+opts = Trollop::options do
+  opt :base_url,       "Host part of URL: eg tinyurl.com",             :type => String, :required => true
+  opt :log,            "Log file name; leave blank to use STDERR",     :type => String
+  # input from file
+  opt :from,           "Location of URLs to scrape",                   :type => String
+  opt :skip,           "Initial lines to skip",                        :type => Integer
+  # OR do a random walk
+  opt :random,         "Generate and visit random URL suffixes"
+  opt :min_limit,      "Smallest sequential URL to randomly visit",    :type => Integer # default in shorturl_sequence.rb
+  opt :max_limit,      "Largest sequential URL to randomly visit",     :type => Integer # default in shorturl_sequence.rb
+  opt :encoding_radix, "36 for most, 62 if URLs are case-sensitive",   :type => Integer, :default => 36
+  # output storage
+  opt :cache_loc,      "URI for cache server",                         :type => String
+  opt :chunk_time,     "Frequency to rotate chunk files (in seconds)", :type => Integer, :default => 60*60*4
+  opt :dest_dir,       "Filename base for output, def /data/ripd",     :type => String,  :default => '/data/ripd'
+  opt :dest_pattern,   "Pattern for dump file output",                 :default => ":dest_dir/:handle_prefix/:handle/:date/:handle+:timestamp-:pid.tsv"
+end
+handle = opts[:base_url].gsub(/\.com$/,'').gsub(/\W+/,'')
+
+# ******************** Log ********************
+opts[:log] = (WORK_DIR+"/log/shorturls_#{handle}-#{Time.now.to_flat}.log") if (opts[:log]=='')
+periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:iters => 10000, :time => 30)
+
+#
+# ******************** Load from store or random walk ********************
+#
+if opts[:from]
+  src_store = Monkeyshines::Store::FlatFileStore.new_from_command_line(opts, :filemode => 'r')
+  src_store.skip!(opts[:skip].to_i) if opts[:skip]
+elsif opts[:random]
+  src_store = Monkeyshines::Store::RandomUrlStream.new_from_command_line(opts)
+else
+  Trollop::die "Need to either say --random or --from=filename"
+end
+
+#
+# ******************** Store output ********************
+#
+# Track visited URLs with key-value database
+#
+RDB_PORTS  = { 'tinyurl' => "localhost:10042", 'bitly' => "localhost:10043", 'other' => "localhost:10044" }
+cache_loc  = opts[:cache_loc] || RDB_PORTS[handle] or raise "Need a handle (bitly, tinyurl or other)."
+dest_cache = Monkeyshines::Store::TyrantRdbKeyStore.new(cache_loc)
+# dest_cache = Monkeyshines::Store::MultiplexShorturlCache.new(RDB_PORTS)
+
+#
+# Store the data into flat files
+#
+dest_pattern = Monkeyshines::Utils::FilenamePattern.new(opts[:dest_pattern],
+  :handle => 'shorturl-'+handle, :dest_dir => opts[:dest_dir])
+dest_files   = Monkeyshines::Store::ChunkedFlatFileStore.new(dest_pattern,
+  opts[:chunk_time].to_i, opts)
+
+#
+# Conditional store uses the key-value DB to boss around the flat files --
+# requests are only made (and thus data is only output) if the url is missing
+# from the key-value store.
+#
+dest_store = Monkeyshines::Store::ConditionalStore.new(dest_cache, dest_files)
+
+#
+# ******************** Fetcher ********************
+#
+fetcher = Monkeyshines::Fetcher::HttpHeadFetcher.new
+
+#
+# ******************** Do this thing ********************
+#
+Log.info "Beginning scrape itself"
+src_store.each do |bareurl, *args|
+  # prepare the request
+  next if bareurl =~ %r{\Ahttp://(poprl.com|short.to|timesurl.at|bkite.com)}
+  req = ShorturlRequest.new(bareurl, *args)
+
+  # conditional store only calls fetcher if url key is missing.
+  result = dest_store.set( req.url ) do
+    response = fetcher.get(req)                             # do the url fetch
+    next unless response.response_code || response.contents # don't store bad fetches
+    [response.scraped_at, response]                         # timestamp into cache, result into flat file
+  end
+  periodic_log.periodically{ ["%7d"%dest_store.misses, 'misses', dest_store.size, req.response_code, result, req.url] }
+end
+dest_store.close
+fetcher.close

data/examples/shorturls/shorturl_request.rb

@@ -0,0 +1,29 @@
+class ShorturlRequest < Struct.new(
+    :url,
+    :scraped_at,
+    :response_code, :response_message,
+    :contents
+    )
+  alias_method :short_url=,    :url=
+  alias_method :expanded_url=, :contents=
+  alias_method :expanded_url,  :contents
+  #
+  # All we care about is the redirect destination.
+  #
+  def response= response
+    self.contents  = response["location"]
+  end
+
+  #
+  # The major shortening services
+  #
+  # Do any of the mainstream shorteners use in-band characters besides \w
+  # alphanum and - dash?  (idek.net uses a ~ and pastoid.com a + but they
+  # are not popular enough to justify the annoyance of allowing extra
+  # chars).
+  #
+  SHORTURL_RE = %r{\Ahttp://(?:1link.in|4url.cc|6url.com|adjix.com|ad.vu|bellypath.com|bit.ly|bkite.com|budurl.com|canurl.com|chod.sk|cli.gs|decenturl.com|dn.vc|doiop.com|dwarfurl.com|easyuri.com|easyurl.net|ff.im|go2cut.com|gonext.org|hulu.com|hypem.com|ifood.tv|ilix.in|is.gd|ix.it|jdem.cz|jijr.com|kissa.be|kurl.us|litturl.com|lnkurl.com|memurl.com|metamark.net|miklos.dk|minilien.com|minurl.org|muhlink.com|myurl.in|myurl.us|notlong.com|ow.ly|plexp.com|poprl.com|qurlyq.com|redirx.com|s3nt.com|shorterlink.com|shortlinks.co.uk|short.to|shorturl.com|shrinklink.co.uk|shrinkurl.us|shrt.st|shurl.net|simurl.com|shorl.com|smarturl.eu|snipr.com|snipurl.com|snurl.com|sn.vc|starturl.com|surl.co.uk|tighturl.com|timesurl.at|tiny123.com|tiny.cc|tinylink.com|tinyurl.com|tobtr.com|traceurl.com|tr.im|tweetburner.com|twitpwr.com|twitthis.com|twurl.nl|u.mavrev.com|ur1.ca|url9.com|urlborg.com|urlbrief.com|urlcover.com|urlcut.com|urlhawk.com|url-press.com|urlsmash.com|urltea.com|urlvi.be|vimeo.com|wlink.us|xaddr.com|xil.in|xrl.us|x.se|xs.md|yatuc.com|yep.it|yweb.com|zi.ma|w3t.org)/.}
+  def self.is_shorturl? url
+    url.to_s =~ SHORTURL_RE
+  end
+end

data/examples/shorturls/shorturl_sequence.rb

@@ -0,0 +1,121 @@
+module Base62
+  # http://refactormycode.com/codes/125-base-62-encoding
+  BASE62_CHARS = ('0'..'9').to_a + ('a'..'z').to_a + ('A'..'Z').to_a
+  BASE62_MAP   = {}
+  BASE62_CHARS.zip((0..61).to_a){|ch,num| BASE62_MAP[ch]=num }
+  def self.i_to_s i
+    return '0' if i == 0
+    s = ''
+    while i > 0
+      s << BASE62_CHARS[i.modulo(62)]
+      i /= 62
+    end
+    s.reverse
+  end
+
+  def self.s_to_i str
+    i_out = 0
+    str.reverse.chars.each_with_index do |c, i|
+      i_out += BASE62_MAP[c] * (62 ** i)
+    end
+    i_out
+  end
+end
+
+module ShorturlSequence
+  def self.encode_integer i, radix
+    case radix.to_s
+    when '36' then i.to_s(36)
+    when '62' then Base62.i_to_s(i)
+    else
+      raise "Can't encode into base #{radix}"
+    end
+  end
+
+  def self.decode_str s, radix
+    s = s.gsub(%r{\W+$},'')
+    case radix.to_s
+    when '36' then s.to_i(36)
+    when '62' then Base62.s_to_i(i)
+    else
+      raise "Can't encode into base #{radix}"
+    end
+  end
+end
+
+class Shorturl
+  attr_accessor :base_url
+  attr_accessor :token
+  def initialize token
+    self.token    = token
+  end
+end
+
+class Shorturl62
+  def to_i
+    Base62.s_to_i token
+  end
+
+  def to_s
+    url
+  end
+
+  def url
+    "#{base_url}/#{token}"
+  end
+end
+
+class IsgdShorturl < Shorturl62
+  def base_url
+    'http://is.gd'
+  end
+end
+
+class Monkeyshines::Store::SequentialUrlStream
+  DEFAULT_MAX_URLSTR = '1zzzzz'.to_i(36)
+  DEFAULT_RADIX = {
+    'http://tinyurl.com/' => 36,
+    'http://bit.ly/'      => 62,
+    'http://is.gd/'       => 62,
+  }
+  attr_accessor :base_url, :min_limit,   :span,                        :encoding_radix
+  def initialize base_url,  min_limit=0,  max_limit=nil, encoding_radix=nil
+    self.base_url  = self.class.fix_url(base_url)
+    self.min_limit = min_limit.to_i
+    max_limit    ||= DEFAULT_MAX_URLSTR
+    self.span      = max_limit.to_i - self.min_limit
+    self.encoding_radix = (encoding_radix || DEFAULT_RADIX[self.base_url]).to_i
+    raise "Please specify either encoding_radix of 36 or 62" unless [36, 62].include?(self.encoding_radix)
+  end
+
+  def self.fix_url url
+    url = 'http://' + url unless (url[0..6]=='http://')
+    url = url + '/'       unless (url[-1..-1]=='/')
+    url
+  end
+
+  # An infinite stream of urls in range
+  def each *args, &block
+    (min_limit..max_limit).each(&block)
+  end
+
+  def self.new_from_command_line cmdline_opts, default_opts={}
+    options = default_opts.merge(cmdline_opts)
+    Trollop::die :base_url  if options[:base_url].blank?
+    self.new *options.values_of(:base_url, :min_limit, :max_limit, :encoding_radix)
+  end
+end
+
+class Monkeyshines::Store::RandomUrlStream < Monkeyshines::Store::SequentialUrlStream
+  # An infinite stream of urls in range
+  def each *args, &block
+    loop do
+      yield url_in_range
+    end
+  end
+
+  def url_in_range
+    idx = rand(span) + min_limit
+    base_url + ShorturlSequence.encode_integer(idx, encoding_radix)
+  end
+end

data/examples/shorturls/shorturl_start_tyrant.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+script_dir=`dirname $0`
+
+# nohup ttserver -port 10001 "$script_dir/distdb/shorturl_scrapes-tinyurl.tct#bnum=40000000#opts=l" 2>&1 >> log/ttserver-shorturl_scrapes-tinyurl+`date "+%Y%m%d"`.log &
+# nohup ttserver -port 10002 "$script_dir/distdb/shorturl_scrapes-bitly.tct#bnum=20000000#opts=l"   2>&1 >> log/ttserver-shorturl_scrapes-bitly+`date "+%Y%m%d"`.log &
+# nohup ttserver -port 10003 "$script_dir/distdb/shorturl_scrapes-other.tct#bnum=20000000#opts=l"   2>&1 >> log/ttserver-shorturl_scrapes-other+`date "+%Y%m%d"`.log &
+
+#
+# Start shorturl readthru cache TokyoTyrant servers
+#
+nohup ttserver -port 10042 "$script_dir/distdb/shorturl_reqs-tinyurl.tch#bnum=40000000#opts=l" 2>&1 >> log/ttserver-shorturl_reqs-tinyurl+`date "+%Y%m%d"`.log &
+nohup ttserver -port 10043 "$script_dir/distdb/shorturl_reqs-bitly.tch#bnum=20000000#opts=l"   2>&1 >> log/ttserver-shorturl_reqs-bitly+`date "+%Y%m%d"`.log &
+nohup ttserver -port 10044 "$script_dir/distdb/shorturl_reqs-other.tch#bnum=20000000#opts=l"   2>&1 >> log/ttserver-shorturl_reqs-other+`date "+%Y%m%d"`.log &
+
+# nohup ttserver -port 10069 "$script_dir/distdb/shorturl_reqs-foo.tch#bnum=40000000#opts=l" 2>&1 >> log/ttserver-shorturl_reqs-tinyurl+`date "+%Y%m%d"`.log &

data/examples/shorturls/start_shorturl_cache.sh

@@ -0,0 +1,2 @@
+script_dir=`dirname $0`
+ttserver -port 10040 $script_dir/work/distdb/shorturl_cache.tct >> $script_dir/work/log/shorturl_cache-`datename`.log 2>&1

data/lib/monkeyshines.rb

@@ -0,0 +1,31 @@
+require 'monkeyshines/extensions'
+require 'monkeyshines/utils/logger'
+require 'wukong'
+require 'wukong/extensions/pathname'
+require 'monkeyshines/utils/factory_module'
+require 'monkeyshines/utils/uri'
+require 'monkeyshines/utils/filename_pattern'
+require 'monkeyshines/options'
+require 'monkeyshines/scrape_request'
+
+module Monkeyshines
+  autoload :ScrapeRequest,         'monkeyshines/scrape_request'
+  autoload :ScrapeRequestCore,     'monkeyshines/scrape_request'
+  autoload :RequestStream,         'monkeyshines/request_stream'
+  autoload :Store,                 'monkeyshines/store'
+  autoload :Fetcher,               'monkeyshines/fetcher'
+  autoload :Monitor,               'monkeyshines/monitor'
+  autoload :Runner,                'monkeyshines/runner'
+  autoload :RawJsonContents,       'monkeyshines/scrape_request/raw_json_contents'
+
+  # Dumping ground for configuration values
+  CONFIG = {} unless defined?(CONFIG)
+
+end
+
+#
+# A convenient logger.
+#
+# Define NO_MONKEYSHINES_LOG (or define Log yourself) to prevent its creation
+#
+Log = Monkeyshines.logger unless (defined?(Log) || defined?(NO_MONKEYSHINES_LOG))

data/lib/monkeyshines/extensions.rb

@@ -0,0 +1,16 @@
+class Numeric
+  def clamp min, max
+    return min if min && (self <= min)
+    return max if max && (self >= max)
+    self
+  end
+end
+
+
+class Hash
+  def self.deep_sum *args
+    args.inject({}) do |result, options|
+      result.deep_merge options
+    end
+  end
+end

data/lib/monkeyshines/fetcher.rb

@@ -0,0 +1,10 @@
+module Monkeyshines
+  module Fetcher
+    extend FactoryModule
+    autoload :Base,            'monkeyshines/fetcher/base'
+    autoload :FakeFetcher,     'monkeyshines/fetcher/fake_fetcher'
+    autoload :HttpFetcher,     'monkeyshines/fetcher/http_fetcher'
+    autoload :HttpHeadFetcher, 'monkeyshines/fetcher/http_head_fetcher'
+
+  end
+end

data/lib/monkeyshines/fetcher/authed_http_fetcher.rb

@@ -0,0 +1,35 @@
+require 'net/http'
+Net::HTTP.version_1_2
+module Monkeyshines
+  module Fetcher
+
+    #
+    class AuthedHttpFetcher
+      cattr_accessor :auth_params
+
+      def get_request_token
+      end
+
+      def authorize
+      end
+
+      def get_access_token
+      end
+
+      def api_key
+      end
+      def api_secret
+      end
+      def session_key
+      end
+      
+      # authenticate request
+      def authenticate req
+        get_session_key unless session_key
+      end
+
+      
+    end
+
+  end
+end