wuclan 0.2.0

data/examples/twitter/old/load_twitter_search_jobs.rb

@@ -0,0 +1,157 @@
+#!/usr/bin/env ruby
+$: << ENV['WUKONG_PATH']
+require File.dirname(__FILE__)+'/config/config_private'
+require 'rubygems'
+require 'trollop'
+require 'wukong'
+require 'monkeyshines'
+#
+require 'wuclan/twitter/scrape' ; include Wuclan
+require 'monkeyshines/utils/uri'
+require 'monkeyshines/fetcher/http_fetcher'
+#
+# Command line options
+#
+opts = Trollop::options do
+  opt :handle,              "Handle to uniquely identify this scrape",          :default => 'com.twitter.search'
+  opt :items_per_job,       "Desired item count per job",                       :default => 675
+  opt :min_resched_delay,   "Don't run jobs more often than this (in seconds)", :default => 30*1
+  opt :job_db,            "Tokyo tyrant db host",                             :default => ':1978', :type => String
+  opt :log,                 "Log file name; leave blank to use STDERR",         :type => String
+  # import from file
+  opt :from,           "Location of scrape store to load from",            :type => String
+  # output storage
+  opt :chunk_time,     "Frequency to rotate chunk files (in seconds)", :type => Integer, :default => 60*60*4
+  opt :dest_dir,       "Filename base to store output. e.g. --dump_basename=/data/ripd", :type => String
+  opt :dest_pattern,   "Pattern for dump file output",                 :default => ":dest_dir/:handle_prefix/:handle/:date/:handle+:datetime-:pid.tsv"
+end
+
+module Wuclan
+  module Domains
+    module Twitter
+      module Scrape
+        TwitterSearchJob = Struct.new(
+          :query_term,
+          :priority,
+          :prev_items,
+          :prev_rate,
+          :prev_span_min,
+          :prev_span_max
+          )
+      end
+    end
+  end
+end
+
+
+# Queue of request import_jobs, with reschedule requests
+beanstalk_tube  = opts[:handle].gsub(/\w+/,'_')
+request_queue   = Monkeyshines::RequestStream::BeanstalkQueue.new(nil, Twitter::Scrape::TwitterSearchJob, opts[:items_per_job], opts.slice(:min_resched_delay))
+# Scrape requests by HTTP
+fetcher         = Monkeyshines::Fetcher::HttpFetcher.new Monkeyshines::CONFIG[:twitter]
+# Log every 60 seconds
+periodic_log    = Monkeyshines::Monitor::PeriodicLogger.new(:time => 60)
+# Persist scrape_job jobs in distributed DB
+job_store       = Monkeyshines::Store::TyrantTdbKeyStore.new(opts[:job_db])
+
+# Import
+if opts[:from]
+  import_jobs = Monkeyshines::Store::FlatFileStore.new(opts[:from], :filemode => 'r')
+end
+
+#
+# Keep one unique copy of each scrape_job.  The most senior instance (the one
+# with the highest prev_items) wins.
+#
+SCRAPES = { }
+def add_scrape_job scrape_job
+  return if SCRAPES[scrape_job.query_term] &&
+    (SCRAPES[scrape_job.query_term].prev_items.to_i >= scrape_job.prev_items.to_i)
+  SCRAPES[scrape_job.query_term] = scrape_job
+end
+
+Monkeyshines::RequestStream::BeanstalkQueue.class_eval do
+  #
+  # An (extremely dangerous) routine to examine all the jobs in the queue--
+  # since I don't know another way we pull all of them out and then put all of
+  # them back in.
+  #
+  def scrub_all &block
+    job_queue.connect()
+    File.open("/tmp/qjobs-#{Time.now.strftime("%H%M%S")}.tsv", "w") do |dump|
+    loop do
+      # Kick a bunch of jobs across all connections
+      $stderr.puts job_queue_stats.inspect
+      kicked = job_queue.open_connections.map{|conxn| conxn.kick(20) }
+      kicked = kicked.inject(0){|sum, n| sum += n }
+      # For all the jobs we can get our hands on quickly,
+      while(qjob = reserve_job!(0.5)) do
+        # send it in for processing
+        scrape_job = scrape_job_from_qjob(qjob)
+        yield scrape_job
+        # last recourse in case something goes wrong.
+        dump << scrape_job.to_flat.join("\t")+"\n"
+        # and remove it from the pool
+        qjob.delete
+      end
+      # stop when there's no more qjobs
+      break if (job_queue_total_jobs == 0) && (!job_queue.peek_ready)
+    end
+    end
+  end
+end
+
+begin
+  #
+  # Catalog the jobs in the persistent store
+  #
+  job_store.each do |key, hsh|
+    scrape_job = Twitter::Scrape::TwitterSearchJob.from_hash hsh
+    periodic_log.periodically{ [scrape_job] }
+    add_scrape_job scrape_job
+  end
+
+  #
+  # Catalog the jobs in the transient queue
+  #
+  request_queue.scrub_all do |scrape_job|
+    periodic_log.periodically{ [scrape_job] }
+    add_scrape_job scrape_job
+  end
+
+  #
+  # Import jobs from a static file
+  #
+  import_jobs.each_as(Twitter::Scrape::TwitterSearchJob) do |scrape_job|
+    next if (scrape_job.query_term =~ /^#/) || (scrape_job.query_term.blank?)
+    periodic_log.periodically{ [scrape_job] }
+    add_scrape_job scrape_job
+    # SCRAPES[scrape_job.query_term].priority  = scrape_job.priority  unless scrape_job.priority.blank?
+    # SCRAPES[scrape_job.query_term] = scrape_job
+  end
+rescue Exception => e
+  warn e
+ensure
+  #
+  # Serialize them to disk
+  #
+  sorted = SCRAPES.sort_by{|term,scrape_job| [scrape_job.priority||65536, -(scrape_job.prev_rate||1440), term] }
+  sorted.each do |term, scrape_job|
+    # scrape_job.prev_rate = [scrape_job.prev_rate.to_f, 0.01].max if scrape_job.prev_rate
+    # scrape_job.prev_items = 1000
+    puts scrape_job.to_flat[1..-1].join("\t")
+  end
+end
+
+request_queue.min_resched_delay = 10
+sorted.each do |term, scrape_job|
+  #
+  # Persist the updated job to the job_store db, so that we can restart queue easily
+  job_store.save "#{scrape_job.class}-#{scrape_job.query_term}", scrape_job.to_hash.compact
+
+  #
+  # re-enqueue the job. If it's run before, accelerate its next call; if never
+  # run before schedule for immediate run.
+  delay = (scrape_job.prev_rate ? request_queue.delay_to_next_scrape(scrape_job)/3 : 0)
+  request_queue.save scrape_job, scrape_job.priority, delay
+end

data/examples/twitter/old/scrape_twitter_api.rb

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'monkeyshines'
+require 'monkeyshines/runner/base'
+require 'pathname'
+
+#
+require 'wuclan/twitter'
+# un-namespace request classes.
+include Wuclan::Twitter::Scrape
+
+WORK_DIR = Pathname.new(File.dirname(__FILE__)+"/rawd").realpath.to_s
+
+# ===========================================================================
+#
+# scrape_shorturls.rb --
+#
+# To scrape from a list of shortened urls:
+#
+#    ./shorturl_random_scrape.rb --from-type=FlatFileStore --from=request_urls.tsv
+#
+# To do a random scrape:
+#
+#    ./shorturl_random_scrape.rb --from-type=RandomUrlStream --base-url=tinyurl.com
+#       --base-url="http://tinyurl.com" --min-limit= --max-limit= --encoding_radix=
+#
+#
+opts = Trollop::options do
+  opt :log,            "Log to file instead of STDERR"
+  # input from file
+  opt :from,           "URI for scrape store to load from",            :type => String
+  opt :skip,           "Initial lines to skip",                        :type => Integer
+  # output storage
+  opt :cache_loc,      "URI for cache server",                         :type => String, :default => ':10022'
+  opt :chunk_time,     "Frequency to rotate chunk files (in seconds)", :type => Integer, :default => 60*60*4
+  opt :dest_dir,       "Filename base to store output. default ./work/ripd", :default => WORK_DIR+'/ripd'
+  opt :dest_pattern,   "Pattern for dump file output",                 :default => ":dest_dir/:date/:handle+:timestamp-:pid.tsv"
+end
+opts[:handle] ||= 'com.twitter'
+scrape_config = YAML.load(File.open(ENV['HOME']+'/.monkeyshines'))
+opts.merge! scrape_config
+
+# ******************** Log ********************
+if (opts[:log])
+  opts[:log] = (WORK_DIR+'/log/'+File.basename(opts[:from],'.tsv'))
+  $stdout = $stderr = File.open(opts[:log]+"-console.log", "a")
+end
+periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:iters => 1, :time => 30)
+
+#
+# ******************** Load from store ********************
+#
+class TwitterRequestStream < Monkeyshines::RequestStream::Base
+  def each *args
+    request_store.each(*args) do |twitter_user_id, *_|
+      yield TwitterUserRequest.new(twitter_user_id, 1, "" )
+    end
+  end
+end
+src_store = Monkeyshines::Store::FlatFileStore.new_from_command_line(opts, :filemode => 'r')
+src_store.skip!(opts[:skip].to_i) if opts[:skip]
+request_stream = TwitterRequestStream.new TwitterUserRequest, src_store
+
+#
+# ******************** Store output ********************
+#
+# Track visited URLs with key-value database
+#
+dest_cache = Monkeyshines::Store::TyrantRdbKeyStore.new(opts[:cache_loc])
+
+#
+# Store the data into flat files
+#
+dest_pattern = Monkeyshines::Utils::FilenamePattern.new(opts[:dest_pattern], :handle => opts[:handle], :dest_dir => opts[:dest_dir])
+dest_files   = Monkeyshines::Store::ChunkedFlatFileStore.new(dest_pattern, opts[:chunk_time].to_i, opts)
+
+#
+# Conditional store uses the key-value DB to boss around the flat files --
+# requests are only made (and thus data is only output) if the url is missing
+# from the key-value store.
+#
+dest_store = Monkeyshines::Store::ConditionalStore.new(dest_cache, dest_files)
+
+#
+# ******************** Fetcher ********************
+#
+fetcher = Monkeyshines::Fetcher::HttpFetcher.new opts[:twitter_api]
+
+
+#
+# ******************** Do this thing ********************
+#
+Log.info "Beginning scrape itself"
+request_stream.each do |req|
+  # conditional store only calls fetcher if url key is missing.
+  result = dest_store.set(req.url) do
+    response = fetcher.get(req)                             # do the url fetch
+    next unless response.healthy?                           # don't store bad fetches
+    [response.scraped_at, response]                         # timestamp into cache, result into flat file
+  end
+  periodic_log.periodically{ ["%7d"%dest_store.misses, 'misses', dest_store.size, req.response_code, result, req.url] }
+end
+dest_store.close
+fetcher.close

data/examples/twitter/old/scrape_twitter_search.rb

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+$: << ENV['WUKONG_PATH']
+require File.dirname(__FILE__)+'/config/config_private'
+require 'rubygems'
+require 'trollop'
+require 'wukong'
+require 'monkeyshines'
+require 'wuclan/twitter/scrape' ; include Wuclan
+
+require 'monkeyshines/fetcher/http_fetcher'
+require 'monkeyshines/utils/filename_pattern'
+#
+# Command line options
+#
+opts = Trollop::options do
+  opt :handle,              "Handle to uniquely identify this scrape",          :default => 'com.twitter.search'
+  opt :items_per_job,       "Desired item count per job",                       :default => 1000
+  opt :min_resched_delay,   "Don't run jobs more often than this (in seconds)", :default => 20*1
+  opt :job_db,              "Tokyo tyrant db host",                             :default => ':1978', :type => String
+  opt :log,                 "Log file name; leave blank to use STDERR",         :type => String
+  # output storage
+  opt :chunk_time,     "Frequency to rotate chunk files (in seconds)", :type => Integer, :default => 60*60*4
+  opt :dest_dir,       "Filename base to store output. e.g. --dump_basename=/data/ripd", :type => String
+  opt :dest_pattern,   "Pattern for dump file output",                         :default => Monkeyshines::Utils::FilenamePattern::DEFAULT_PATTERN_STR
+end
+Trollop::die :dest_dir unless opts[:dest_dir]
+
+# Queue of request scrape_jobs, with reschedule requests
+beanstalk_tube  = opts[:handle].gsub(/\w+/,'_')
+request_queue   = Monkeyshines::RequestStream::BeanstalkQueue.new(nil, Twitter::Scrape::TwitterSearchJob, opts[:items_per_job], opts.slice(:min_resched_delay))
+# Scrape Store for completed requests
+dest_pattern    = Monkeyshines::Utils::FilenamePattern.new(opts[:dest_pattern], opts.slice(:handle, :dest_dir))
+dest            = Monkeyshines::Store::ChunkedFlatFileStore.new dest_pattern, opts[:chunk_time].to_i
+# Scrape requests by HTTP
+fetcher         = Monkeyshines::Fetcher::HttpFetcher.new Monkeyshines::CONFIG[:twitter]
+# Log every 60 seconds
+periodic_log    = Monkeyshines::Monitor::PeriodicLogger.new(:time => 60)
+# Persist scrape_job jobs in distributed DB
+job_store       = Monkeyshines::Store::TyrantTdbKeyStore.new(opts[:job_db])
+
+request_queue.each do |scrape_job|
+  # Run through all pages for this search term
+  scrape_job.each_request do |req|
+    # Fetch request
+    response = fetcher.get(req)
+    # save it if successful
+    dest.save response if response
+    # log progress
+    periodic_log.periodically{ ["%7d"%response.num_items, response.url] }
+    # return it to the scrape_job for bookkeeping
+    response
+  end
+  # Persist the updated job to the scrape_jobs db, so that we can restart queue easily
+  job_store.save "#{scrape_job.class}-#{scrape_job.query_term}", scrape_job.to_hash.compact
+  # sleep 0.5
+end
+request_queue.close

data/examples/twitter/old/scrape_twitter_trending.rb

@@ -0,0 +1,73 @@
+#!/usr/bin/env ruby
+$: << ENV['WUKONG_PATH']
+require File.dirname(__FILE__)+'/config/config_private'
+require 'rubygems'
+require 'trollop'
+require 'wukong'
+require 'monkeyshines'
+require 'wuclan/twitter/scrape' ; include Wuclan
+
+require 'monkeyshines/fetcher/http_fetcher'
+require 'monkeyshines/utils/filename_pattern'
+
+#
+# Command line options
+#
+opts = Trollop::options do
+  opt :dumpfile_dir,        "Filename base to store output. e.g. --dump_basename=/data/ripd", :type => String
+  opt :dumpfile_pattern,    "Pattern for dump file output",
+    :default => Monkeyshines::Utils::FilenamePattern::DEFAULT_PATTERN_STR
+  opt :dumpfile_chunk_time, "Frequency to rotate chunk files (in seconds)", :type => Integer,
+    :default => 60*60*24
+  opt :handle,              "Handle to uniquely identify this scrape",
+    :default => 'com.twitter.search'
+  opt :min_resched_delay,   "Don't run jobs more often than this (in seconds)",
+    :default => 60*1
+end
+Trollop::die :dumpfile_dir unless opts[:dumpfile_dir]
+
+# Queue of request jobs, with reschedule requests
+# opts[:beanstalk_tube] ||= opts[:handle].gsub(/\w+/,'_')
+request_queue     = Monkeyshines::RequestStream::BeanstalkQueue.new(nil, Twitter::Scrape::TwitterSearchJob, opts[:items_per_job], opts.slice(:min_resched_delay)) # , :beanstalk_tube
+# Scrape Store for completed requests
+dumpfile_pattern  = Monkeyshines::Utils::FilenamePattern.new(opts[:dumpfile_pattern], opts.slice(:handle, :dumpfile_dir))
+store             = Monkeyshines::Store::ChunkedFlatFileStore.new dumpfile_pattern, opts[:dumpfile_chunk_time].to_i
+# Scrape requests by HTTP
+fetcher           = Monkeyshines::Fetcher::HttpFetcher.new Monkeyshines::CONFIG[:twitter]
+# Log every 60 seconds
+periodic_log      = Monkeyshines::Monitor::PeriodicLogger.new(:time => 60)
+
+
+class TwitterTrendingJob < Struct.new(
+    :query_term,
+    :priority,
+    :period
+    )
+
+end
+
+
+# %w[
+#   http://search.twitter.com/trends/current.format  ,    60*60
+#   http://search.twitter.com/trends/daily.json?date=2009-03-19
+#   http://search.twitter.com/trends/weekly.json?date=2009-03-19
+# ]
+
+
+
+request_queue.each do |scrape_job|
+  # Run through all pages for this search term
+  scrape_job.each_request do |req|
+    # Make request
+    response = fetcher.get(req)
+    # save it if successful
+    store.save response if response
+    # log progress
+    periodic_log.periodically{ ["%7d"%response.num_items, response.url] }
+    # return it to the scrape_job for bookkeeping
+    response
+  end
+end
+request_queue.finish
+
+# Twitter::Scrape::Scrape_Job.hard_request_limit = 15

data/examples/twitter/parse/parse_twitter_requests.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+#$: << ENV['WUKONG_PATH']
+require 'rubygems'
+require 'wukong'
+require 'monkeyshines'
+
+require 'wuclan/twitter'
+# if you're anyone but original author this next require is useless but harmless.
+require 'wuclan/twitter/scrape/old_skool_request_classes'
+# un-namespace request classes.
+include Wuclan::Twitter::Scrape
+include Wuclan::Twitter::Model
+
+#
+#
+# Instantiate each incoming request.
+# Stream out the contained classes it generates.
+#
+#
+class TwitterRequestParser < Wukong::Streamer::StructStreamer
+
+  def process request, *args, &block
+    request.parse(*args) do |obj|
+      next if obj.is_a? BadRecord
+      yield obj.to_flat(false)
+    end
+  end
+end
+
+#
+# We want to record each individual state of the resource, with the last-seen of
+# its timestamps (if there are many). So if we saw
+#
+#     rsrc  id   screen_name   followers_count  friends_count  (... more)
+#     user  23   skidoo        47               61
+#     user  23   skidoo        48               62
+#     user  23   skidoo        48               62
+#     user  23   skidoo        52               62
+#     user  23   skidoo        52               63
+#
+#
+class TwitterRequestUniqer < Wukong::Streamer::UniqByLastReducer
+  include Wukong::Streamer::StructRecordizer
+
+  attr_accessor :uniquer_count
+
+  #
+  #
+  #
+  #
+  # for immutable objects we can just work off their ID.
+  #
+  # for mutable objects we want to record each unique state: all the fields
+  # apart from the scraped_at timestamp.
+  #
+  def get_key obj
+    case obj
+    when Tweet
+      obj.id
+    when AFollowsB, AFavoritesB, ARepliesB, AAtsignsB, AAtsignsBId, ARetweetsB, ARetweetsBId, TwitterUserId
+      obj.key
+    when TwitterUser, TwitterUserProfile, TwitterUserStyle, TwitterUserPartial
+      [obj.id] + obj.to_a[2..-1]
+    else
+      raise "Don't know how to extract key from #{obj.class}"
+    end
+  end
+
+  def start! *args
+    self.uniquer_count = 0
+    super *args
+  end
+
+  def accumulate obj
+    self.uniquer_count      += 1
+    self.final_value = [self.uniquer_count, obj.to_flat].flatten
+  end
+end
+
+# This makes the script go.
+Wukong::Script.new(TwitterRequestParser, TwitterRequestUniqer).run

data/examples/twitter/parse/parse_twitter_search_requests.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+#$: << ENV['WUKONG_PATH']
+require 'rubygems'
+require 'wukong'
+require 'monkeyshines'
+
+require 'wuclan/twitter'
+require 'wuclan/twitter/scrape/twitter_search_request'
+require 'wuclan/twitter/parse/twitter_search_parse'
+include Wuclan::Twitter::Scrape
+
+#
+#
+# Instantiate each incoming request.
+# Stream out the contained classes it generates.
+#
+#
+class TwitterRequestParser < Wukong::Streamer::StructStreamer
+  def process request, *args, &block
+    request.parse(*args) do |obj|
+      next if obj.is_a? BadRecord
+      yield obj.to_flat(false)
+    end
+  end
+end
+
+# This makes the script go.
+Wukong::Script.new(TwitterRequestParser, nil).run

data/examples/twitter/scrape_twitter_api/scrape_twitter_api.rb

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'monkeyshines'
+require 'wuclan/twitter' ; include Wuclan::Twitter::Scrape
+
+#
+# * jobs stream from a flat file
+#
+# * each job generates one or several requests (ex: followers_ids,
+#   friends_ids, user_timeline, favorites). Paginated requests stop when results
+#   overlap the prev_max item, as tracked from a central store).
+#
+# * Each request is fetched with the standard HTTP fetcher.
+#
+# * Jobs are rescheduled based on the observed item rate
+#
+# * results are sent to a ChunkedFlatFileStore
+#
+
+#
+# Follow-on requests to make for each user
+# You can also specify these with --source-fetches on the command line
+#
+DEFAULT_SOURCE_FETCHES = [
+  :user,
+  # :followers_ids, :friends_ids,
+  :followers, :friends,
+  # :favorites
+]
+
+Monkeyshines::CMDLINE_OPTIONS << [:source_fetches, "Follow-on requests to make. Default '#{DEFAULT_SOURCE_FETCHES.join(',')}'", { :default => DEFAULT_SOURCE_FETCHES.join(',')  }]
+Monkeyshines::CMDLINE_OPTIONS << [:source_skip,    "Initial lines to skip",                                                     { :default => 1 }]
+
+# Setup
+WORK_DIR = Subdir[__FILE__,'work'].expand_path.to_s
+Monkeyshines.load_global_options!
+Monkeyshines.load_cmdline_options!
+Monkeyshines::CONFIG[:fetcher] = Monkeyshines::CONFIG[:twitter_api]
+
+#
+# Don't spend all day on follow-on requests
+#
+{ TwitterFollowersRequest => 10,
+  TwitterFriendsRequest   => 10,
+  TwitterFavoritesRequest => 4, }.each{|klass, limit| klass.hard_request_limit = limit }
+
+#
+# Set up scraper
+#
+scraper = Monkeyshines::Runner.new({
+    :log     => { :iters => 100, :dest => Monkeyshines::CONFIG[:handle] },
+    :source  => { :type  => TwitterRequestStream },
+    :dest    => { :type  => :chunked_flat_file_store, :rootdir => WORK_DIR },
+    # :fetcher => { :type => TwitterFakeFetcher },
+    :sleep_time  => 0,
+  })
+
+#
+# Run scraper
+#
+scraper.run

data/examples/twitter/scrape_twitter_api/seed.tsv

@@ -0,0 +1,4 @@
+infochimps
+AustinOnRails
+hadoop
+mrflip

data/examples/twitter/scrape_twitter_api/start_cache_twitter.sh

@@ -0,0 +1,2 @@
+script_dir=`dirname $0`
+ttserver -port 10022 $script_dir/rawd/distdb/twitter_api-`hostname`.tch >> $script_dir/rawd/log/twitter_api-ttserver-`datename`.log 2>&1