monkeyshines 0.0.2

data/lib/monkeyshines/fetcher/base.rb

@@ -0,0 +1,44 @@
+module Monkeyshines
+  module Fetcher
+    #
+    # Base URL fetcher.
+    #
+    # Subclasses must provide
+    #   get(scrape_request)
+    # returning the same scrape_request with its contents, scraped_at and
+    # response fields appropriately filled in.
+    #
+    class Base
+      attr_accessor :options
+      #
+      # Options hash configures any subclass behavior
+      #
+      def initialize _options={}
+        self.options = _options
+      end
+
+      # Make request, return satisfied scrape_request
+      def get scrape_request
+      end
+
+      # inscribes request with credentials
+      def authenticate req
+      end
+
+      # Based on the response code, sleep (in case servers are overheating) and
+      # log response.
+      def backoff response
+        sleep
+      end
+
+      # A compact timestamp, created each time it's called
+      def self.timestamp
+        Time.now.utc.to_flat
+      end
+
+      # Release any persistent connections to the remote server
+      def close
+      end
+    end
+  end
+end

data/lib/monkeyshines/fetcher/fake_fetcher.rb

@@ -0,0 +1,19 @@
+module Monkeyshines
+  module Fetcher
+    FakeResponse = Struct.new(:code, :message, :body)
+
+    class FakeFetcher < Base
+
+      # Fake a satisfied scrape_request
+      def get scrape_request
+        response = FakeResponse.new('200', 'OK', { :fetched => scrape_request.url }.to_json )
+        scrape_request.response_code    = response.code
+        scrape_request.response_message = response.message
+        scrape_request.response         = response
+        scrape_request.scraped_at       = Time.now.utc.to_flat
+        scrape_request
+      end
+
+    end
+  end
+end

data/lib/monkeyshines/fetcher/http_fetcher.rb

@@ -0,0 +1,127 @@
+require 'net/http'
+Net::HTTP.version_1_2
+module Monkeyshines
+  module Fetcher
+    #
+    # Opens a persistent connection and makes repeated requests.
+    #
+    # * authentication
+    # * backoff and logging on client or server errors
+    #
+    class HttpFetcher < Base
+
+      #
+      # Notes:
+      #
+      # On HTTP:
+      # * "RFC 2616 (HTTP/1.1)":http://tools.ietf.org/html/rfc2616
+      # * "Header Fields":http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
+      # * "Notes on Keep-Alive":http://www.hpl.hp.com/personal/ange/archives/archives-95/http-wg-archive/1661.html
+      #
+      # * "right_http_connection is another HTTP lib":http://github.com/rightscale/right_http_connection/tree/master/lib/right_http_connection.rb
+
+      # amount to throttle non-persistent connections:
+      # if http is
+      CNXN_SLEEP_TIME = 0.5
+      # Default user agent presented to servers
+      USER_AGENT = "Net::HTTP #{RUBY_VERSION}"
+      attr_accessor :connection_opened_at, :username, :password, :http_req_options
+      def initialize _options={}
+        super _options
+        self.username = options[:username]
+        self.password = options[:password]
+        self.http_req_options = {}
+        self.http_req_options["User-Agent"] = options[:user_agent] || USER_AGENT
+        self.http_req_options["Connection"] = "keep-alive"
+      end
+
+      #
+      # Current session (starting a new one if necessary)
+      # If the host has changed, closes old conxn and opens new one
+      #
+      def http host, port=nil
+        return @http if (@http && (@http.started?) && (@host == host))
+        close        if (@http && (@http.started?) && (@host != host))
+        @host = host
+        @connection_opened_at = Time.now
+        Log.info "Opening HTTP connection for #{@host} at #{@connection_opened_at}"
+        @http = Net::HTTP.new(@host)
+        @http.set_debug_output($stderr) if options[:debug_requests]
+        @http.start
+      end
+
+      # Close the current session, if any
+      def close
+        if @http && @http.started?
+          @http.finish
+          Log.info "Closing HTTP connection for #{@host} from #{@connection_opened_at}"
+        end
+        @http = nil
+      end
+
+      # Build and dispatch request
+      def perform_request url_str
+        url = URI.parse(url_str)
+        req = Net::HTTP::Get.new(url.send(:path_query), http_req_options)
+        authenticate req
+        http(url.host, url.port).request req
+      end
+
+      # authenticate request
+      def authenticate req
+        req.basic_auth(username, password) if username && password
+      end
+
+      #
+      # Based on the response code, sleep (in case servers are overheating) and
+      # log response.
+      #
+      def backoff response
+        # backoff when server isn't persisting connection
+        sleep CNXN_SLEEP_TIME if (! @http.started?)
+        # Response-based sleep time
+        sleep_time = 0
+        case response
+        when Net::HTTPSuccess             then return          # 2xx
+        when Net::HTTPRedirection         then return          # 3xx
+        when Net::HTTPBadRequest          then sleep_time =  5 # 400 (rate limit, probably)
+        when Net::HTTPUnauthorized        then sleep_time =  0 # 401 (protected user, probably)
+        when Net::HTTPForbidden           then sleep_time =  4 # 403 update limit
+        when Net::HTTPNotFound            then sleep_time =  0 # 404 deleted
+        when Net::HTTPServiceUnavailable  then sleep_time =  9 # 503 Fail Whale
+        when Net::HTTPServerError         then sleep_time =  2 # 5xx All other server errors
+        else                              sleep_time = 1
+        end
+        Log.warn "Received #{response.code}, sleeping #{sleep_time} ('#{response.message[0..200].gsub(%r{[\r\n\t]}, " ")}' from #{@host}+#{@connection_opened_at})"
+        sleep sleep_time
+      end
+
+      # Make request, return satisfied scrape_request
+      def get scrape_request
+        begin
+          response = perform_request(scrape_request.url)
+          scrape_request.response_code    = response.code
+          scrape_request.response_message = response.message[0..200].gsub(/[\n\r\t]+/, ' ')
+          scrape_request.response         = response
+          backoff response
+        rescue StandardError, SignalException, Timeout::Error => e
+          Log.warn ["Recovering from fetcher error:", e.to_s, scrape_request.inspect[0..2000].gsub(/[\n\r]+/, ' ')].join("\t")
+          close # restart the connection
+        rescue Exception => e
+          Log.warn e
+          raise e
+        end
+        scrape_request.scraped_at = self.class.timestamp
+        scrape_request
+      end
+
+      def get_and_report_timing *args
+        start = Time.now.to_f
+        response = get *args
+        Log.info( Time.now.to_f - start )
+        response
+      end
+    end
+
+  end
+end

data/lib/monkeyshines/fetcher/http_head_fetcher.rb

@@ -0,0 +1,23 @@
+require 'monkeyshines/fetcher/http_fetcher'
+module Monkeyshines
+  module Fetcher
+    #
+    # Requests the HEAD only, for cases where you don't need to know actual page
+    # contents (e.g. you're looking for server info or scraping URL shorteners)
+    #
+    class HttpHeadFetcher < HttpFetcher
+
+      #
+      # Build and dispatch request
+      # We do a HEAD request only, no reason to get the body.
+      #
+      def perform_request url_str
+        url = URI.parse(url_str)
+        req = Net::HTTP::Head.new(url.send(:path_query), http_req_options)
+        authenticate req
+        http(url.host, url.port).request req
+      end
+
+    end
+  end
+end

data/lib/monkeyshines/monitor.rb

@@ -0,0 +1,7 @@
+module Monkeyshines
+  module Monitor
+    autoload :PeriodicMonitor, 'monkeyshines/monitor/periodic_monitor'
+    autoload :PeriodicLogger,  'monkeyshines/monitor/periodic_logger'
+  end
+end
+

data/lib/monkeyshines/monitor/chunked_store.rb

@@ -0,0 +1,23 @@
+require 'monkeyshines/monitor/periodic_monitor'
+module Monkeyshines
+  module Monitor
+    module ChunkedStore
+      attr_accessor :file_pattern
+      def initialize file_pattern
+        self.file_pattern = file_pattern
+        super file_pattern.make
+      end
+
+      def close_and_reopen
+        close
+        self.filename = file_pattern.make
+        dump_file
+      end
+
+      def save *args
+        chunk_monitor.periodically{ close_rename_and_open }
+        super *args
+      end
+    end
+  end
+end

data/lib/monkeyshines/monitor/periodic_logger.rb

@@ -0,0 +1,33 @@
+module Monkeyshines
+  module Monitor
+
+    #
+    # Emits a log line but only every +iter_interval+ calls or +time_interval+
+    # lapse.
+    #
+    # Since the contents of the block aren't called until the criteria are met,
+    # you can put relatively expensive operations in the log without killing
+    # your iteration time.
+    #
+    class PeriodicLogger < PeriodicMonitor
+      #
+      # Call with a block that returns a string or array to log.
+      # If you return
+      #
+      # Ex: log if it has been at least 5 minutes since last announcement:
+      #
+      #   periodic_logger = Monkeyshines::Monitor::PeriodicLogger.new(:time => 300)
+      #   loop do
+      #     # ... stuff ...
+      #     periodic_logger.periodically{ [morbenfactor, crunkosity, exuberance] }
+      #   end
+      #
+      def periodically &block
+        super do
+          result = [ "%10d"%iter, "%7.1f"%since, "%7.1f"%rate, (block ? block.call : nil) ].flatten.compact
+          Log.info result.join("\t")
+        end
+      end
+    end
+  end
+end

data/lib/monkeyshines/monitor/periodic_monitor.rb

@@ -0,0 +1,65 @@
+module Monkeyshines
+  module Monitor
+    #
+    # Accepts a lightweight call every iteration.
+    #
+    # Once either a time or an iteration criterion is met, executes the block
+    # and resets the timer until next execution.
+    #
+    # Note that the +time_interval+ is measured *excution to execution* and not
+    # in multiples of iter_interval. Say I set a time_interval of 300s, and
+    # happen to iterate at 297s and 310s after start.  Then the monitor will
+    # execute at 310s, and the next execution will happen on or after 610s.
+    #
+    # Also note that when *either* criterion is met, *both* criteria are
+    # reset. Say I set a time interval of 300s and an +iter_interval+ of 10_000;
+    # and that at 250s I reach iteration 10_000.  Then the monitor will execute
+    # on or after 20_000 iteration or 550s, whichever happens first.
+    #
+    class PeriodicMonitor
+      attr_accessor :time_interval, :iter_interval
+      attr_accessor :last_time, :iter, :started_at
+
+      def initialize options={}
+        self.started_at    = Time.now.utc.to_f
+        self.last_time     = started_at
+        self.iter          = 0
+        self.time_interval = options[:time]
+        self.iter_interval = options[:iters]
+      end
+
+      # True if more than +iter_interval+ has elapsed since last execution.
+      def enough_iterations?
+        iter % iter_interval == 0 if iter_interval
+      end
+
+      # True if more than +time_interval+ has elapsed since last execution.
+      def enough_time? now
+        (now - last_time) > time_interval if time_interval
+      end
+
+      # Time since monitor was created
+      def since
+        Time.now.utc.to_f - started_at
+      end
+      # Iterations per second
+      def rate
+        iter.to_f / since.to_f
+      end
+
+      #
+      # if the interval conditions are met, executes block; otherwise just does
+      # bookkeeping and returns.
+      #
+      def periodically &block
+        self.iter += 1
+        now       = Time.now.utc.to_f
+        if enough_iterations? || enough_time?(now)
+          block.call(iter, (now-last_time))
+          self.last_time = now
+        end
+      end
+    end
+
+  end
+end

data/lib/monkeyshines/options.rb

@@ -0,0 +1,59 @@
+require 'monkeyshines/utils/trollop'
+module Monkeyshines
+
+  CMDLINE_OPTIONS = [
+    [:handle,          "Identifying string for scrape",     { :type => String, :required => true } ],
+    [:source_filename, "URI for scrape store to load from", { :type => String                    } ],
+    [:dest_filename,   "Filename for results",              { :type => String                    } ],
+    [:log_dest,        "Log file location",                 { :type => String                    } ],
+  ]
+
+  #
+  # Load the YAML file ~/.monkeyshines
+  # and toss it into Monkeyshines::CONFIG
+  #
+  def self.load_global_options! *keys
+    all_defaults = YAML.load(File.open(ENV['HOME']+'/.monkeyshines'))
+    if keys.blank?
+      CONFIG.deep_merge! all_defaults
+    else
+      keys.each do |key|
+        CONFIG.deep_merge!( all_defaults[key] || {} )
+      end
+    end
+  end
+
+
+  def self.load_cmdline_options!
+    Monkeyshines::CONFIG.deep_merge! options_from_cmdline
+  end
+
+  #
+  # Takes the values set on the command line
+  # and merges them into the options hash:
+  #   --source-filename
+  # sets the value for options[:source][:filename], etc.
+  #
+  def self.options_from_cmdline
+    result = {}
+    cmdline = self.get_cmdline_args
+    cmdline.each do |key, val|
+      next if key.to_s =~ /_given$/
+      args = key.to_s.split(/_/).map(&:to_sym)+[val]
+      result.deep_set(*args) # if val
+    end
+    result[:handle] = result[:handle].to_s.gsub(/\W/,'').to_sym
+    result
+  end
+
+  # Use the trollop options defined in Monkeyshines::CMDLINE_OPTIONS
+  # to extract command-line args
+  def self.get_cmdline_args
+    cmdline = Trollop::Parser.new
+    Monkeyshines::CMDLINE_OPTIONS.each do |args|
+      cmdline.opt *args
+    end
+    Trollop::do_parse_args(cmdline)
+  end
+
+end

data/lib/monkeyshines/recursive_runner.rb

@@ -0,0 +1,26 @@
+require 'monkeyshines/runner'
+
+module Monkeyshines
+  class RecursiveRunner < Monkeyshines::Runner
+    GENERATION_LIMIT = 5
+
+    #
+    # Generate requests that ensue from this one
+    #
+    # if GENERATION_LIMIT is 5, requests at generation 4 *do* generate recursive
+    # jobs, ones at generation 5 do not (so, generation 6 shouldn't exist)
+    #
+    def bookkeep result
+      super result
+      if result
+        result.req_generation = result.req_generation.to_i
+        return if (result.req_generation >= GENERATION_LIMIT)
+        iter = 0
+        result.recursive_requests do |rec_req|
+          source.put rec_req
+        end
+      end
+    end
+
+  end
+end

data/lib/monkeyshines/repository/base.rb

@@ -0,0 +1,57 @@
+require 'right_aws'
+module Monkeyshines
+  module Repository
+    class Base
+
+      def exists?(key)
+      end
+      alias_method :include?, :exists?
+
+      def put key, val
+      end
+
+      def get key
+      end
+
+      def open
+      end
+
+      def close
+      end
+
+      def uri key
+      end
+
+      def md5 key
+        metadata key, :md5
+      end
+      alias_method :checksum, :md5
+
+      def size key
+      end
+
+      def timestamp key
+      end
+
+      # By default,
+      #     size+timestamp-md5
+      # Ex:
+      #     1251777182+20090222121200-577416a26499f6facf45973298be5276
+      def version_id
+        "#{size}+#{timestamp}-#{md5}"
+      end
+
+      CACHED_METADATA = {}
+      def metadata key, datum=nil
+        attrs = CACHED_METADATA[key] || get_metadata(key, datum)
+        datum ? attrs[datum] : attrs
+      end
+
+      # fetch
+      def get_metadata key, datum=nil
+        #
+      end
+
+    end
+  end
+end