monkeyshines 0.0.2

data/lib/monkeyshines/scrape_request/paginated.rb

@@ -0,0 +1,290 @@
+require 'time'
+require 'monkeyshines/utils/union_interval'
+module Monkeyshines
+  module ScrapeRequestCore
+
+    #
+    # Paginated lets you make repeated requests to collect a timeline or
+    # collection of items.
+    #
+    # You will typically want to set the
+    #
+    # A Paginated-compatible ScrapeRequest should inherit from or be compatible
+    # with +Monkeyshines::ScrapeRequest+ and additionally define
+    # * [#items]  list of individual items in the response; +nil+ if there was an
+    #   error, +[]+ if the response was well-formed but returned no items.
+    # * [#num_items] number of items from this response
+    # * [#span] the range of (typically) IDs within this scrape. Used to know when
+    #   we've reached results from previous session
+    #
+    #
+    module Paginated
+      #
+      # Soft limit on the number of pages to scrape.
+      #
+      # If we know the max_total_items, use it to set the number of pages;
+      # otherwise, let it run up to the hard limit.
+      #
+      # Typically, use this to set an upper limit that you know beforehand, and
+      # use #is_last? to decide based on the results
+      #
+      def max_pages
+        return hard_request_limit if (!max_total_items)
+        (max_total_items.to_f / max_items).ceil.clamp(0, hard_request_limit)
+      end
+
+      # Number of items returned in this request
+      def num_items()
+        items ? items.length : 0
+      end
+
+      # inject class variables
+      def self.included base
+        base.class_eval do
+          # Hard request limit: do not in any case exceed this number of requests
+          class_inheritable_accessor :hard_request_limit
+
+          # max items per page the API might return
+          class_inheritable_accessor :max_items
+
+          # Total items in all requests, if known ahead of time -- eg. a
+          # twitter_user's statuses_count can be used to set the max_total_items
+          # for TwitterUserTimelineRequests
+          attr_accessor :max_total_items
+        end
+      end
+    end # Paginated
+
+    module Paginating
+      #
+      # Generates request for each page to be scraped.
+      #
+      # The job class must define a #request_for_page(page) method.
+      #
+      # * request is generated
+      # * ... and yielded to the call block. (which must return the fulfilled
+      #   scrape_request response.)
+      # * after_fetch method chain invoked
+      #
+      # Scraping stops when is_last?(response, page) is true
+      #
+      def each_request info=nil, &block
+        before_pagination()
+        (1..hard_request_limit).each do |page|
+          request  = request_for_page(page, info)
+          response = yield request
+          after_fetch(response, page)
+          break if is_last?(response, page)
+        end
+        after_pagination()
+      end
+
+      # return true if the next request would be pointless (true if, perhaps, the
+      # response had no items, or the API page limit is reached)
+      def is_last? response, page
+        ( (page >= response.max_pages) ||
+          (response && response.healthy? && partial_response?(response)) )
+      end
+      def partial_response? response
+        (response.num_items < response.max_items)
+      end
+
+      # Bookkeeping/setup preceding pagination
+      def before_pagination
+      end
+
+      # Finalize bookkeeping at conclusion of scrape_job.
+      def after_pagination
+      end
+
+      # Feed back info from the fetch
+      def after_fetch response, page
+      end
+
+      # inject class variables
+      def self.included base
+        base.class_eval do
+          # Hard request limit: do not in any case exceed this number of requests
+          class_inheritable_accessor :hard_request_limit
+        end
+      end
+    end # Paginating
+
+    #
+    # Scenario: you request paginated search requests with a limit parameter (a
+    # max_id or min_id, for example).
+    #
+    # * request successive pages,
+    # * use info on the requested page to set the next limit parameter
+    # * stop when max_pages is reached or a successful request gives fewer than
+    #   max_items
+    #
+    #
+    # The first
+    #
+    #    req?min_id=1234&max_id=
+    #    => [ [8675, ...], ..., [8012, ...] ] # 100 items
+    #    req?min_id=1234&max_id=8011
+    #    => [ [7581, ...], ..., [2044, ...] ] # 100 items
+    #    req?min_id=1234&max_id=2043
+    #    => [ [2012, ...], ..., [1234, ...] ] #  69 items
+    #
+    # * The search terminates when
+    # ** max_requests requests have been made, or
+    # ** the limit params interval is zero,    or
+    # ** a successful response with fewer than max_items is received.
+    #
+    # * You will want to save <req?min_id=8676&max_id=""> for later scrape
+    #
+    module PaginatedWithLimit
+      # Set up bookkeeping for pagination tracking
+      def before_pagination
+        self.started_at      = Time.now.utc
+        self.sess_span       = UnionInterval.new
+        self.sess_timespan   = UnionInterval.new
+        super
+      end
+
+      #
+      # Feed back info from the scrape
+      #
+      def after_fetch response, page
+        super response, page
+        update_spans(response) if (response && response.items)
+      end
+
+      # Update intervals to include new response
+      def update_spans response
+        self.sess_span     << response.span
+        self.sess_timespan << response.timespan
+      end
+
+      # Return true if the next request would be pointless (true if, perhaps, the
+      # response had no items, or the API page limit is reached)
+      def is_last? response, page
+        sess_span.include?(prev_max) || super(response, page)
+      end
+
+      def after_pagination
+        self.prev_max      = [prev_max, sess_span.max].compact.max
+        self.sess_span     = UnionInterval.new
+        self.sess_timespan = UnionInterval.new
+        super
+      end
+
+      # inject class variables
+      def self.included base
+        base.class_eval do
+          # Span of items gathered in this scrape scrape_job.
+          attr_accessor :sess_span, :sess_timespan, :started_at
+        end
+      end
+    end # PaginatedWithLimit
+
+    module PaginatedWithRate
+      def before_pagination
+        self.sess_items    ||= 0
+        super
+      end
+
+      #
+      # Feed back info from the scrape
+      #
+      def after_fetch response, page
+        super response, page
+        update_counts(response) if (response && response.items)
+        # p [response.items.map{|item| item['id']}.max, response.items.map{|item| item['id']}.min, prev_max, sess_span, response.parsed_contents.slice('max_id','next_page')]
+        # p response.items.map{|item| ("%6.2f" % [Time.now - Time.parse(item['created_at'])])}
+      end
+
+      # Count the new items from this response among the session items
+      def update_counts response
+        self.sess_items += response.num_items
+      end
+
+      RATE_PARAMETERS = {
+        :max_session_timespan  => (60 * 60 * 24 * 5), # 5 days
+        :default_scrape_period => (60 * 60 * 2     ), # 2 hours
+        :max_resched_delay     => (60 * 60 * 24 * 1), # 1 days
+        :min_resched_delay     => (5),                # 5 seconds
+        :sess_weight_slowing   => 0.35,  # how fast to converge when rate < average
+        :sess_weight_rising    => 1.0,   # how fast to converge when rate > average
+      }
+
+      #
+      # * session returns one result
+      # * session returns no result
+      # * session results clustered at center of nominal timespan
+      #
+      def recalculate_rate!
+        # If there's no good session timespan, we can fake one out
+        self.sess_timespan.max ||= Time.now.utc
+        self.sess_timespan.min ||= self.last_run
+        # Whatever its origin, limit the session timespan
+        if sess_timespan.size > RATE_PARAMETERS[:max_session_timespan]
+          sess_timespan.min = sess_timespan.max - RATE_PARAMETERS[:max_session_timespan]
+        end
+        # Find and limit the session items rate
+        if self.sess_items.to_f < 2
+          self.sess_items = 2
+          sess_items_rate = self.sess_items.to_f / RATE_PARAMETERS[:default_scrape_period]
+        else
+          # Find the items rate
+          sess_items_rate = self.sess_items.to_f / sess_timespan.size.to_f
+        end
+        # Find and limit the previous items rate
+        self.prev_items_rate = self.prev_items_rate.to_i rescue 0
+        if self.prev_items_rate == 0
+          self.prev_items_rate = target_items_per_job.to_f / RATE_PARAMETERS[:default_scrape_period]
+          self.delay           = RATE_PARAMETERS[:default_scrape_period].to_f
+        end
+
+        # New items rate is a weighted average of new and old
+        #
+        # If new rate is faster than the prev_rate, we use a high weight
+        # (~1.0). When
+        sess_wt         = (sess_items_rate > prev_items_rate) ? RATE_PARAMETERS[:sess_weight_rising] : RATE_PARAMETERS[:sess_weight_slowing]
+        new_items_rate  = (prev_items_rate + (sess_items_rate * sess_wt)) / (1.0 + sess_wt)
+        new_total_items = prev_items.to_i + sess_items.to_i
+        since_start     = (Time.now.utc - self.started_at).to_f
+        new_period      = (target_items_per_job / new_items_rate)
+        new_delay       = new_period - since_start
+
+        # puts %Q{rates %6.3f %6.3f => %6.3f delay %5.2f %5.2f => %5.2f (%5.2f) want %d sess %d items/%5.1fs -- %10d < %10d -- %s } %
+        #   [sess_items_rate, prev_items_rate, new_items_rate,
+        #   target_items_per_job / sess_items_rate, self.delay, new_period, new_delay,
+        #   target_items_per_job, sess_items, sess_timespan.size.to_f,
+        #   sess_span.max, prev_max,
+        #   self.key]
+
+        Log.info(
+          %Q{resched\tit %4d\t%7.3f\t%7.2f\t%7.2f\t%7.2f\t%7.2f\t%10d\t%s } %
+          [sess_items, sess_timespan.size.to_f, target_items_per_job / sess_items_rate, self.delay, new_period, new_delay, prev_max, self.key])
+
+        self.delay           = new_delay.to_f.clamp(RATE_PARAMETERS[:min_resched_delay], RATE_PARAMETERS[:max_resched_delay])
+        self.prev_items_rate = new_items_rate
+        self.prev_items      = new_total_items
+      end
+
+      #
+      # Recalculate the item rates
+      # using the accumulated response
+      #
+      def after_pagination
+        recalculate_rate!
+        self.sess_items    = 0
+        super
+      end
+
+      # inject class variables
+      def self.included base
+        base.class_eval do
+          # Span of items gathered in this scrape scrape_job.
+          attr_accessor  :sess_items
+          # How many items we hope to pull in for every job
+          cattr_accessor :target_items_per_job
+        end
+      end
+    end # PaginatedWithRate
+  end
+end

data/lib/monkeyshines/scrape_request/raw_json_contents.rb

@@ -0,0 +1,16 @@
+require 'json'
+module Monkeyshines
+  module RawJsonContents
+    def parsed_contents
+      return @parsed_contents if @parsed_contents
+      return nil unless contents
+      begin
+        @parsed_contents = JSON.load(contents.to_s)
+      rescue Exception => e
+        warn "JSON not parsing : #{e.to_s[0..2000].gsub(/[\r\n]+/,"")}" ; return nil
+      end
+      @parsed_contents
+    end
+
+  end
+end

data/lib/monkeyshines/scrape_request/signed_url.rb

@@ -0,0 +1,86 @@
+module Monkeyshines
+  module ScrapeRequestCore
+    module SignedUrl
+
+      def sign_url parsed_uri, request_key
+        qq = parsed_uri.query_values || {}
+        qq.merge!(request_key)
+        qq.merge!(
+          'api_key'     => api_key,
+          'nonce'       => nonce,
+          'format'      => 'json')
+        p qq
+        qq = qq.sort.map{|k,v| k+'='+v }
+        str = [ parsed_uri.path, qq, api_secret].flatten.join("")
+        sig = Digest::MD5.hexdigest(str)
+        [qq, sig]
+      end
+
+      def authed_url(url, request_key)
+        parsed_uri = Addressable::URI.parse(url)
+        qq, sig = sign_url(parsed_uri, request_key)
+        [parsed_uri.scheme, '://', parsed_uri.host, parsed_uri.path, '?', qq.join("&"), "&sig=#{sig}"].join("")
+      end
+
+      def nonce
+        Time.now.utc.to_f.to_s
+      end
+
+      def token_request_url
+        "http://api.friendster.com/v1/token?api_key=#{api_key}&nonce=#{nonce}&format=json"
+      end
+    end
+  end
+end
+
+#
+# class TokenRequest < Base
+#   def authed_url
+#     qq = parsed_uri.query_values.merge(
+#       'api_key' => api_key,
+#       'nonce' => nonce,
+#       # 'auth_token' => auth_token,
+#       'format' => 'json').sort.map{|k,v| k+'='+v }
+#     p qq
+#     str = [
+#       parsed_uri.path,
+#       qq,
+#       api_secret].flatten.join("")
+#     p str
+#     sig = Digest::MD5.hexdigest(str)
+#     qq << "sig=#{sig}"
+#     au = [parsed_uri.scheme, '://', parsed_uri.host, parsed_uri.path, '?', qq.join("&")].join("")
+#     p au
+#     au
+#   end
+# end
+#
+# class SessionRequest < Base
+#   def authed_url(auth_token)
+#     qq = parsed_uri.query_values.merge(
+#       'api_key' => api_key,
+#       'nonce' => nonce,
+#       'auth_token' => auth_token,
+#       'format' => 'json').sort.map{|k,v| k+'='+v }
+#     p qq
+#     str = [
+#       parsed_uri.path,
+#       qq,
+#       api_secret].flatten.join("")
+#     p str
+#     sig = Digest::MD5.hexdigest(str)
+#     qq << "sig=#{sig}"
+#     au = [parsed_uri.scheme, '://', parsed_uri.host, parsed_uri.path, '?', qq.join("&")].join("")
+#     p au
+#     au
+#   end
+#   def make_url()
+#     "http://api.friendster.com/v1/session?"
+#   end
+# end
+#
+# # require 'monkeyshines' ; require 'wuclan' ; require 'wukong' ; require 'addressable/uri' ; require 'rest_client' ; scrape_config = YAML.load(File.open(ENV['HOME']+'/.monkeyshines'))
+# # load(ENV['HOME']+'/ics/wuclan/lib/wuclan/friendster/scrape/base.rb') ; Wuclan::Friendster::Scrape::Base.api_key = scrape_config[:friendster_api][:api_key] ; tokreq = Wuclan::Friendster::Scrape::TokenRequest.new(scrape_config[:friendster_api][:user_id]) ; tok= RestClient.post(tokreq.authed_url, {}).gsub(/\"/,"")
+# # sessreq = Wuclan::Friendster::Scrape::SessionRequest.new(scrape_config[:friendster_api][:user_id])
+# # sessreq.auth_token = '' ; sessreq.make_url! ; RestClient.post(sessreq.url+'&sig='+sessreq.url_sig[1], {})
+# # # => "{"session_key":"....","uid":"...","expires":"..."}"

data/lib/monkeyshines/store.rb

@@ -0,0 +1,14 @@
+module Monkeyshines
+  module Store
+    extend FactoryModule
+    autoload :Base,                 'monkeyshines/store/base'
+    autoload :FlatFileStore,        'monkeyshines/store/flat_file_store'
+    autoload :ConditionalStore,     'monkeyshines/store/conditional_store'
+    autoload :ChunkedFlatFileStore, 'monkeyshines/store/chunked_flat_file_store'
+    autoload :KeyStore,             'monkeyshines/store/key_store'
+    autoload :TokyoTdbKeyStore,     'monkeyshines/store/tokyo_tdb_key_store'
+    autoload :TyrantTdbKeyStore,    'monkeyshines/store/tyrant_tdb_key_store'
+    autoload :TyrantRdbKeyStore,    'monkeyshines/store/tyrant_rdb_key_store'
+    autoload :ReadThruStore,        'monkeyshines/store/read_thru_store'
+  end
+end

data/lib/monkeyshines/store/base.rb

@@ -0,0 +1,29 @@
+module Monkeyshines
+  module Store
+    class Base
+      attr_accessor :options
+      def initialize _options={}
+        self.options = _options
+        Log.info "Creating #{self.class}"
+      end
+
+      #
+      def each_as klass, &block
+        self.each do |*args|
+          begin
+            item = klass.new *args[1..-1]
+          rescue Exception => e
+            Log.info [args, e.to_s, self].join("\t")
+            raise e
+          end
+          yield item
+        end
+      end
+
+      def log_line
+        nil
+      end
+
+    end
+  end
+end

data/lib/monkeyshines/store/chunked_flat_file_store.rb

@@ -0,0 +1,37 @@
+module Monkeyshines
+  module Store
+    class ChunkedFlatFileStore < Monkeyshines::Store::FlatFileStore
+      attr_accessor :filename_pattern, :chunk_monitor, :handle
+
+      DEFAULT_OPTIONS = {
+        :chunktime    => 4*60*60, # default 4 hours
+        :pattern   => ":rootdir/:date/:handle+:timestamp-:pid.tsv",
+        :rootdir   => nil,
+        :filemode  => 'w',
+      }
+
+      def initialize _options
+        self.options = DEFAULT_OPTIONS.deep_merge(_options)
+        raise "You don't really want a chunk time this small: #{options[:chunktime]}" unless options[:chunktime] > 600
+        self.chunk_monitor    = Monkeyshines::Monitor::PeriodicMonitor.new( :time => options[:chunktime] )
+        self.handle           = options[:handle] || Monkeyshines::CONFIG[:handle]
+        self.filename_pattern = Monkeyshines::Utils::FilenamePattern.new(options[:pattern], :handle => handle, :rootdir => options[:rootdir])
+        super options.merge(:filename => filename_pattern.make())
+        self.mkdir!
+      end
+
+      def save *args
+        result = super *args
+        chunk_monitor.periodically do
+          new_filename = filename_pattern.make()
+          Log.info "Rotating chunked file #{filename} into #{new_filename}"
+          self.close
+          @filename = new_filename
+          self.mkdir!
+        end
+        result
+      end
+
+    end
+  end
+end