monkeyshines 0.0.2

data/lib/monkeyshines/runner_core/options.rb

@@ -0,0 +1,5 @@
+module Monkeyshines
+  class Runner
+
+  end
+end

data/lib/monkeyshines/runner_core/parsing_runner.rb

@@ -0,0 +1,29 @@
+require 'yaml'
+require 'monkeyshines/runner_core/options'
+
+module Monkeyshines
+
+  #
+  # In general, you should
+  #
+  # But where an external library is alread providing cooked results or it's
+  # otherwise most straightforward to directly emit model objects, you can use
+  # a parsing runner
+  #
+  class ParsingRunner < Runner
+
+    #
+    # Fetch and store result
+    #
+    #
+    def fetch_and_store req
+      result = fetcher.get(req)       # do the url fetch
+      # results.each do |result|
+        result.parse do |obj|
+          dest.save(obj)
+        end
+    #end
+    end
+
+  end
+end

data/lib/monkeyshines/scrape_job/old_paginated.rb

@@ -0,0 +1,343 @@
+# require 'time'
+# module Monkeyshines
+#   #
+#   # Paginated lets you make repeated requests to collect a timeline or
+#   # collection of items.
+#   #
+#   # You will typically want to set the
+#   #
+#   # A Paginated-compatible ScrapeRequest should inherit from or be compatible
+#   # with +Monkeyshines::ScrapeRequest+ and additionally define
+#   # * [#items]  list of individual items in the response; +nil+ if there was an
+#   #   error, +[]+ if the response was well-formed but returned no items.
+#   # * [#num_items] number of items from this response
+#   # * [#span] the range of (typically) IDs within this scrape. Used to know when
+#   #   we've reached results from previous session
+#   #
+#   #
+#   module Paginated
+#     #
+#     # Generates request for each page to be scraped
+#     #
+#     # Block must return the fulfilled scrape_request response.  This response is
+#     # passed to +#acknowledge+ for bookkeeping, then the next request is
+#     # made.
+#     #
+#     # Scraping stops after max_pages requests or when is_last?(response, page)
+#     #
+#     def each_request pageinfo={}, &block
+#       begin_pagination!
+#       (1..hard_request_limit).each do |page|
+#         response = yield make_request(page, pageinfo)
+#         warn 'nil response' unless response
+#         acknowledge(response, page)
+#         break if is_last?(response, page)
+#       end
+#       finish_pagination!
+#     end
+#
+#     # Set up bookkeeping for pagination tracking
+#     def begin_pagination!
+#     end
+#
+#     # Finalize bookkeeping at conclusion of scrape_job.
+#     def finish_pagination!
+#     end
+#
+#     #
+#     # Feed back info from the scrape
+#     #
+#     def acknowledge response, page
+#     end
+#
+#     # return true if the next request would be pointless (true if, perhaps, the
+#     # response had no items, or the API page limit is reached)
+#     def is_last? response, page
+#       ( (page >= max_pages) ||
+#         (response && response.healthy? && (response.num_items < max_items)) )
+#     end
+#
+#     #
+#     # Soft limit on the number of pages to scrape.
+#     #
+#     # Typically, leave this set to the hard_request_limit if you don't know
+#     # beforehand how many pages to scrape, and override is_last? to decide when
+#     # to stop short of the API limit
+#     #
+#     def max_pages
+#       hard_request_limit
+#     end
+#
+#     # inject class variables
+#     def self.included base
+#       base.class_eval do
+#         # Hard request limit: do not in any case exceed this number of requests
+#         class_inheritable_accessor :hard_request_limit
+#         # max items per page, from API
+#         class_inheritable_accessor :max_items
+#         #
+#         # Span of items gathered in this scrape scrape_job.
+#         attr_accessor :sess_items, :sess_span, :sess_timespan
+#       end
+#     end
+#   end
+#
+#   module PaginatedTimeline
+#     # Soft limit on the number of pages to scrape.
+#     #
+#     # Typically, leave this set to the hard_request_limit if you don't know
+#     # beforehand how many pages to scrape, and override is_last? to decide when
+#     # to stop short of the API limit
+#     #
+#     def max_pages
+#       mp = fudge_factor * (n_items - prev_scraped_items) / max_items
+#       return 0 if mp == 0
+#       (mp+1).clamp(1, hard_request_limit).to_i
+#     end
+#     # inject class variables
+#     def self.included base
+#       base.class_eval do
+#         include Monkeyshines::Paginated
+#       end
+#     end
+#
+#     # #
+#     # # Threshold count-per-page and actual count to get number of expected pages.
+#     # # Cap the request with max
+#     # def pages_from_count per_page, count, max=nil
+#     #   num = [ (count.to_f / per_page.to_f).ceil, 0 ].max
+#     #   [num, max].compact.min
+#     # end
+#   end
+#
+#   #
+#   # Scenario: you request paginated search requests with a limit parameter (a
+#   # max_id or min_id, for example).
+#   #
+#   # * request successive pages,
+#   # * use info on the requested page to set the next limit parameter
+#   # * stop when max_pages is reached or a successful request gives fewer than
+#   #   max_items
+#   #
+#   #
+#   # The first
+#   #
+#   #    req?min_id=1234&max_id=
+#   #    => [ [8675, ...], ..., [8012, ...] ] # 100 items
+#   #    req?min_id=1234&max_id=8011
+#   #    => [ [7581, ...], ..., [2044, ...] ] # 100 items
+#   #    req?min_id=1234&max_id=2043
+#   #    => [ [2012, ...], ..., [1234, ...] ] #  69 items
+#   #
+#   # * The search terminates when
+#   # ** max_requests requests have been made, or
+#   # ** the limit params interval is zero,    or
+#   # ** a successful response with fewer than max_items is received.
+#   #
+#   # * You will want to save <req?min_id=8676&max_id=""> for later scrape
+#   #
+#   module PaginatedWithLimit
+#
+#     #
+#     # Return true if the next request would be pointless (true if, perhaps, the
+#     # response had no items, or the API page limit is reached)
+#     def is_last? response, page
+#       unscraped_span.empty? || super(response, page)
+#     end
+#
+#     # Set up bookkeeping for pagination tracking
+#     def begin_pagination!
+#       self.sess_items    ||= 0
+#       self.sess_span       = UnionInterval.new
+#       self.sess_timespan   = UnionInterval.new
+#       super
+#     end
+#
+#     def finish_pagination!
+#       # piw = [(prev_items.to_f ** 0.66), (max_items * hard_request_limit * 4.0)].min
+#       # puts ([Time.now.strftime("%M:%S"), "%-23s"%query_term] + [prev_rate, sess_rate, avg_rate, sess_timespan.size.to_f, prev_items, sess_items, piw, (1000/avg_rate)].map{|s| "%15.4f"%(s||0) }).join("\t") rescue nil
+#       self.prev_rate     = avg_rate
+#       if sess_items == (hard_request_limit * max_items)
+#         # bump the rate if we hit the hard cap:
+#         new_rate = [prev_rate * 1.25, 1000/120.0].max
+#         Log.info "Bumping rate on #{query_term} from #{prev_rate} to #{new_rate}"
+#         self.prev_rate = new_rate
+#       end
+#       self.prev_items    = prev_items.to_i + sess_items.to_i
+#       self.prev_span     = sess_span + prev_span
+#       self.new_items     = sess_items.to_i + new_items.to_i
+#       self.sess_items    = 0
+#       self.sess_span     = UnionInterval.new
+#       self.sess_timespan = UnionInterval.new
+#       super
+#     end
+#
+#     #
+#     # Feed back info from the scrape
+#     #
+#     def acknowledge response, page
+#       super response, page
+#       return unless response && response.items
+#       count_new_items response
+#       update_spans response
+#     end
+#
+#     # account for additional items
+#     def count_new_items response
+#       num_items = response.num_items
+#       # if there was overlap with a previous scrape, we have to count the items by hand
+#       prev_span = self.prev_span
+#       if prev_span.max && response.span && (response.span.min < prev_span.max)
+#         num_items = response.items.inject(0){|n,item| (prev_span.include? item['id']) ? n : n+1 }
+#       end
+#       self.sess_items += num_items
+#     end
+#
+#
+#     def sess_rate
+#       return nil if (!sess_timespan) || (sess_timespan.size == 0)
+#       sess_items.to_f / sess_timespan.size.to_f
+#     end
+#     #
+#     # How often an item rolls in, on average
+#     #
+#     def avg_rate
+#       return nil if (sess_items.to_f == 0 && (prev_rate.blank? || prev_items.to_f == 0))
+#       prev_weight = prev_items.to_f ** 0.66
+#       sess_weight = sess_items.to_f
+#       prev_weight = [prev_weight, sess_weight*3].min if sess_weight > 0
+#       weighted_sum = (
+#         (prev_rate.to_f * prev_weight) + # damped previous avg
+#         (sess_rate.to_f * sess_weight) ) # current avg
+#       rt = weighted_sum / (prev_weight + sess_weight)
+#       rt
+#     end
+#
+#
+#     # inject class variables
+#     def self.included base
+#       base.class_eval do
+#         attr_accessor :new_items
+#         # include Monkeyshines::Paginated
+#       end
+#     end
+#   end
+#
+# end
+#
+#
+# module Monkeyshines
+#   module Paginated
+#   end
+#
+#   module PaginatedTimeline
+#     # Soft limit on the number of pages to scrape.
+#     #
+#     # Typically, leave this set to the hard_request_limit if you don't know
+#     # beforehand how many pages to scrape, and override is_last? to decide when
+#     # to stop short of the API limit
+#     #
+#     def max_pages
+#       mp = fudge_factor * (n_items - prev_scraped_items) / max_items
+#       return 0 if mp == 0
+#       (mp+1).clamp(1, hard_request_limit).to_i
+#     end
+#     # inject class variables
+#     def self.included base
+#       base.class_eval do
+#         include Monkeyshines::Paginated
+#       end
+#     end
+#
+#     # #
+#     # # Threshold count-per-page and actual count to get number of expected pages.
+#     # # Cap the request with max
+#     # def pages_from_count per_page, count, max=nil
+#     #   num = [ (count.to_f / per_page.to_f).ceil, 0 ].max
+#     #   [num, max].compact.min
+#     # end
+#   end
+#
+#   module PaginatedWithRateAndLimit
+#
+#     def after_pagination
+#       # piw = [(prev_items.to_f ** 0.66), (max_items * hard_request_limit * 4.0)].min
+#       # puts ([Time.now.strftime("%M:%S"), "%-23s"%query_term] + [prev_rate, sess_rate, avg_rate, sess_timespan.size.to_f, prev_items, sess_items, piw, (1000/avg_rate)].map{|s| "%15.4f"%(s||0) }).join("\t") rescue nil
+#       self.prev_rate     = avg_rate
+#       if sess_items == (hard_request_limit * max_items)
+#         # bump the rate if we hit the hard cap:
+#         new_rate = [prev_rate * 1.25, 1000/120.0].max
+#         Log.info "Bumping rate on #{query_term} from #{prev_rate} to #{new_rate}"
+#         self.prev_rate = new_rate
+#       end
+#       self.prev_items    = prev_items.to_i + sess_items.to_i
+#       self.prev_span     = sess_span + prev_span
+#       self.new_items     = sess_items.to_i + new_items.to_i
+#       self.sess_items    = 0
+#       self.sess_span     = UnionInterval.new
+#       self.sess_timespan = UnionInterval.new
+#       super
+#     end
+#
+#     #
+#     # Feed back info from the scrape
+#     #
+#     def after_fetch response, page
+#       super response, page
+#       return unless response && response.items
+#       count_new_items response
+#       update_spans response
+#     end
+#
+#     # account for additional items
+#     def count_new_items response
+#       num_items = response.num_items
+#       # if there was overlap with a previous scrape, we have to count the items by hand
+#       prev_span = self.prev_span
+#       if prev_span.max && response.span && (response.span.min < prev_span.max)
+#         num_items = response.items.inject(0){|n,item| (prev_span.include? item['id']) ? n : n+1 }
+#       end
+#       self.sess_items += num_items
+#     end
+#
+#     def update_spans response
+#       # Update intervals
+#       self.sess_span     << response.span
+#       self.sess_timespan << response.timespan
+#     end
+#
+#     # gap between oldest scraped in this scrape_job and last one scraped in
+#     # previous scrape_job.
+#     def unscraped_span
+#       UnionInterval.new(prev_span_max, sess_span.min)
+#     end
+#     # span of previous scrape
+#     def prev_span
+#       @prev_span ||= UnionInterval.new(prev_span_min, prev_span_max)
+#     end
+#     def prev_span= min_max
+#       self.prev_span_min, self.prev_span_max = min_max.to_a
+#       @prev_span = UnionInterval.new(prev_span_min, prev_span_max)
+#     end
+#
+#     def sess_rate
+#       return nil if (!sess_timespan) || (sess_timespan.size == 0)
+#       sess_items.to_f / sess_timespan.size.to_f
+#     end
+#     #
+#     # How often an item rolls in, on average
+#     #
+#     def avg_rate
+#       return nil if (sess_items.to_f == 0 && (prev_rate.blank? || prev_items.to_f == 0))
+#       prev_weight = prev_items.to_f ** 0.66
+#       sess_weight = sess_items.to_f
+#       prev_weight = [prev_weight, sess_weight*3].min if sess_weight > 0
+#       weighted_sum = (
+#         (prev_rate.to_f * prev_weight) + # damped previous avg
+#         (sess_rate.to_f * sess_weight) ) # current avg
+#       rt = weighted_sum / (prev_weight + sess_weight)
+#       rt
+#     end
+#   end
+#
+# end

data/lib/monkeyshines/scrape_job/recursive.rb

@@ -0,0 +1,9 @@
+require 'time'
+require 'monkeyshines/utils/union_interval'
+module Monkeyshines
+  module ScrapeJob
+    module RecursesJobs
+
+    end
+  end
+end

data/lib/monkeyshines/scrape_request.rb

@@ -0,0 +1,136 @@
+require 'digest/md5'
+module Monkeyshines
+  def self.url_encode str
+    return '' if str.blank?
+    str = str.gsub(/ /, '+')
+    Addressable::URI.encode_component(str, Addressable::URI::CharacterClasses::UNRESERVED+'+')
+  end
+
+  def self.url_decode str
+    return '' if str.blank?
+    str = str.gsub(/\+/, ' ')
+    Addressable::URI.unencode(str)
+  end
+
+  XML_ENCODED_BADNESS = { "\r" => "
", "\n" => "
", "\t" => "	" }
+  #
+  # Takes an already-encoded XML string and replaces ONLY the characters in
+  # XML_ENCODED_BADNESS (by default, \r newline, \n carriage return and \t tab)
+  # with their XML encodings (
 and so forth).  Doesn't do any other
+  # encoding, and leaves exiting entities alone.
+  #
+  def self.scrub_xml_encoded_badness str
+    str.chomp.gsub(/[\r\n\t]/){|c| XML_ENCODED_BADNESS[c]}
+  end
+end
+
+module Monkeyshines
+  #
+  # Base class for Scrape requests
+  #
+  module ScrapeRequestCore
+
+    autoload :SignedUrl,          'monkeyshines/scrape_request/signed_url'
+    autoload :Paginated,          'monkeyshines/scrape_request/paginated'
+    autoload :Paginating,         'monkeyshines/scrape_request/paginated'
+    autoload :PaginatedWithLimit, 'monkeyshines/scrape_request/paginated'
+
+    def initialize *args
+      super *args
+      if (moreinfo.is_a?(String)) then self.moreinfo = JSON.load(moreinfo) rescue nil  end
+      make_url! if (! url)
+    end
+
+    def to_hash *args
+      hsh = super *args
+      if hsh['moreinfo'].is_a?(Hash)
+        hsh['moreinfo'] = moreinfo.to_json
+      end
+      hsh
+    end
+
+    def to_a *args
+      to_hash.values_of(*members).to_flat
+    end
+
+    #
+    def healthy?
+      (! url.blank?) && (           # has a URL and either:
+        scraped_at.blank?        || # hasn't been scraped,
+        (! response_code.blank?) || # or has, with response code
+        (! contents.blank?) )       # or has, with response
+    end
+
+    # Set URL from other attributes
+    def make_url!
+      self.url = make_url
+    end
+
+    def response= response
+      return unless response
+      self.contents = Monkeyshines.scrub_xml_encoded_badness(response.body)
+    end
+
+    def url_encode str
+      Monkeyshines.url_encode str
+    end
+
+    def key
+      Digest::MD5.hexdigest(self.url)
+    end
+
+    def req_generation= val
+      (self.moreinfo||={})['req_generation'] = val
+    end
+    def req_generation
+      (self.moreinfo||={})['req_generation']
+    end
+
+    # inject methods at class level
+    module ClassMethods
+      # Builds a URL query string from a hash of key,value pairs
+      #
+      # parameters are in sort order by encoded string
+      #
+      # Ex.
+      #   make_url_query( :foo => 'bar', :q => 'happy meal', :angle => 90 )
+      #   #=> "angle=90&foo=bar&q=happy%20meal"
+      #
+      def make_url_query hsh
+        hsh.map{|attr, val| "#{attr}=#{Monkeyshines.url_encode(val)}" }.sort.join("&")
+      end
+    end
+    def self.included base
+      base.class_eval do
+        include ClassMethods
+      end
+    end
+  end
+
+  class ScrapeRequest < TypedStruct.new(
+      [:identifier,       Integer],
+      [:page,             Integer],
+      [:moreinfo,         String],
+      [:url,              String],
+      [:scraped_at,       Bignum],
+      [:response_code,    Integer],
+      [:response_message, String],
+      [:contents,         String]
+      )
+    include ScrapeRequestCore
+  end
+
+  #
+  # A SimpleRequest just holds a URL and the fetch result.
+  #
+  class SimpleRequest < TypedStruct.new(
+    [:url,              String],
+    [:scraped_at,       Bignum],
+    [:response_code,    Integer],
+    [:response_message, String],
+    [:contents,         String]
+    )
+    include ScrapeRequestCore
+  end
+
+end