get_them_all 1.0.0

data/lib/get_them_all/extensions/gauge_display.rb

@@ -0,0 +1,155 @@
+
+module GetThemAll
+  ##
+  # This extension will show you a progress bar for each worker
+  # showing its current state:
+  # ~ = working
+  # D = url downloaded
+  # E = url examined
+  # . = url skipped
+  # x = url download failed
+  # 
+  # The extension knowns how to handle the terminal width nicely, if
+  # it reachs the right end of the terminal the line will scroll
+  # (characters will disappear on the left while new appear on the right)
+  # 
+  class GaugeDisplay < Extension
+    module Cursor
+      class << self
+        def up(n);        print "\e[#{n}A" if n > 0; end
+        def down(n);      print "\e[#{n}B" if n > 0; end
+        def right(n);     print "\e[#{n}C" if n > 0; end
+        def left(n);      print "\e[#{n}D" if n > 0; end
+
+        def col(n);       print "\e[#{n}G"; end
+        def clear_line;   print "\e[0K";    end
+
+        # save / restore
+        def save;         print "\e[s";     end
+        def restore;      print "\e[u";     end
+
+        # hide / show
+        def hide_cursor;  print "\e[?25l";  end
+        def show_cursor;  print "\e[?25h";  end
+
+        def screen_width
+          `tput cols`.strip.to_i
+        end
+      end
+    end
+  
+    def initialize
+      register_handler('downloader.started', &method(:downloader_started))
+      # examine actions
+      register_handler('action.examine.started', &method(:work_started))
+      register_handler('action.examine.success', &method(:work_completed))
+      register_handler('action.examine.failure', &method(:work_failed))
+      register_handler('action.examine.skipped', &method(:work_skipped))
+    
+      # download actions
+      register_handler('action.download.started', &method(:work_started))
+      register_handler('action.download.success', &method(:work_completed))
+      register_handler('action.download.failure', &method(:work_failed))
+      register_handler('action.download.skipped', &method(:work_skipped))
+    end
+  
+
+    def downloader_started(event_name, downloader)
+      # initialize the screen
+      @examiners = downloader.examiners_count
+      @downloaders = downloader.downloaders_count
+    
+      # save cursor position before doing anything
+      Cursor.save()
+    
+      # store screen state
+      @state = []
+      (@examiners + @downloaders).times do |n|
+        puts "#{n} "
+        @state[n] = "#{n} "
+      end
+    end
+  
+    def work_started(event_name, worker, action, *args)
+      Cursor.restore()
+      line = worker_line(worker)
+    
+      # move to the correct line
+      Cursor.down( line )
+    
+      # and column
+      Cursor.right(@state[line].size)
+    
+      print "~"
+    end
+  
+    def update_line(line, added_str)
+      # update internal state
+      state = @state[line]
+      state << added_str
+    
+      # resize line if required
+      if state.size > Cursor.screen_width
+        header = state[0,2]
+        size = Cursor.screen_width - header.size
+        state = header + state[-size..-1]
+      end
+    
+      # move to column 0
+      Cursor.col(0)
+      # erase entire line
+      Cursor.clear_line()
+    
+      print state
+    end
+  
+  
+    def work_completed(event_name, worker, action, *args)
+      Cursor.restore()
+      line = worker_line(worker)
+    
+      Cursor.down( line )
+      #Cursor.right(@state[line].size)
+    
+      if worker.type == :examiner
+        update_line(line, "E")
+      else
+        update_line(line, "D")
+      end
+    end
+  
+  
+    def work_failed(event_name, worker, action, *args)
+      Cursor.restore()
+      line = worker_line(worker)
+    
+      # move to the correct line
+      Cursor.down( line )
+    
+      # and column
+      # Cursor.right(2 + @state[line].size)
+    
+      update_line(line, "x")
+    end
+  
+  
+    def work_skipped(event_name, worker, action, *args)
+      Cursor.restore()
+      line = worker_line(worker)
+    
+      # move to the correct line
+      Cursor.down( line )
+    
+      # and column
+      # Cursor.right(2 + @state[line].size)
+    
+      update_line(line, ".")
+    end
+  
+  private
+    def worker_line(worker)
+      base_index = (worker.type == :examiner) ? 0 : @examiners
+      base_index + worker.index
+    end
+  end
+end

data/lib/get_them_all/extensions/graph_builder.rb

@@ -0,0 +1,138 @@
+
+module GetThemAll
+  ##
+  # This extension will generate a textfile showing
+  # he hierarchy of the site which may be considered
+  # as a map.
+  # 
+  # Its main purpose as a debugging tool to have a
+  # better view of what was going on inside.
+  # 
+  # It was also capable of generating a dot file but this code
+  # is not up to date.
+  # 
+  class GraphBuilder < Extension
+    class TreeNode
+      attr_reader :text, :children
+
+      def initialize(prefix, text)
+        @prefix = prefix
+        @text = text
+        @children = []
+      end
+
+      def add_child(prefix, text)
+        ret = TreeNode.new(prefix, text)
+        @children << ret
+        ret
+      end
+
+      def find_node(text)
+        if @text == text
+          self
+        elsif !@children.empty?
+          @children.map{|node| node.find_node(text) }.compact.first
+        else
+          nil
+        end
+      end
+
+      def inspect
+        "{#{@text}}"
+      end
+
+      def dump_to_file(path)
+        File.open(path, 'w') do |f|
+          f.write( dump_node(0) )
+        end
+      end
+
+      def dump_node(level)
+        ret = "#{level-1} #{'  ' * level}#{@prefix} #{@text}\n"
+        @children.each do |node|
+          ret << node.dump_node(level + 1)
+        end
+        ret
+      end
+      protected :dump_node
+
+
+
+
+      def dump_to_dot_file(path)
+        File.open(path, 'w') do |f|
+          f.write(<<-EOF)
+    digraph toto {
+      node [shape=box];
+      #{dump_node_dot()}
+    }      
+          EOF
+        end
+      end
+
+      def dump_node_dot(prefix = nil)
+        # ret = "#{level-1} #{'  ' * level}#{@prefix} #{@text}\n"
+        if prefix.nil?
+          prefix = "\"#{@text}\""
+        else
+          prefix = " #{prefix} -> \"#{@text}\""
+        end
+
+        ret = [prefix]
+
+        @children.each do |node|
+          ret << node.dump_node("\"#{@text}\"")
+        end
+
+        ret.join("\n")
+      end
+      protected :dump_node_dot
+
+    end
+
+
+
+
+    ##
+    # @param [String] path where the resulting file will be written.
+    # 
+    def initialize(path)
+      @path = path
+    
+      register_handler('downloader.started') do |name, downloader|
+        @graph = TreeNode.new("", downloader.base_url)
+      end
+    
+      register_handler('downloader.completed') do |name, worker, downloader|
+        @graph.dump_to_file(@path)
+        # @graph.dump_to_dot_file(@path)
+      end
+    
+      register_handler('action.download.success') do |name, worker, action|
+        add_to_graph("D", action.url, action.parent_url)
+      end
+    
+      register_handler('action.examine.success') do |name, worker, action, returned_actions|
+        add_to_graph("E[ret:#{returned_actions.size}]", action.url, action.parent_url)
+      end
+    
+      register_handler('action.examine.failure') do |name, worker, action, error_status|
+        add_to_graph(error_status, action.url, action.referer)
+      end
+    
+    end
+  
+  private
+
+    def add_to_graph(prefix, url, referer)
+      if referer.nil?
+        @graph.add_child(prefix, url)
+      else
+        # find referer
+        parent = @graph.find_node(referer) or fail("node not found: #{referer}")
+        parent.add_child(prefix, url)
+      end
+    end
+  
+  end
+end

data/lib/get_them_all/history.rb

@@ -0,0 +1,23 @@
+module GetThemAll
+  class History
+    def initialize(data = [])
+      @data = data
+    end
+    
+    def load(data)
+      @data = data.split("\n")
+    end
+  
+    def dump
+      @data.join("\n")
+    end
+    
+    def include?(line)
+      @data.include?(line)
+    end
+    
+    def add(url)
+      @data << url
+    end
+  end
+end

data/lib/get_them_all/javascript_loader.rb

@@ -0,0 +1,74 @@
+
+require 'v8'
+
+module GetThemAll
+  class JavascriptLoader
+    
+    ##
+    # The goal is just to have something acting like a dom
+    # from the outside
+    class DOM
+      def search; self; end
+      def substr(*); ""; end
+      def protocol; 'http'; end
+    
+      def location; self; end
+      def appendChild(*); self; end
+      def createElement(*); self; end
+      def window; self; end
+    
+      def getElementsByTagName(*); [self]; end
+    
+      def ready(f)
+        f.call()
+      end
+    
+      def click(*); end
+      def change(*); end
+      def keydown(*); end
+      def keyup(*); end
+      def setInterval(*); end
+      def setTimeout(*); end
+      def attr(*); self; end
+      def text(*); self; end
+      def animate(*); self; end
+      def empty(*); self; end
+      def hide(*); self; end
+      def show(*); self; end
+      def focus(*); self; end
+    end
+  
+    class JQuery
+    
+    end
+  
+    def initialize(source)
+      @context = V8::Context.new do |ctx|
+        ctx[:document] = DOM.new()
+        ctx[:window] = DOM.new()
+        ctx[:jQuery] = JQuery.new
+        ctx['setInterval'] = ctx[:window].method(:setInterval)
+        ctx['setTimeout'] = ctx[:window].method(:setTimeout)
+      end
+    
+      @context.eval(%{
+        $ = function(){
+          return document;
+        };
+      
+        $.cookie = function(){};
+        $.each = function(){};
+      
+        // hash method cannot be defined on ruby side...
+        window.location.hash = window;
+      })
+    
+      @context.eval(source)
+    end
+  
+    def eval(str)
+      @context.eval(str)
+    end
+  
+  end
+end

data/lib/get_them_all/logger.rb

@@ -0,0 +1,35 @@
+require 'logger'
+
+module GetThemAll
+  class SiteDownloaderLogger < Logger
+    def format_message(level, time, progname, msg)
+      "[#{level} -- #{time.strftime("%Y/%m/%d %H:%M:%S")}] #{msg}\n"
+    end
+  end
+
+  class SiteDownloader
+    cattr_accessor :logger
+  
+    self.logger= SiteDownloaderLogger.new(STDOUT)
+    self.logger.level = Logger::ERROR
+  end
+end
+
+# helpers
+def debug(msg, short_msg = nil)
+  GetThemAll::SiteDownloader::logger.debug(msg)
+end
+
+def info(msg)
+  GetThemAll::SiteDownloader::logger.info(msg)
+end
+
+def warn(msg)
+  GetThemAll::SiteDownloader::logger.warn(msg)
+end
+
+def error(msg, fatal = false)
+  GetThemAll::SiteDownloader::logger.error(msg)
+  exit(1) if fatal
+end
+

data/lib/get_them_all/notifier.rb

@@ -0,0 +1,7 @@
+module GetThemAll
+  module Notifier
+    def notify(name, *args)
+      ActiveSupport::Notifications.publish(name, *args)
+    end
+  end
+end

data/lib/get_them_all/site_downloader.rb

@@ -0,0 +1,332 @@
+
+require 'addressable/uri'
+require 'active_support/hash_with_indifferent_access'
+
+require 'em-http-request'
+require 'em-priority-queue'
+
+module GetThemAll
+  
+  ##
+  # The main class, all your crawlers will derive from this class
+  # see examples/standalone.rb file for an example.
+  # 
+  class SiteDownloader
+    include Notifier
+    
+    class_attribute :examiners_count, :downloaders_count
+    class_attribute :config
+  
+    ##
+    # Determine what will be stored in the history file,
+    # the default is to store the last url before the download
+    # so we can ignore it sooner next time.
+    # 
+    # The other mode is :download, in this mode the download
+    # url itself will be stored, it is meant for special cases as
+    # the default should work better most of the time.
+    # 
+    class_attribute :history_tracking
+  
+    self.examiners_count = 1
+    self.downloaders_count = 1
+  
+    self.history_tracking = :default
+  
+    attr_reader :base_url, :storage, :history
+    
+    ##
+    # Create and start the crawler.
+    # 
+    # @param [Hash] args arguments
+    # @option args [String] :base_url The root url, every other
+    #   url will be relative to this.
+    # @option args [String] :start_url What is the very first url
+    #   to examine (level 0) relative to base_url, default is "/"
+    # @option args [String] :folder_name The root path where
+    #   downloaded files will be saved (appended to the storage root).
+    # @option args [Array] :extensions Array of Extension object.
+    # 
+    # @option args [Hash] :storage Configure storage backend
+    #   :type is the backend name
+    #   :params is a hash with backend specific options
+    # 
+    def initialize(args)
+      @cookies = []
+      @history = []
+      @connections = {}
+      @examine_queue = EM::PriorityQueue.new
+      @download_queue = EM::PriorityQueue.new
+      @history = History.new
+      
+      @base_url= args.delete(:base_url)
+      @start_url = args.delete(:start_url) || '/'
+      @folder_name= args.delete(:folder_name)
+      
+      # keep a pointer to each extension
+      @extensions = args.delete(:extensions) || [ActionLogger]
+      
+      storage_options = args.delete(:storage)
+      raise "storage required" unless storage_options
+      raise "storage type required" unless storage_options[:type]
+      raise "storage params required" unless storage_options[:params]
+      
+      storage_class = "#{storage_options[:type].camelize}Storage"
+      raise "unknown storage: #{storage_class}" unless defined?(storage_class)
+
+      storage_class = GetThemAll.const_get(storage_class)
+    
+      storage_class = storage_class
+      storage_options = ActiveSupport::HashWithIndifferentAccess.new( storage_options[:params] )
+      
+      @storage = storage_class.new(storage_options.merge(:folder_name => @folder_name))
+    
+      @parsed_base_url = Addressable::URI.parse(@base_url)
+      
+      # start_url is relative to base_url
+      @start_url = File.join(@base_url, @start_url)
+      
+      # if any unknown option was passed, do not silently walk away
+      # tell the user !
+      unless args.empty?
+        raise "unknown parameters: #{args.inspect}"
+      end
+      
+    end
+  
+    ##
+    # Start the crawler, if you pass a block it
+    # will be called after the engine is iniailized, you can
+    # queue the level 0 urls here and handle authenticating if needed.
+    # 
+    def start
+      load_history()
+    
+      notify('downloader.started', self)
+    
+      EM::run do
+        EM::add_periodic_timer(5) do
+          if (EM::connection_count() == 0) && !@storage.working?
+            debug("no connections, exiting")
+            EM::stop_event_loop()
+          end
+        end
+      
+        EM::error_handler do |err|
+          if err.is_a?(AssertionFailed)
+            error("Assertion failed: #{err.message}")
+          else
+            error("#{err.class}: #{err.message}")
+            err.backtrace.each do |line|
+              error(line)
+            end
+          end
+        end
+     
+        
+        # queue the first action to start crawling
+        #  
+        @examine_queue.push(ExamineAction.new(self,
+            :url => @start_url,
+            :destination_folder => '/',
+            :level => 0,
+          ), 0)
+        
+      
+        # now that actions are queued, start handling them
+        # start each "worker"
+        # dequeuing is priority based, the download actions
+        # first and then the higher the level the higher the
+        # priority for examine actions, this is done this way
+        # to give work to the download workers asap.
+        # 
+        1.upto(self.class.examiners_count) do |n|
+          Worker.new(:examiner, n - 1, self, @examine_queue)
+        end
+      
+        1.upto(self.class.downloaders_count) do |n|
+          Worker.new(:downloader, n - 1, self, @download_queue)
+        end
+
+      end
+   
+      save_history()
+    
+      notify('downloader.completed', self)
+    end
+    
+    class AssertionFailed < RuntimeError; end
+  
+    def assert(cond, msg = "")
+      unless cond
+        raise AssertionFailed, msg
+      end
+    end
+  
+    # toto.com
+    # sub.toto.com
+    
+    ##
+    # Check if two urls are from the same domain.
+    # 
+    # @return [Boolean] true if same domain
+    # 
+    def same_domain?(host1, host2)
+
+      host1_parts = host1.split(".")
+      host2_parts = host2.split(".")
+
+      size = [host1_parts.size, host2_parts.size].min # => 2
+
+      host1_parts= host1_parts[-size..-1] if host1_parts.size > size
+      host2_parts= host2_parts[-size..-1] if host2_parts.size > size
+
+      ret = host1_parts.join(".") == host2_parts.join(".")
+
+      return ret
+    end
+  
+    HISTORY_FILE_PATH = 'history.txt'.freeze
+  
+    # load already downloaded pictures from disk
+    def load_history
+      if @storage.exist?(HISTORY_FILE_PATH)
+        data = @storage.read(HISTORY_FILE_PATH)
+        @history.load(data)
+      else
+        debug "History file not found: #{HISTORY_FILE_PATH}"
+      end
+    end
+  
+    def save_history
+      @storage.write(HISTORY_FILE_PATH, @history.dump)
+    end
+
+
+  
+  
+    # Plugin API
+    def open_url(url, method = "GET", params = nil, referer = nil, deferrable = nil, &block)    
+      deferrable ||= EM::DefaultDeferrable.new
+      referer ||= @base_url
+    
+      url = Addressable::URI.parse( (url[0...4] == "http") ? url : URI.join(@base_url, url) )
+
+      # url = (url[0...4] == "http") ? URI.parse(url) : URI.join(@base_url, url)
+      # url_path = url.path
+    
+      # get queries with params
+      # if method == "GET" && url.query
+      #   url_path << "?#{url.query}"
+      # end
+
+      external = !same_domain?(@parsed_base_url.host, url.host)
+    
+      if external
+        debug("Opening external page: #{url}")
+      else
+        # debugger if url.to_s == "http://fan tasti.cc/user/pussylover75/images/image/367771"
+        debug("#{method.upcase} #{url}")
+      end
+    
+    
+      # find a connection for this host
+      host_key = "#{url.host}:#{url.port}"
+      if false
+      # if @connections.has_key?(host_key) && !@connections[host_key].error?
+        http = @connections[host_key]
+      else
+        # debug("New connection to http://#{url.host}:#{url.port}", "C")
+        http = EM::HttpRequest.new("http://#{url.host}:#{url.port}")
+      
+        @connections[host_key] = http
+      end
+      
+      req = http.setup_request(method.downcase.to_sym,
+          :path => url.path,
+          :query => url.query,
+          # :redirects => 2,
+          :head => {
+              :cookie => @cookies,
+              :referer => referer,
+              # "accept-encoding" => "gzip, compressed",
+              'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1'
+            }
+        )
+    
+      # req.timeout(10)
+      # req.errback do
+      #   error("error while opening #{url} :(")
+      # end
+    
+      req.callback do
+        case req.response_header.status
+        when 200
+          # handle cookies
+          unless external
+            # [["a=42", "PHPSESSID2=u0ctlbfrlrnus1qv8425uv4p42"], ["PHPSESSID=2jek8d61dlt134e0djft4hnn54; path=/", "OAGEO=FR%7C%7C%7C%7C%7C%7C%7C%7C%7C%7C; path=/", "OAID=924ff65ed90c7834d8b37b29bdffc831; expires=Sun, 14-Oct-2012 12:22:22 GMT; path=/", "OAID=924ff65ed90c7834d8b37b29bdffc831; expires=Sun, 14-Oct-2012 12:22:22 GMT; path=/", "OAID=924ff65ed90c7834d8b37b29bdffc831; expires=Sun, 14-Oct-2012 12:22:22 GMT; path=/"]]
+            # [["a=42"], "PHPSESSID=p12aet71oemrfb3olffqaptss3; path=/"]
+            added_cookies = Array(req.cookies[1])
+            
+            added_cookies.each do |str|
+              @cookies << str.split(';').first
+            end
+            
+            # remove duplicates
+            @cookies.uniq!
+            
+            # req.added_cookies.each{|key,val| @cookies[key] = val }
+            # req.deleted_cookies.each{|key, _| @cookies.delete(key) }
+          end
+        
+          # debug("page loaded succesfully: #{url}")
+          deferrable.set_deferred_status(:succeeded, req)
+          if block
+            if block.arity == 2
+              doc = Hpricot(req.response)
+              block.call(req, doc)
+            else
+              block.call(req)
+            end
+          end
+        
+        # em-http-request does not handle redirection between hosts
+        # so handle them ourselves
+        when 301, 302
+          location = req.response_header.location
+          if location
+            debug("Following redirection: #{location}")
+            # reuse the same deferrable object
+            open_url(location, method, params, referer, deferrable, &block)
+          end
+        
+        else
+          puts "#{method} #{url} => Status: #{req.response_header.status}"
+          deferrable.set_deferred_status(:failed, req.response_header.http_reason)
+        end
+      end
+    
+      req.errback do
+        deferrable.set_deferred_status(:failed, -1)
+      end
+    
+      deferrable
+    end
+  
+    def eval_javascript(data)
+      JavascriptLoader.new(data)
+    end
+  
+  
+  
+    # to be redefine in subclasses
+    def examine_page(doc, level)
+      raise "Need to implement examine_page in #{self.class}"
+    end
+  
+    def get_file_destpath_from_action(action)
+      url_folder = action.uri.path
+      File.join(action.destination_folder, url_folder)
+    end
+  end
+end