get_them_all 1.0.0

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2009-2011 Julien Ammous
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,77 @@
+= What is it ?
+
+Get Them All is my personal try at building a versatile and powerful web downloader, its goal is pretty simple:
+download all the targets and keep up to date with new content by remembering what was downloaded.
+
+It should be able to download ay file type and try as much as possible to not make any assumptions on how the
+targeted website is built.
+
+EventMachine is used to power the core, hpricot is used to parse the html.
+
+# Why ?
+
+I simply never found any tool fulfilling my needs so I made mine ;)
+
+
+# What can it do for you
+
+First let's start by what is currently supported:
+
+- authentication (partially by hand)
+- the referer is passed from one page to another so any leecher detection
+  by referer will fail
+- cookies are passed too
+- parallel download, you decide how many parallel tasks are executed
+  you can go as high as you want but don't be stupid !
+- multiple storage backend, currently the files can be saved in:
+  - local disk
+  - dropbox
+- javascript parsing with therubyracer, yes you read that well,
+  if you are crawling a javascript powered site and need to read javascript
+  you can use this to extract the informations you need.
+
+Any website is considered as a reversed pyramid, let's take a gallery website as an example:
+
+- the first level would be the page containing all the thumbnails
+- the second level would be a page showing the picture (each link collected in level 0
+  will lead to a different page on level 2)
+- the third level would be the link to the picture itself
+
+I decided on this model after some testing and until now I never found a
+website where this cannot be applied (a website with fiels to download)
+
+
+# Current state
+
+The application is already ready for my needs and may be for someone else.
+Currently all the connections errors may not be correctly handled especially if
+the web server really has trouble keeping connections alive to serve the clients
+(like for the example above).
+
+
+# Usage
+
+Look at the examples folder, there is two way of using this gem:
+
+As an application, try running:
+
+```bash
+./bin/gta exec examples/wallpaper -s data
+```
+
+Or as a library, try this:
+
+```bash
+ruby examples/standalone.rb
+```
+
+
+
+# Disclaimer
+
+As with most open source projects you are responsible for your actions, if you start
+a crawler with a lot of parallel tasks and manage to get banned for your favorite
+wallpaper site I have nothing to do with this ok ?  
+Don't be stupid and everything will be fine, for my needs I rarely need more than 
+2 examiners and 1/2 downloaders.
+

data/bin/gta

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+require "rubygems"
+
+$LOAD_PATH.unshift( File.expand_path('../../lib', __FILE__) )
+require "get_them_all"
+require "thor"
+
+
+class GtaRunner < Thor
+  
+  desc "exec [-s <path>] <script_path>", "run a user script"
+  method_option :storage_path,        :aliases => '-s', :desc => "path where the data will be saved"
+  def exec(script_path)    
+    
+    storage_path = options[:storage_path]
+    raise("storage_path required") unless storage_path
+    
+    if storage_path[0,1] != '/'
+      # relative path
+      storage_path = File.join(Dir.pwd, storage_path)
+    end
+    
+    if script_path[0,1] != '/'
+      script_path = File.join(Dir.pwd, script_path)
+    end
+
+    # the file exist, load it
+    require script_path
+
+    # check that the class exist
+    class_name = File.basename(script_path, ".rb").camelize + "Downloader"
+    fail("file #{script_path} should define class #{class_name} !") unless Object.const_defined?( class_name.to_sym )
+
+    info("Started with config file #{File.basename(script_path)}")
+    
+    # CTRL+C
+    trap("INT") do
+      EM::stop_event_loop()
+    end
+    
+    # create the instance (and start download)
+    class_name.constantize.new(
+        :storage => {
+            :type => 'file',
+            :params => {
+                :root => storage_path
+              }
+          },
+        :extensions => [GetThemAll::ActionLogger.new]
+      ).start()
+  end
+end
+
+GtaRunner.start

data/get_them_all.gemspec

@@ -0,0 +1,28 @@
+$:.push File.expand_path("../lib", __FILE__)
+require "get_them_all/version"
+
+Gem::Specification.new do |s|
+  s.name        = "get_them_all"
+  s.version     = GetThemAll::VERSION
+  s.authors     = ["Julien Ammous"]
+  s.email       = []
+  s.homepage    = ""
+  s.summary     = %q{Mass downloader}
+  s.description = %q{Mass downloader useable as standalone or as a library}
+
+  s.rubyforge_project = "get_them_all"
+
+  s.files         = `git ls-files lib/* *.gemspec README.* LICENSE`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_runtime_dependency 'thor'
+  s.add_runtime_dependency 'em-http-request',    '~> 1.0.0'
+  s.add_runtime_dependency 'em-priority-queue',  '~> 0.0.2'
+  s.add_runtime_dependency 'hpricot',            '~> 0.8.1'
+  s.add_runtime_dependency 'i18n'
+  s.add_runtime_dependency 'activesupport',      '~> 3.1.0'
+  s.add_runtime_dependency 'therubyracer',       '~> 0.9.8'
+  s.add_runtime_dependency 'dropbox'
+  s.add_runtime_dependency 'girl_friday'
+end

data/lib/get_them_all.rb

@@ -0,0 +1,47 @@
+require File.expand_path('../get_them_all/version', __FILE__)
+
+# system libraries
+require 'logger'
+require 'fileutils'
+require 'zlib'
+
+# gems
+require 'eventmachine'
+require 'active_support/core_ext/object/duplicable'
+require 'active_support/core_ext/class'
+require 'active_support/core_ext/string'
+require 'active_support/core_ext/array'
+
+# local files
+Dir.chdir( File.join(File.dirname(__FILE__), "get_them_all") ) do
+  require './logger'
+  
+  # libraries
+  require './notifier'
+  require './javascript_loader'
+  require './history'
+  
+  # Storage
+  require './storage'
+  require './storage/file_storage'
+  require './storage/dropbox_storage'
+  
+  # extensions
+  require './extension'
+  require './extensions/graph_builder'
+  require './extensions/action_logger'
+  require './extensions/gauge_display'
+  
+  # main files
+  require './site_downloader'
+  require './worker'
+  require './action'
+  require './actions/examine_action'
+  require './actions/download_action'
+end
+
+
+
+module GetThemAll
+
+end

data/lib/get_them_all/action.rb

@@ -0,0 +1,55 @@
+module GetThemAll
+  class Action
+    include Notifier
+  
+    attr_accessor :url, :level, :destination_folder, :params, :referer
+    attr_accessor :parent_url
+  
+    include EM::Deferrable
+  
+    def initialize(downloader, h, params = {})
+      @downloader = downloader
+    
+      @storage = @downloader.storage
+    
+      @level= 0
+      @params= h.delete(:params)
+      @destination_folder= nil
+    
+      h.each do |key, val|
+        raise ("unknown properties #{key} !") unless respond_to?("#{key}=")
+        send("#{key}=", val) unless val.nil?
+      end
+    end
+    
+    def inspect
+      "{#{self.class}[#{level}] #{url} }"
+    end
+  
+    def uri
+      URI.parse(@url)
+    end
+  
+  
+    def already_visited?(url)
+      @downloader.history.include?(url)
+    end
+  
+  
+    # internals      
+    def queue_action(action)
+      action.parent_url = @url
+      action.destination_folder ||= @destination_folder
+    
+      queue = action.is_a?(ExamineAction) ? "@examine_queue" : "@download_queue"
+      @downloader.instance_variable_get(queue).push(action, action.priority)
+    end
+  
+    # return a number between 0.1 and 1
+    def retry_time
+      0.1 * (rand(1000)+1)/100
+    end
+    protected :retry_time
+  
+  end
+end

data/lib/get_them_all/actions/download_action.rb

@@ -0,0 +1,80 @@
+module GetThemAll
+  class DownloadAction < Action
+    def priority
+      10
+    end
+  
+    def do_action(worker = nil)
+      notify('action.download.started', worker, self)
+    
+      if already_visited?(@url)
+        notify('action.download.skipped', worker, self)
+        set_deferred_status(:succeeded)
+      else
+    
+        req = @downloader.open_url(@url, "GET", nil, @referer)
+        req.callback do |req|
+      
+          destpath = compute_filename(worker)
+          download = @storage.write(destpath, req.response)
+      
+          download.callback do      
+            add_to_history()
+            set_deferred_status(:succeeded)
+      
+            notify('action.download.success', worker, self, destpath)
+          end
+      
+          download.errback do
+            notify('action.download.failure', worker, self)
+          end      
+        end
+    
+        req.timeout(5)
+    
+        req.errback do |*args|
+          status = (args.size == 1) ? args.first : 0
+      
+          # remove file if created
+          path = compute_filename(worker)
+          File.delete(path) if File.exist?(path)
+      
+          notify('action.download.failure', worker, self)
+      
+          set_deferred_status(:failed)
+        end
+      end
+    
+    end
+  
+  private
+    def random_string(len=5)
+      ret= ""
+      chars= ("a".."z").to_a
+      1.upto(len) { |i| ret<< chars[rand(chars.size-1)] }
+      ret
+    end
+  
+    def add_to_history()
+      if @downloader.class.history_tracking == :default
+        @downloader.history.add(@parent_url)
+      else
+        @downloader.history.add(@url)
+      end
+    end
+  
+    def compute_filename(worker)
+      destpath= @downloader.get_file_destpath_from_action(self)
+        
+      # find an unused filename
+      while @storage.exist?(destpath)        
+        path, filename= File.dirname(destpath), File.basename(destpath).split(".")
+        filename= "#{filename[0]}_#{random_string(2)}.#{filename[1]}"
+        destpath= File.join(path, filename)
+        notify('action.download.renamed', worker, self, destpath)
+      end
+    
+      destpath
+    end
+  end
+end

data/lib/get_them_all/actions/examine_action.rb

@@ -0,0 +1,47 @@
+
+require 'hpricot'
+
+module GetThemAll
+  class ExamineAction < Action
+  
+    def priority
+      @level
+    end  
+  
+    def do_action(worker = nil)
+      notify('action.examine.started', worker, self)
+    
+      if already_visited?(@url)
+        notify('action.examine.skipped', worker, self)
+        set_deferred_status(:succeeded)
+      
+      else
+        req = @downloader.open_url(@url, "GET", nil, @referer)
+        req.callback do |req|
+          doc = Hpricot( req.response )
+      
+          actions = @downloader.examine_page(doc, @level, self)
+          actions.each do |action|
+            action.level = @level + 1
+            # action.params = @params.merge(action.params)
+            queue_action(action)
+          end
+        
+          notify('action.examine.success', worker, self, actions)
+          set_deferred_status(:succeeded)
+        end
+      
+        req.timeout(5)
+      
+        req.errback do |*args|
+          status = (args.size == 1) ? args.first : 0
+          notify('action.examine.failure', worker, self, status)
+          set_deferred_status(:failed)
+        
+        end
+      
+      end
+    end
+  
+  end
+end

data/lib/get_them_all/extension.rb

@@ -0,0 +1,17 @@
+require 'active_support/notifications'
+
+module GetThemAll
+  class Extension
+  
+    ##
+    # Register a handler to call when this notification
+    # is sent
+    # 
+    # @param [String] name notification identifier
+    # 
+    def register_handler(name, &block)
+      ActiveSupport::Notifications.subscribe(name, &block)
+    end
+  
+  end
+end

data/lib/get_them_all/extensions/action_logger.rb

@@ -0,0 +1,58 @@
+
+module GetThemAll
+  ##
+  # This extension can be considered as a verbose mode, it
+  # logs nearly every everything that happens.
+  # 
+  class ActionLogger < Extension
+    def initialize
+      register_handler('downloader.started') do |name, downloader|
+        @skipped_files = 0
+        @download_files = 0
+      end
+    
+      register_handler('action.examine.started') do |name, worker, action|
+        log("Examining[#{action.level}] #{action.url}")
+      end
+    
+      register_handler('action.examine.skipped') do |name, worker, action|
+        @skipped_files += 1
+        log("Skipping #{action.url}")
+      end
+    
+      register_handler('action.examine.success') do |name, worker, action|
+        # do nothing
+      end
+    
+    
+      register_handler('action.download.started') do |name, worker, action|
+        log("Downloading #{action.url}")
+      end
+    
+      register_handler('action.download.renamed') do |name, worker, action, new_path|
+        log("Renamed as #{File.basename(new_path)}")
+      end
+    
+      register_handler('action.download.skipped') do |name, worker, action|
+        log("url Skipped: #{action.url}")
+      end
+    
+      register_handler('action.download.success') do |name, worker, action, destpath|
+        @download_files += 1
+        log("File downloaded: #{destpath}")
+      end
+    
+      register_handler('downloader.completed') do |name, worker, downloader|
+        log ""
+        log "Downloaded #{@download_files} files"
+        log "Skipped: #{@skipped_files}"
+      end
+    
+    end
+  
+    def log(str)
+      puts "[log] #{str}"
+    end
+  
+  end
+end