get_them_all 1.0.0

data/lib/get_them_all/storage.rb

@@ -0,0 +1,27 @@
+module GetThemAll
+  class Storage
+    
+    class StorageError < RuntimeError; end
+    class WriteError < StorageError; end
+    class ReadError < StorageError; end
+  
+    def initialize(options = {})
+      @root = options.delete(:root)
+      @folder_name = options.delete(:folder_name)
+      raise "missing required option: folder_name" unless @folder_name
+      raise "missing required option: root" unless @root
+    
+      @root = File.join(@root, @folder_name)
+      raise "missing required option: root" unless @root
+    end
+  
+    def working?
+      false
+    end
+  
+    def build_destpath(path)
+      File.join(@root, path)
+    end
+  
+  end
+end

data/lib/get_them_all/storage/dropbox_storage.rb

@@ -0,0 +1,114 @@
+require 'stringio'
+require 'dropbox'
+require 'girl_friday'
+
+module GetThemAll
+  class DropboxStorage < Storage
+  
+  
+    # def self.deserialize(data)
+    #   consumer_key, consumer_secret, authorized, token, token_secret, ssl, mode = YAML.load(StringIO.new(data))
+    #   raise ArgumentError, "Must provide a properly serialized #{self.to_s} instance" unless [ consumer_key, consumer_secret, token, token_secret ].all? and authorized == true or authorized == false
+    # 
+    #   session = self.new(consumer_key, consumer_secret, :ssl => ssl, :already_authorized => authorized)
+    #   if authorized then
+    #     session.set_access_token token, token_secret
+    #   else
+    #     session.set_request_token token, token_secret
+    #   end
+    #   session.mode = mode if mode
+    # 
+    #   return session
+    # end
+  
+    def initialize(options = {})
+      super
+    
+      session_data = options.delete(:session)
+      raise "session missing" unless session_data
+    
+      consumer_key, consumer_secret, authorized, token, token_secret, ssl, mode = session_data
+    
+      @session = Dropbox::Session.new(consumer_key, consumer_secret, :ssl => ssl, :already_authorized => authorized)
+      @session.set_access_token(token, token_secret)
+      @session.mode = mode.to_sym
+    
+      @queue = GirlFriday::WorkQueue.new(:dropbox_upload, :size => 1) do |msg|
+        # puts "Uploading file #{msg[:path]}..."
+        write(msg[:path], msg[:data], false, msg[:deferrable])
+        # puts "Upload ok."
+      end
+    end
+  
+    ##
+    # Can we exit or is there still some work to do ?
+    # 
+    # @return [Boolean]
+    # 
+    def working?
+      s = @queue.status['dropbox_upload']
+      (s[:backlog] > 0) || (s[:busy] > 0)
+    end
+  
+    ##
+    # Check if the remote file/folder exists,
+    # there may be a better way but it works.
+    # 
+    # @return [Boolean] true if the remote path exists
+    # 
+    def exist?(path)
+      destpath = build_destpath(path)
+    
+      metadata = @session.metadata(destpath)
+      if metadata.respond_to?(:is_deleted) && metadata.is_deleted
+        false
+      else
+        true
+      end
+    rescue Dropbox::FileNotFoundError
+      false
+    end
+  
+    ##
+    # Write data in a remote file.
+    # Folders in the path will be automatically created as
+    # needed.
+    # 
+    def write(path, data, delay = true, deferrable = EM::DefaultDeferrable.new)
+      retries = 0
+    
+      if delay
+        @queue.push(:path => path, :data => data, :deferrable => deferrable)
+      else
+        destpath = build_destpath(path)
+        filename, dirname = File.basename(destpath), File.dirname(destpath)
+        @session.upload(StringIO.new(data), dirname, :as => filename)
+        EM::next_tick{ deferrable.succeed }
+      end
+    
+      deferrable
+    rescue => err
+      if retries < 4
+        # puts "[#{retries}] Upload error, retrying: #{err}"
+        retries += 1
+        retry
+      else
+        EM::next_tick{ deferrable.fail }
+        raise WriteError, "cannot write file: #{err}"
+      end
+    end
+  
+    ##
+    # Read file's content and return it.
+    # 
+    # @return [String] file's content
+    # 
+    def read(path)
+      destpath = build_destpath(path)
+      @session.download(destpath)
+    rescue => err
+      raise ReadError, "cannot read file: #{err}"
+    end
+  
+  end
+end

data/lib/get_them_all/storage/file_storage.rb

@@ -0,0 +1,40 @@
+module GetThemAll
+  class FileStorage < Storage
+  
+    # def initialize(options = {})
+    #   super
+    # end
+  
+    def exist?(path)
+      destpath = build_destpath(path)
+      File.exist?(destpath)
+    end
+  
+    def write(path, data)
+      deferrable = EM::DefaultDeferrable.new
+    
+      destpath = build_destpath(path)
+      FileUtils.mkdir_p( File.dirname(destpath) )
+      open(destpath, "wb") do |f|
+        f.write( data )
+      end
+    
+      deferrable.succeed
+      deferrable
+    
+    rescue Errno::EINVAL
+      deferrable.fail
+      raise Storage::WriteError("cannot write file")
+    end
+  
+    def read(path)
+      destpath = build_destpath(path)
+      if File.readable?(destpath)
+        File.read(destpath)
+      else
+        raise Storage::ReadError("cannot open file")
+      end
+    end
+  
+  end
+end

data/lib/get_them_all/version.rb

@@ -0,0 +1,3 @@
+module GetThemAll
+  VERSION = "1.0.0"
+end

data/lib/get_them_all/worker.rb

@@ -0,0 +1,48 @@
+module GetThemAll
+  # 
+  # workers are created to follow a queue
+  # each time an action is put in the queue it will handle it
+  #
+  class Worker
+    attr_reader :type, :index
+  
+    def initialize(type, index, downloader, queue)
+      @type = type
+      @index = index
+    
+      @downloader = downloader
+      @queue = queue
+    
+      @queue.pop do |action|
+        handle_action(action)
+      end
+    
+    end
+  
+    ##
+    # Take the next action in queue
+    # 
+    def take_next_job
+      EM::next_tick do
+        @queue.pop{|act| handle_action(act) }
+      end
+    end
+  
+  
+    def handle_action(action)
+      action.callback( &method(:take_next_job) )
+      action.errback do
+        # in case of failure, try again later (with slightly lower priority)
+        EM::add_timer(50) do
+          @queue.push(action, [action.level - 1, 0].max)
+        end
+      
+        # and take the next job
+        take_next_job()
+      end
+    
+      action.do_action(self)
+    end
+  
+  end
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: get_them_all
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Julien Ammous
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-10-16 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: &70256844459680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70256844459680
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: &70256844459180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70256844459180
+- !ruby/object:Gem::Dependency
+  name: em-priority-queue
+  requirement: &70256844458680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *70256844458680
+- !ruby/object:Gem::Dependency
+  name: hpricot
+  requirement: &70256844458220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: *70256844458220
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: &70256844457840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70256844457840
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &70256844457300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70256844457300
+- !ruby/object:Gem::Dependency
+  name: therubyracer
+  requirement: &70256844456800 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.8
+  type: :runtime
+  prerelease: false
+  version_requirements: *70256844456800
+- !ruby/object:Gem::Dependency
+  name: dropbox
+  requirement: &70256844456420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70256844456420
+- !ruby/object:Gem::Dependency
+  name: girl_friday
+  requirement: &70256844455800 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70256844455800
+description: Mass downloader useable as standalone or as a library
+email: []
+executables:
+- gta
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- get_them_all.gemspec
+- lib/get_them_all.rb
+- lib/get_them_all/action.rb
+- lib/get_them_all/actions/download_action.rb
+- lib/get_them_all/actions/examine_action.rb
+- lib/get_them_all/extension.rb
+- lib/get_them_all/extensions/action_logger.rb
+- lib/get_them_all/extensions/gauge_display.rb
+- lib/get_them_all/extensions/graph_builder.rb
+- lib/get_them_all/history.rb
+- lib/get_them_all/javascript_loader.rb
+- lib/get_them_all/logger.rb
+- lib/get_them_all/notifier.rb
+- lib/get_them_all/site_downloader.rb
+- lib/get_them_all/storage.rb
+- lib/get_them_all/storage/dropbox_storage.rb
+- lib/get_them_all/storage/file_storage.rb
+- lib/get_them_all/version.rb
+- lib/get_them_all/worker.rb
+- bin/gta
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3600791884245333025
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3600791884245333025
+requirements: []
+rubyforge_project: get_them_all
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Mass downloader
+test_files: []