puushload 0.1.0

data/bin/puushload

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require 'puushload'
+gem 'trollop'
+
+require 'trollop'
+
+opts = Trollop::options do 
+  opt :output_dir, "Output-Directory", :short => 'o', :default => "output", :type => String
+  opt :log_file, "Logile-Path", :short => 'l', :default => "STDOUT", :type => String
+end
+
+logger = Logger.new(opts[:log_file], 10, 1024000) if opts[:log_file] != "STDOUT" 
+
+#PuushDownloader.download_file('12345', File.absolute_path("test"))
+puush_load = PuushLoad.new(opts[:output_dir], logger) 
+puush_load.download_files

data/lib/puushload/puush_downloader.rb

@@ -0,0 +1,40 @@
+require 'net/http'
+require 'fileutils'
+
+class PuushDownloader
+  MAX_RETRIES = 5
+  
+  def self.download_file(id, subfolder, logger, retries=0)
+    logger.error "Too many retries, this file doesn't seem to exist." and return false if retries > MAX_RETRIES 
+    begin
+      Net::HTTP.start("puu.sh") do |http|
+        http.request_get('/' + URI.encode(id)) do |response|
+          filename = PuushHelper.get_file_name_from_response(response)
+          return false if !filename
+        
+          filename = id + " - " + filename
+          path = subfolder + "/" + filename
+        
+          return filename if File.exists?(path)
+        
+          FileUtils.mkdir_p(subfolder)
+          FileUtils.touch(path)
+        
+          file = open path, 'wb'
+        
+          response.read_body do |segment|
+            file.write(segment)
+          end
+        
+          return filename
+        end
+      end
+    rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError, Errno::EHOSTUNREACH, SocketError,
+      Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError => e
+      
+      logger.warn "Connection error, retrying (#{retries})...: " + e.to_s
+      self.download_file(id, subfolder, logger, retries+1)
+      #return false
+    end
+  end
+end

data/lib/puushload/puush_helper.rb

@@ -0,0 +1,9 @@
+class PuushHelper
+  def self.get_file_name_from_response(response)
+    contentDisposition = response['content-disposition']
+    if contentDisposition.nil? then
+      return false
+    end
+    contentDisposition.split(";")[1].split("=")[1].gsub("\"", "")
+  end
+end

data/lib/puushload/puush_id_generator.rb

@@ -0,0 +1,30 @@
+class PuushIDGenerator
+  CHARS = ("0".."9").to_a + ("a".."z").to_a + ("A".."Z").to_a
+  
+  def self.generate(&block)
+    
+    # this is really fuckin dirty atm, but you may optimize this if you want to
+    
+    # Example-ID: 12345
+    
+    # first char (1) Only Numbers at the moment. Should take a bit until chars are being used here
+    ("1".."9").each do |char1|
+      # second char (2)
+      CHARS.each do |char2|
+        # third char (3)
+        CHARS.each do |char3|
+          # fourth char (4)
+          CHARS.each do |char4|
+            # fifth char (5)
+            CHARS.each do |char5|
+              # combine se IDs
+              id = char1 + char2 + char3 + char4 + char5
+              # call se block
+              block.call(id)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/puushload.rb

@@ -0,0 +1,77 @@
+require 'rubygems'
+
+gem "archive-tar-minitar", "~> 0.5.2"
+
+require 'puushload/puush_downloader'
+require 'puushload/puush_helper'
+require 'puushload/puush_id_generator'
+
+require 'zip/zip'
+
+require 'logger'
+require 'tmpdir'
+
+class PuushLoad
+  MAX_THREADS = 50
+  ARCHIVE_SIZE = 1000
+  
+  def initialize(outputFolder, logger)
+    @logger = logger
+    @logger = Logger.new(STDOUT) if @logger.nil?
+    @logger.level = Logger::INFO
+    @output_folder = File.absolute_path(outputFolder)
+  end
+  
+  def download_files
+    @logger.info "Starting download!"
+    
+    spawned_threads = []
+    downloaded_files = 0
+    archives = 0
+    current_tmp_dir = Dir.mktmpdir
+    
+    FileUtils.mkdir_p(@output_folder)
+    
+    PuushIDGenerator.generate do |id|
+      
+      spawned_threads << Thread.new {
+        PuushDownloader.download_file(id, current_tmp_dir, @logger)
+        downloaded_files += 1
+      }
+      
+      if spawned_threads.length >= MAX_THREADS
+        spawned_threads.each { |spawnedThread| spawnedThread.join }
+        spawned_threads.clear
+        @logger.info "Killed some threads. #{downloaded_files} Files so far!"
+        
+        if downloaded_files % ARCHIVE_SIZE == 0 && downloaded_files > 0 then
+          make_archive((@output_folder + "/" + archive_name(archives.to_s)), (current_tmp_dir))
+          
+          @logger.info "Created archive with id: #{archives} !"
+          
+          current_tmp_dir = Dir.mktmpdir
+          archives += 1
+        end
+      end
+      
+    end
+    @logger.info "Actually, this is impossible, but you finished it!"
+  end
+  
+  private
+  def archive_name(archives)
+    "puush" + archives + ".zip"
+  end
+  
+  def make_archive(out_file, in_folder)
+    files = Dir.entries(in_folder)
+    
+    Zip::ZipFile.open(out_file, Zip::ZipFile::CREATE) do |zipfile|
+      files.each do |filename|
+        zipfile.add(filename, in_folder + '/' + filename)
+      end
+    end
+    
+    FileUtils.remove_entry_secure in_folder
+  end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: puushload
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Jan-Henrik Bruhn
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-11 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: jh.bruhn@me.com
+executables:
+- puushload
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/puushload.rb
+- lib/puushload/puush_downloader.rb
+- lib/puushload/puush_helper.rb
+- lib/puushload/puush_id_generator.rb
+- bin/puushload
+homepage: 
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Download all files from Puush.me!
+test_files: []