xunlei 0.0.3 → 0.0.5

data/README.md

@@ -13,16 +13,6 @@ so please make sure you have Google Chrome installed first.
 It can automatically dump task file names and urls as well as browser cookies,
 so that it can delegate to wget to download files for you.
 
-It can handle both normal tasks and BT tasks, and won't complain even if you
-have pages and pages of tasks on http://lixian.vip.xunlei.com
-
-And since it uses wget -c option,
-you don't need to worry about interrupted downloads
-or completely downloaded files been overwritten any more.
-
-Oh, did I mention it also handles Chinese/Japanese/Korean characters
-in file names without any problems?
-
 WARNING:
 it stores your USERNAME and PASSWORD for
 lixian.vip.xunlei.com as PLAINTEXT at ~/.xunlei/credentials.yml

data/bin/xunlei

@@ -1,6 +1,8 @@
 #!/usr/bin/env ruby
 
 require "xunlei"
+# require File.expand_path("lib/xunlei/version.rb")
+# require File.expand_path("lib/xunlei/engine.rb")
 require "yaml"
 require "commander/import"
 
@@ -44,6 +46,12 @@ def do_dump_cookies(c)
   end
 end
 
+def all_downloaded_tasks
+  system("touch #{xunlei_downloaded_tasks_path}")
+  
+  YAML.load_file(xunlei_downloaded_tasks_path) || []
+end
+
 def xunlei_cookies_path
   File.join(xunlei_folder_path, "cookies.txt")
 end
@@ -56,6 +64,10 @@ def xunlei_credential_file_path
   File.join(xunlei_folder_path, "credentials.yml")
 end
 
+def xunlei_downloaded_tasks_path
+  File.join(xunlei_folder_path, "downloaded.yml")
+end
+
 def xunlei_folder_name
   "~/.xunlei"
 end
@@ -125,6 +137,45 @@ def check_for_config_files
   check_for_credentials
 end
 
+def move_to_downloaded_file(task)
+  puts "#{task[:name]} successfully downloaded."
+  
+  downloaded = all_downloaded_tasks
+  downloaded << task[:name]
+  File.open(xunlei_downloaded_tasks_path, "w") do |file|
+    file.write(downloaded.to_yaml)
+  end
+  
+  all_tasks = YAML.load_file(xunlei_tasks_path)
+  all_tasks.delete(task)
+  File.open(xunlei_tasks_path, "w") do |file|
+    file.write(all_tasks.to_yaml)
+  end
+end
+
+def total_size(file_sizes)
+  total_megabytes = 0
+  
+  file_sizes.each do |file_size|
+    if file_size =~ /G/
+      total_megabytes += 1000 * file_size.to_f
+    else
+      total_megabytes += file_size.to_f
+    end
+  end
+  
+  total_megabytes
+end
+
+def show_files(files)
+  files.each do |file|
+    puts "#{file[:size]}\t#{file[:name]}"
+  end
+  
+  puts
+  puts "Total: #{files.size} files, #{total_size(files.map { |file| file[:size] })} MB (estimated)"
+end
+
 command :dump_tasks do |c|
   c.syntax= "dump_tasks"
   c.description = "dump all files to #{xunlei_tasks_path}"
@@ -132,16 +183,26 @@ command :dump_tasks do |c|
   do_dump_cookies(c) do |engine|
     tasks = engine.dump_tasks
     
+    current_downloaded_tasks = all_downloaded_tasks & tasks.map { |task| task[:name] }
+    tasks.reject! { |task| current_downloaded_tasks.include?(task[:name]) }
+    
     File.open(xunlei_tasks_path, "w") do |file|
       file.write(tasks.to_yaml)
     end
     
     puts "Successfully dumped following tasks:"
     puts
-    tasks.each do |task|
-      puts task[:name]
-    end
+    show_files(tasks)
+    
     puts
+    
+    unless current_downloaded_tasks.empty?
+      puts "Finished downloads: (you should delete them on website)"
+      puts
+      current_downloaded_tasks.each do |task_name|
+        puts task_name
+      end
+    end
   end
 end
 
@@ -174,8 +235,12 @@ command :download do |c|
       
       if agree("\nConfirm? (yes or no)")
         filter_files(options).each do |file|
-          cmd = "wget --load-cookies=#{xunlei_cookies_path} '#{file[:url]}' -c -O '#{file[:name]}'"
-          break unless system(cmd)
+          cmd = "wget --load-cookies=#{xunlei_cookies_path} '#{file[:url]}' -c -O \"#{file[:name]}\""
+          if system(cmd)
+            move_to_downloaded_file(file)
+          else
+            break
+          end
         end
       end
     end
@@ -190,9 +255,8 @@ command :show do |c|
     options.default :only => nil, :except => nil
     puts "You have following files in your #{xunlei_tasks_path}:"
     puts
-    filter_files(options).each do |file|
-      puts file[:name]
-    end
+    files = filter_files(options)
+    show_files(files)
   end
 end
 
@@ -207,13 +271,14 @@ end
 command :google do |c|
   c.syntax = "google KEYWORD"
   c.description = "google for ed2k links using given KEYWORD"
-  c.option "--with OPTIONS", String, "include additional info when Googling"
+  c.option "--with WORDS", String, "include additional info when Googling"
+  c.option "--without WORDS", String, "exclude additional info when Googling"
   c.action do |args, options|
-    options.default :with => nil
+    options.default :with => nil, :without => nil
     puts "Searching the web for #{args.join(" ")}..."
     puts
     search = Xunlei::Search.new
-    links = search.google(args, options.with)
+    links = search.google(args, options)
     puts
     puts "#{links.count} links found."
   end

data/lib/xunlei/engine.rb

@@ -12,8 +12,7 @@ module Xunlei
     end
     
     def dump_cookies
-      # wait until cookies are ready
-      get_task_list
+      wait_until_all_loaded
       
       cookies = []
       @browser.driver.manage.all_cookies.each do |cookie|
@@ -42,7 +41,7 @@ module Xunlei
       @browser.close
     end
     
-  private  
+  private
     
     def wait_until_all_loaded
       get_task_list
@@ -56,24 +55,16 @@ module Xunlei
     
     def next_page
       next_li = @browser.li(:class => "next")
-      if next_li.present?
-        next_li.as.first.click
-        true
-      else
-        false
-      end
+      
+      return false unless next_li.present?
+      
+      next_li.as.first.click
     end
     
     def process_current_page
-      task_list = get_task_list
-      
-      all_files = []
-
-      task_list.divs(:class => "rw_list").each do |task_div|
+      get_task_list.divs(:class => "rw_list").inject([]) do |all_files, task_div|
         all_files += process_task(task_div)
       end
-
-      all_files
     end
     
     def process_task(task_div)
@@ -81,9 +72,10 @@ module Xunlei
       
       task_div.wait_until_present
 
-      if task_is_ready?(task_div)  
+      if task_is_ready?(task_div)
+        # puts "task is ready"
         task_div.click
-        task_div.a(:class => "rwbtn ic_down").wait_until_present
+        task_div.a(:class => "rwbtn ic_redownloca").wait_until_present
 
         if task_div.div(:class => "w03img").imgs.first.src == "http://cloud.vip.xunlei.com/160/img/icon_type/tpimg_bt.png"
           task_files += process_bt_task(task_div)
@@ -103,8 +95,9 @@ module Xunlei
     
     def process_normal_task(task_div)
       normal_task_a = task_div.span(:class => "namelink").as.first
-      normal_task_input = task_div.input(:id => "dl_url" + task_div.id.gsub(/\D+/, ""))
-      { :name => normal_task_a.text, :url => normal_task_input.value }
+      task_id = task_div.id.gsub(/\D+/, "")
+      normal_task_input = task_div.input(:id => "dl_url" + task_id)
+      { :name => normal_task_a.text.gsub(/'|\\/,""), :url => normal_task_input.value, :size => task_div.span(:id => "size#{task_id}").text }
     end
     
     def process_bt_task(task_div)
@@ -114,8 +107,12 @@ module Xunlei
       folder_list = @browser.div(:id => "rwbox_bt_list")
       folder_list.wait_until_present
       
-      folder_list.as(:name => "bturls").each do |a|
-        task_files << { :name => a.text, :url => a.href }
+      index = 0
+      folder_list.spans(:class => "namelink").each do |span|
+        s = span.spans.first
+        size = folder_list.input(:id => "bt_size#{index}").attribute_value('value')
+        task_files << { :name => s.title, :url => s.attribute_value('href'), :size => size }
+        index += 1
       end
 
       go_back_from_bt_task

data/lib/xunlei/search.rb

@@ -1,5 +1,6 @@
 require "watir-webdriver"
 require "Nokogiri"
+require "uri"
 
 module Xunlei
   class Search
@@ -7,8 +8,11 @@ module Xunlei
       @browser = Watir::Browser.new driver
     end
     
-    def google(keywords, options=nil)
-      @browser.goto "http://www.google.com/search?q=#{[keywords, options, "ed2k"].flatten.join("+")}"
+    def google(keywords, options)
+      q = [keywords, options.with, "ed2k"].flatten.join("+")
+      q += "+-" + options.without unless options.without.nil?
+      
+      @browser.goto "http://www.google.com/search?q=#{q}"
       
       @browser.div(:id => "ires").wait_until_present
       
@@ -20,7 +24,8 @@ module Xunlei
         @browser.goto page_link
         doc = Nokogiri::HTML(@browser.html)
         doc.css("a").each do |link|
-          href = link['href']
+          next if link['href'].nil?
+          href = URI.escape(link['href'])
           if href =~ /ed2k:|magnet:/ && !ed2k_links.include?(href)
             puts href
             ed2k_links << href

data/lib/xunlei/version.rb

@@ -1,3 +1,3 @@
 module Xunlei
-  VERSION = "0.0.3"
+  VERSION = "0.0.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: xunlei
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-21 00:00:00.000000000Z
+date: 2011-12-03 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70191331322300 !ruby/object:Gem::Requirement
+  requirement: &70300705578980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70191331322300
+  version_requirements: *70300705578980
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70191331321880 !ruby/object:Gem::Requirement
+  requirement: &70300705578560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70191331321880
+  version_requirements: *70300705578560
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70191331321320 !ruby/object:Gem::Requirement
+  requirement: &70300705578140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70191331321320
+  version_requirements: *70300705578140
 - !ruby/object:Gem::Dependency
   name: watir-webdriver
-  requirement: &70191331320640 !ruby/object:Gem::Requirement
+  requirement: &70300705577720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70191331320640
+  version_requirements: *70300705577720
 - !ruby/object:Gem::Dependency
   name: commander
-  requirement: &70191331319880 !ruby/object:Gem::Requirement
+  requirement: &70300705577300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70191331319880
+  version_requirements: *70300705577300
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70191331319100 !ruby/object:Gem::Requirement
+  requirement: &70300705576880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70191331319100
+  version_requirements: *70300705576880
 description: A browser script to access lixian.vip.xunlei.com tasks automatically
 email:
 - afu@forresty.com
@@ -113,15 +113,21 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 2684566908890057272
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 2684566908890057272
 requirements: []
 rubyforge_project: xunlei
-rubygems_version: 1.8.10
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: A browser script to access lixian.vip.xunlei.com tasks automatically