pcapr 0.0.4 → 0.0.6

data/lib/pcapr/pcapr.rb

@@ -3,6 +3,7 @@ require 'nokogiri'
 require 'patron'
 require 'digest'
 require 'timeout'
+require 'uri'
 
 require 'fileutils'
 
@@ -15,13 +16,17 @@ class Pcapr
     
     #驱动浏览器底层的接口, patron对象
     @driver = Patron::Session.new
-    @driver.timeout = 10000
+    @driver.timeout = 60 * 60 # 1 hour
+    @driver.connect_timeout = 10000
     @driver.base_url = "http://www.pcapr.net"
     @driver.handle_cookies
     
     @protos = nil
   end
   
+  # for spec test
+  attr_reader :driver
+  
   attr_accessor :logger
   
   def login
@@ -45,9 +50,17 @@ class Pcapr
   end
   
   def pcap_urls(proto)
-    #TODO
-    url = @driver.get("/browse?proto=#{proto}").body
-    Nokogiri::HTML(url).css("ul#p-main div.p-body>a").collect { |link| link['href'] }
+    ret = []
+    proto_url = URI.encode("/browse?proto=#{proto}")
+    url = @driver.get(proto_url).body
+    nokogiri_parser = Nokogiri::HTML(url)
+    ret += nokogiri_parser.css("ul#p-main div.p-body>a").collect { |link| link['href'] }
+    if nokogiri_parser.css('li.p-overflow a').size > 0
+      href = nokogiri_parser.css('li.p-overflow a').attr('href').value
+      url = @driver.get( "/browse" + href.gsub(" ","%20") ).body
+      ret += Nokogiri::HTML(url).css('li.l0 div.p-body>a').collect { |link| link['href'] }
+    end
+    ret
   end
   
   #获取该数据包文件
@@ -69,16 +82,14 @@ class Pcapr
       logger.info "proto: #{proto}, downloading...(pcap save as: #{proto_dir}"
       pcap_urls(proto).each do |pcap_url|
         file = File.join( proto_dir, File.basename(pcap_url).gsub(/\.html$/,"").tr("\\/:*?\"<>|"," ") )
-        logger.info "  pcap file: #{file} save at '#{file}'"
+        logger.info "  pcap file: #{pcap_url} save at '#{file}'"
         begin
-          Timeout.timeout(60 * 60 * 2) do
-            pcap_file(pcap_url, file)
-          end
+          pcap_file(pcap_url, file)
           logger.debug "  save ok"
+        rescue Patron::TimeoutError
+          logger.error "  save fail: timeout after #{@driver.timeout} seconds"
         rescue =>e
           logger.error "  save fail: #{$!}"
-        rescue Timeout::Error
-          logger.error "  save fail: timeout in 2 hours"
         end
       end
     end

data/lib/pcapr/version.rb

@@ -1,3 +1,3 @@
 class Pcapr
-  VERSION = "0.0.4"
+  VERSION = "0.0.6"
 end

data/pcapr.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |s|
   s.authors     = ["yafei LI"]
   s.email       = ["lyfi2003@gmail.com"]
   s.homepage    = ""
-  s.summary     = %q{a libarily for downloading all pcap files from pcapr.net}
-  s.description = %q{a libarily for downloading all pcap files from pcapr.net}
+  s.summary     = %q{a library for downloading all pcap files from pcapr.net}
+  s.description = %q{a library for downloading all pcap files from pcapr.net}
 
   s.rubyforge_project = "pcapr"
 

data/spec/pcapr_spec.rb

@@ -35,11 +35,55 @@ describe Pcapr do
     @o.protos.should be_include("m2pa (id 12)")
   end
   
+  it "should get pcapfile urls when proto include quote" do
+    @o.login
+    urls = @o.pcap_urls("afs (rx)")
+    urls.size.should >= 7
+    urls.should be_include("/view/tyson.key/2009/10/0/6/LiquidWar_Lobby_1_00002_20091101140004.pcap.html")
+    urls = @o.pcap_urls("fc els")
+    urls.size.should > 0
+    urls = @o.pcap_urls("gtp <ftp>")
+    urls.size.should > 0
+  end
+  
+  it "should support more pcap urls" do
+    @o.login
+    urls = @o.pcap_urls("dns")
+    urls.size.should == 34
+  end
+  
   it "all_proto should get auth.cap" do
     @o.login
     @o.pcap_urls("dns").should be_include("/view/siim/2011/11/3/7/capture.pcap.html")
   end
   
+  #~ it "should have pcap files more than 3000(excute slow)" do
+    #~ @o.login
+    #~ files = []
+    #~ @o.protos.each_with_index do |proto,i|
+      #~ @o.logger.info("proto: #{proto}, index=#{i}")
+      #~ files += @o.pcap_urls(proto)
+    #~ end
+    #~ files.should >= 3085
+  #~ end
+  
+  it "file get it but timeout" do
+    begin
+      @o.login
+      pcap_url = "/view/sudhakar_gajjala/2010/6/1/21/6462.pcap.html"
+      file = File.join($helper_dir, 'timeout.pcap')
+      @o.driver.timeout = 1
+      lambda { @o.pcap_file(pcap_url,file) }.should raise_error(Patron::TimeoutError)
+    ensure
+      @o.driver.timeout = 60*60
+      File.delete(file) if File.exist?(file)
+    end
+  end
+  
+  it "driver timeout default is 1 hour" do
+    @o.driver.timeout.should == 60*60
+  end
+  
   it "file get it " do
     @o.login
     pcap_url = "/view/siim/2011/11/3/7/capture.pcap.html"

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 4
-  version: 0.0.4
+  - 6
+  version: 0.0.6
 platform: ruby
 authors: 
 - yafei LI
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-12 00:00:00 +08:00
+date: 2012-03-14 00:00:00 +08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -43,7 +43,7 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id002
-description: a libarily for downloading all pcap files from pcapr.net
+description: a library for downloading all pcap files from pcapr.net
 email: 
 - lyfi2003@gmail.com
 executables: 
@@ -96,7 +96,7 @@ rubyforge_project: pcapr
 rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
-summary: a libarily for downloading all pcap files from pcapr.net
+summary: a library for downloading all pcap files from pcapr.net
 test_files: 
 - spec/pcapr_spec.rb
 - spec/spec_helper.rb