trackit_scraper 1.1.0 → 1.2.0

data/lib/trackit_scraper/2003.10.1/lib/parallel_request_scraper.rb

@@ -24,7 +24,7 @@ class ParallelRequestScraper
 
     until @request_ids.empty?
       request = scrape_next_request b
-      request ? yield(request) : break
+      yield request
     end
 
     b.close

data/lib/trackit_scraper/2003.10.1/lib/request_page.rb

@@ -8,9 +8,12 @@ class RequestPage
   end
 
   def get_request
+    r = { id: request_id }
+    return r if request_not_found?
+
     request_info_table = RequestInfoTable.new @b.tables[1]
     request_history_table = RequestHistoryTable.new @b.tables[3]
-    r = { id: request_id }
+
     r.merge! request_info_table.get_request_info
     r.merge! request_history_table.get_resolution_info
     r
@@ -22,4 +25,8 @@ class RequestPage
     @b.text[/Request #(\d+)/, 1]
   end
 
+  def request_not_found?
+    @b.text =~ /Request #\d+ not found/
+  end
+
 end

data/lib/trackit_scraper/2003.10.1/trackit.rb

@@ -32,9 +32,14 @@ class TrackIt
 
   def scrape_requests(request_ids)
     FileUtils.mkdir_p @output_dir
+    request_ids = request_ids - existing_request_ids
     @parallel_request_scraper.scrape(request_ids) { |request| write_request_file request }
   end
 
+  def existing_request_ids
+    Dir["#@output_dir/*"].map { |f| File.basename(f, '.json').to_i }
+  end
+
   def write_request_file(request)
     File.write request_file_path(request), request.to_json
     print '.'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: trackit_scraper
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-11 00:00:00.000000000 Z
+date: 2013-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json