trackit_scraper 1.0.0 → 1.1.0

data/lib/trackit_scraper/2003.10.1/lib/login_command.rb

@@ -1,22 +1,22 @@
-class LoginCommand
-
-  def initialize(username, password, navigator)
-    @username = username
-    @password = password
-    @navigator = navigator
-  end
-
-  def execute(b)
-    @navigator.goto 'hd/index.ssp', b
-    login b
-  end
-
-  private
-
-  def login(b)
-    b.text_field(name: 'user_id').set @username
-    b.text_field(name: 'user_pwd').set @password
-    b.button(value: 'Log on').click
-  end
-
+class LoginCommand
+
+  def initialize(username, password, navigator)
+    @username = username
+    @password = password
+    @navigator = navigator
+  end
+
+  def execute(b)
+    @navigator.goto 'hd/index.ssp', b
+    login b
+  end
+
+  private
+
+  def login(b)
+    b.text_field(name: 'user_id').set @username
+    b.text_field(name: 'user_pwd').set @password
+    b.button(value: 'Log on').click
+  end
+
 end

data/lib/trackit_scraper/2003.10.1/lib/navigator.rb

@@ -1,12 +1,12 @@
-class Navigator
-
-  def initialize(base_url)
-    @base_url = base_url
-  end
-
-  def goto(relative_url, b)
-    url = File.join @base_url, relative_url
-    b.goto url
-  end
-
+class Navigator
+
+  def initialize(base_url)
+    @base_url = base_url
+  end
+
+  def goto(relative_url, b)
+    url = File.join @base_url, relative_url
+    b.goto url
+  end
+
 end

data/lib/trackit_scraper/2003.10.1/lib/parallel_request_scraper.rb

@@ -1,45 +1,45 @@
-require 'thread'
-require 'watir-webdriver'
-require_relative 'request_page'
-
-class ParallelRequestScraper
-
-  def initialize(navigator, login_command, threads)
-    @navigator = navigator
-    @login_command = login_command
-    @threads = threads
-  end
-
-  def scrape(request_ids, &block)
-    @request_ids = request_ids.clone
-    @semaphore = Mutex.new
-    @threads.times.map { Thread.new { scrape_requests &block } }.each { |t| t.join }
-  end
-
-  private
-
-  def scrape_requests
-    b = Watir::Browser.new
-    @login_command.execute b
-
-    until @request_ids.empty?
-      request = scrape_next_request b
-      request ? yield(request) : break
-    end
-
-    b.close
-  end
-
-  def scrape_next_request(b)
-    id = next_request_id
-    return unless id
-    @navigator.goto "hd/ticket/euTicketView.ssp?ticket_id=#{id}&log=show", b
-    request_page = RequestPage.new b
-    request_page.get_request
-  end
-
-  def next_request_id
-    @semaphore.synchronize { return @request_ids.pop }
-  end
-
+require 'thread'
+require 'watir-webdriver'
+require_relative 'request_page'
+
+class ParallelRequestScraper
+
+  def initialize(navigator, login_command, threads)
+    @navigator = navigator
+    @login_command = login_command
+    @threads = threads
+  end
+
+  def scrape(request_ids, &block)
+    @request_ids = request_ids.clone
+    @semaphore = Mutex.new
+    @threads.times.map { Thread.new { scrape_requests &block } }.each { |t| t.join }
+  end
+
+  private
+
+  def scrape_requests
+    b = Watir::Browser.new
+    @login_command.execute b
+
+    until @request_ids.empty?
+      request = scrape_next_request b
+      request ? yield(request) : break
+    end
+
+    b.close
+  end
+
+  def scrape_next_request(b)
+    id = next_request_id
+    return unless id
+    @navigator.goto "hd/ticket/euTicketView.ssp?ticket_id=#{id}&log=show", b
+    request_page = RequestPage.new b
+    request_page.get_request
+  end
+
+  def next_request_id
+    @semaphore.synchronize { return @request_ids.pop }
+  end
+
 end

data/lib/trackit_scraper/2003.10.1/lib/request_finder.rb

@@ -1,31 +1,46 @@
-require 'watir-webdriver'
-
-class RequestFinder
-
-  def initialize(navigator)
-    @navigator = navigator
-  end
-
-  def get_request_ids_for_users(users)
-    b = Watir::Browser.new
-    request_ids = users.map { |user| get_request_ids_for_user user, b }.flatten.sort
-    b.close
-    request_ids
-  end
-
-  private
-
-  def get_request_ids_for_user(user, b)
-    login_command = LoginCommand.new user[:username], user[:password], @navigator
-    login_command.execute b
-    @navigator.goto 'hd/ticket/euTicketFind.ssp', b
-    get_all_request_ids b
-  end
-
-  def get_all_request_ids(b)
-    b.button(value: 'Find').click
-    b.links(href: /ticket_id=/).to_a.map { |link| link.href[/ticket_id=(\d+)/, 1].to_i }
-  end
-
-end
-
+require 'time'
+require 'watir-webdriver'
+
+class RequestFinder
+
+  def initialize(navigator)
+    @navigator = navigator
+  end
+
+  def get_request_ids_for_users(users, options={})
+    b = Watir::Browser.new
+    requests = users.map { |user| get_requests_for_user user, b }.flatten
+    b.close
+    start_date = options[:start_date]
+    requests.reject! { |r| start_date > r[:submitted_on] } if start_date
+    requests.map { |r| r[:id] }.sort
+  end
+
+  private
+
+  def get_requests_for_user(user, b)
+    login_command = LoginCommand.new user[:username], user[:password], @navigator
+    login_command.execute b
+    @navigator.goto 'hd/ticket/euTicketFind.ssp', b
+    get_all_requests b
+  end
+
+  def get_all_requests(b)
+    b.button(value: 'Find').click
+    rows = b.tables[1].rows.to_a
+    rows.shift 2
+
+    requests = rows.map do |row|
+      cells = row.cells.to_a
+      next if cells.empty?
+      {
+          id: cells[0].text.to_i,
+          submitted_on: Time.parse(cells[2].text)
+      }
+    end
+
+    requests.compact
+  end
+
+end
+

data/lib/trackit_scraper/2003.10.1/lib/request_history_table.rb

@@ -1,25 +1,25 @@
-require 'time'
-
-class RequestHistoryTable
-
-  def initialize(table)
-    @table = table
-  end
-
-  def get_resolution_info
-    r = {}
-    rows = @table.trs.to_a
-    resolved_row_index = rows.index { |r| r.text =~ /Resolved by/ }
-
-    if resolved_row_index
-      row_above = rows[resolved_row_index-1]
-      captures = row_above.text.scan(/(.+) by (.+)/)[0]
-      r[:resolved_on] = Time.parse captures[0]
-      r[:resolved_by] = captures[1]
-    end
-
-    r[:history] = @table.text
-    r
-  end
-
+require 'time'
+
+class RequestHistoryTable
+
+  def initialize(table)
+    @table = table
+  end
+
+  def get_resolution_info
+    r = {}
+    rows = @table.trs.to_a
+    resolved_row_index = rows.index { |r| r.text =~ /Resolved by/ }
+
+    if resolved_row_index
+      row_above = rows[resolved_row_index-1]
+      captures = row_above.text.scan(/(.+) by (.+)/)[0]
+      r[:resolved_on] = Time.parse captures[0]
+      r[:resolved_by] = captures[1]
+    end
+
+    r[:history] = @table.text
+    r
+  end
+
 end

data/lib/trackit_scraper/2003.10.1/lib/request_info_table.rb

@@ -1,43 +1,43 @@
-class RequestInfoTable
-
-  def initialize(table)
-    @cells = table.tds.to_a
-  end
-
-  def get_request_info
-    {
-        title: cell(0),
-        status: cell(2),
-        service: cell(3),
-        request_type: cell(4),
-        time_spent: cell(5),
-        priority: cell(7),
-        deadline: time_cell(8),
-        submitted_to: cell(10),
-        submitted_by: cell(11),
-        submitted_on: time_cell(12),
-        assigned_to: cell(15),
-        assigned_by: cell(16),
-        assigned_on: time_cell(17),
-        department_id: cell(19),
-        closed_by: cell(20),
-        closed_on: maybe_time_cell(21, 'None')
-    }
-  end
-
-  private
-
-  def maybe_time_cell(index, non_time_value)
-    cell(index) { |s| s == non_time_value ? cell(index) : time_cell(index) }
-  end
-
-  def time_cell(index)
-    cell(index) { |s| Time.parse s }
-  end
-
-  def cell(index)
-    text = @cells[index].text
-    block_given? ? yield(text) : text
-  end
-
+class RequestInfoTable
+
+  def initialize(table)
+    @cells = table.tds.to_a
+  end
+
+  def get_request_info
+    {
+        title: cell(0),
+        status: cell(2),
+        service: cell(3),
+        request_type: cell(4),
+        time_spent: cell(5),
+        priority: cell(7),
+        deadline: time_cell(8),
+        submitted_to: cell(10),
+        submitted_by: cell(11),
+        submitted_on: time_cell(12),
+        assigned_to: cell(15),
+        assigned_by: cell(16),
+        assigned_on: time_cell(17),
+        department_id: cell(19),
+        closed_by: cell(20),
+        closed_on: maybe_time_cell(21, 'None')
+    }
+  end
+
+  private
+
+  def maybe_time_cell(index, non_time_value)
+    cell(index) { |s| s == non_time_value ? cell(index) : time_cell(index) }
+  end
+
+  def time_cell(index)
+    cell(index) { |s| Time.parse s }
+  end
+
+  def cell(index)
+    text = @cells[index].text
+    block_given? ? yield(text) : text
+  end
+
 end

data/lib/trackit_scraper/2003.10.1/lib/request_page.rb

@@ -1,25 +1,25 @@
-require_relative 'request_info_table'
-require_relative 'request_history_table'
-
-class RequestPage
-
-  def initialize(b)
-    @b = b
-  end
-
-  def get_request
-    request_info_table = RequestInfoTable.new @b.tables[1]
-    request_history_table = RequestHistoryTable.new @b.tables[3]
-    r = { id: request_id }
-    r.merge! request_info_table.get_request_info
-    r.merge! request_history_table.get_resolution_info
-    r
-  end
-
-  private
-
-  def request_id
-    @b.text[/Request #(\d+)/, 1]
-  end
-
+require_relative 'request_info_table'
+require_relative 'request_history_table'
+
+class RequestPage
+
+  def initialize(b)
+    @b = b
+  end
+
+  def get_request
+    request_info_table = RequestInfoTable.new @b.tables[1]
+    request_history_table = RequestHistoryTable.new @b.tables[3]
+    r = { id: request_id }
+    r.merge! request_info_table.get_request_info
+    r.merge! request_history_table.get_resolution_info
+    r
+  end
+
+  private
+
+  def request_id
+    @b.text[/Request #(\d+)/, 1]
+  end
+
 end

data/lib/trackit_scraper/2003.10.1/trackit.rb

@@ -1,51 +1,51 @@
-$stdout.sync = true
-
-require 'fileutils'
-require 'json'
-require_relative 'lib/login_command'
-require_relative 'lib/navigator'
-require_relative 'lib/parallel_request_scraper'
-require_relative 'lib/request_finder'
-
-class TrackIt
-
-  def initialize(base_url, username, password, options={})
-    navigator = Navigator.new base_url
-    login_command = LoginCommand.new username, password, navigator
-    threads = options[:threads] || 1
-    @parallel_request_scraper = ParallelRequestScraper.new navigator, login_command, threads
-    @request_finder = RequestFinder.new navigator
-    @output_dir = options[:output_dir] || default_output_dir
-  end
-
-  def scrape_requests_for_users(users)
-    request_ids = @request_finder.get_request_ids_for_users users
-    scrape_requests request_ids
-  end
-
-  def scrape_requests_in_range(from_id, to_id)
-    request_ids = (from_id..to_id).to_a
-    scrape_requests request_ids
-  end
-
-  private
-
-  def scrape_requests(request_ids)
-    FileUtils.mkdir_p @output_dir
-    @parallel_request_scraper.scrape(request_ids) { |request| write_request_file request }
-  end
-
-  def write_request_file(request)
-    File.write request_file_path(request), request.to_json
-    print '.'
-  end
-
-  def request_file_path(request)
-    File.join @output_dir, "#{request[:id]}.json"
-  end
-
-  def default_output_dir
-    File.join 'output', Time.now.strftime('%Y-%m-%d-%H-%M')
-  end
-
+$stdout.sync = true
+
+require 'fileutils'
+require 'json'
+require_relative 'lib/login_command'
+require_relative 'lib/navigator'
+require_relative 'lib/parallel_request_scraper'
+require_relative 'lib/request_finder'
+
+class TrackIt
+
+  def initialize(base_url, username, password, options={})
+    navigator = Navigator.new base_url
+    login_command = LoginCommand.new username, password, navigator
+    threads = options[:threads] || 1
+    @parallel_request_scraper = ParallelRequestScraper.new navigator, login_command, threads
+    @request_finder = RequestFinder.new navigator
+    @output_dir = options[:output_dir] || default_output_dir
+  end
+
+  def scrape_requests_for_users(users, options={})
+    request_ids = @request_finder.get_request_ids_for_users users, options
+    scrape_requests request_ids
+  end
+
+  def scrape_requests_in_range(from_id, to_id)
+    request_ids = (from_id..to_id).to_a
+    scrape_requests request_ids
+  end
+
+  private
+
+  def scrape_requests(request_ids)
+    FileUtils.mkdir_p @output_dir
+    @parallel_request_scraper.scrape(request_ids) { |request| write_request_file request }
+  end
+
+  def write_request_file(request)
+    File.write request_file_path(request), request.to_json
+    print '.'
+  end
+
+  def request_file_path(request)
+    File.join @output_dir, "#{request[:id]}.json"
+  end
+
+  def default_output_dir
+    File.join 'output', Time.now.strftime('%Y-%m-%d-%H-%M')
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: trackit_scraper
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-06 00:00:00.000000000 Z
+date: 2013-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70292605876980 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,15 @@ dependencies:
         version: 1.7.7
   type: :runtime
   prerelease: false
-  version_requirements: *70292605876980
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.7
 - !ruby/object:Gem::Dependency
   name: watir-webdriver
-  requirement: &70292605876500 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,7 +37,12 @@ dependencies:
         version: 0.6.4
   type: :runtime
   prerelease: false
-  version_requirements: *70292605876500
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.4
 description: 
 email: matthew-github@matthewriley.name
 executables: []
@@ -68,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Screen scrapes data from the Track-It help desk web application.