trackit_scraper 1.0.0

data/lib/trackit_scraper/2003.10.1/lib/login_command.rb

@@ -0,0 +1,22 @@
+class LoginCommand
+
+  def initialize(username, password, navigator)
+    @username = username
+    @password = password
+    @navigator = navigator
+  end
+
+  def execute(b)
+    @navigator.goto 'hd/index.ssp', b
+    login b
+  end
+
+  private
+
+  def login(b)
+    b.text_field(name: 'user_id').set @username
+    b.text_field(name: 'user_pwd').set @password
+    b.button(value: 'Log on').click
+  end
+
+end

data/lib/trackit_scraper/2003.10.1/lib/navigator.rb

@@ -0,0 +1,12 @@
+class Navigator
+
+  def initialize(base_url)
+    @base_url = base_url
+  end
+
+  def goto(relative_url, b)
+    url = File.join @base_url, relative_url
+    b.goto url
+  end
+
+end

data/lib/trackit_scraper/2003.10.1/lib/parallel_request_scraper.rb

@@ -0,0 +1,45 @@
+require 'thread'
+require 'watir-webdriver'
+require_relative 'request_page'
+
+class ParallelRequestScraper
+
+  def initialize(navigator, login_command, threads)
+    @navigator = navigator
+    @login_command = login_command
+    @threads = threads
+  end
+
+  def scrape(request_ids, &block)
+    @request_ids = request_ids.clone
+    @semaphore = Mutex.new
+    @threads.times.map { Thread.new { scrape_requests &block } }.each { |t| t.join }
+  end
+
+  private
+
+  def scrape_requests
+    b = Watir::Browser.new
+    @login_command.execute b
+
+    until @request_ids.empty?
+      request = scrape_next_request b
+      request ? yield(request) : break
+    end
+
+    b.close
+  end
+
+  def scrape_next_request(b)
+    id = next_request_id
+    return unless id
+    @navigator.goto "hd/ticket/euTicketView.ssp?ticket_id=#{id}&log=show", b
+    request_page = RequestPage.new b
+    request_page.get_request
+  end
+
+  def next_request_id
+    @semaphore.synchronize { return @request_ids.pop }
+  end
+
+end

data/lib/trackit_scraper/2003.10.1/lib/request_finder.rb

@@ -0,0 +1,31 @@
+require 'watir-webdriver'
+
+class RequestFinder
+
+  def initialize(navigator)
+    @navigator = navigator
+  end
+
+  def get_request_ids_for_users(users)
+    b = Watir::Browser.new
+    request_ids = users.map { |user| get_request_ids_for_user user, b }.flatten.sort
+    b.close
+    request_ids
+  end
+
+  private
+
+  def get_request_ids_for_user(user, b)
+    login_command = LoginCommand.new user[:username], user[:password], @navigator
+    login_command.execute b
+    @navigator.goto 'hd/ticket/euTicketFind.ssp', b
+    get_all_request_ids b
+  end
+
+  def get_all_request_ids(b)
+    b.button(value: 'Find').click
+    b.links(href: /ticket_id=/).to_a.map { |link| link.href[/ticket_id=(\d+)/, 1].to_i }
+  end
+
+end
+

data/lib/trackit_scraper/2003.10.1/lib/request_history_table.rb

@@ -0,0 +1,25 @@
+require 'time'
+
+class RequestHistoryTable
+
+  def initialize(table)
+    @table = table
+  end
+
+  def get_resolution_info
+    r = {}
+    rows = @table.trs.to_a
+    resolved_row_index = rows.index { |r| r.text =~ /Resolved by/ }
+
+    if resolved_row_index
+      row_above = rows[resolved_row_index-1]
+      captures = row_above.text.scan(/(.+) by (.+)/)[0]
+      r[:resolved_on] = Time.parse captures[0]
+      r[:resolved_by] = captures[1]
+    end
+
+    r[:history] = @table.text
+    r
+  end
+
+end

data/lib/trackit_scraper/2003.10.1/lib/request_info_table.rb

@@ -0,0 +1,43 @@
+class RequestInfoTable
+
+  def initialize(table)
+    @cells = table.tds.to_a
+  end
+
+  def get_request_info
+    {
+        title: cell(0),
+        status: cell(2),
+        service: cell(3),
+        request_type: cell(4),
+        time_spent: cell(5),
+        priority: cell(7),
+        deadline: time_cell(8),
+        submitted_to: cell(10),
+        submitted_by: cell(11),
+        submitted_on: time_cell(12),
+        assigned_to: cell(15),
+        assigned_by: cell(16),
+        assigned_on: time_cell(17),
+        department_id: cell(19),
+        closed_by: cell(20),
+        closed_on: maybe_time_cell(21, 'None')
+    }
+  end
+
+  private
+
+  def maybe_time_cell(index, non_time_value)
+    cell(index) { |s| s == non_time_value ? cell(index) : time_cell(index) }
+  end
+
+  def time_cell(index)
+    cell(index) { |s| Time.parse s }
+  end
+
+  def cell(index)
+    text = @cells[index].text
+    block_given? ? yield(text) : text
+  end
+
+end

data/lib/trackit_scraper/2003.10.1/lib/request_page.rb

@@ -0,0 +1,25 @@
+require_relative 'request_info_table'
+require_relative 'request_history_table'
+
+class RequestPage
+
+  def initialize(b)
+    @b = b
+  end
+
+  def get_request
+    request_info_table = RequestInfoTable.new @b.tables[1]
+    request_history_table = RequestHistoryTable.new @b.tables[3]
+    r = { id: request_id }
+    r.merge! request_info_table.get_request_info
+    r.merge! request_history_table.get_resolution_info
+    r
+  end
+
+  private
+
+  def request_id
+    @b.text[/Request #(\d+)/, 1]
+  end
+
+end

data/lib/trackit_scraper/2003.10.1/trackit.rb

@@ -0,0 +1,51 @@
+$stdout.sync = true
+
+require 'fileutils'
+require 'json'
+require_relative 'lib/login_command'
+require_relative 'lib/navigator'
+require_relative 'lib/parallel_request_scraper'
+require_relative 'lib/request_finder'
+
+class TrackIt
+
+  def initialize(base_url, username, password, options={})
+    navigator = Navigator.new base_url
+    login_command = LoginCommand.new username, password, navigator
+    threads = options[:threads] || 1
+    @parallel_request_scraper = ParallelRequestScraper.new navigator, login_command, threads
+    @request_finder = RequestFinder.new navigator
+    @output_dir = options[:output_dir] || default_output_dir
+  end
+
+  def scrape_requests_for_users(users)
+    request_ids = @request_finder.get_request_ids_for_users users
+    scrape_requests request_ids
+  end
+
+  def scrape_requests_in_range(from_id, to_id)
+    request_ids = (from_id..to_id).to_a
+    scrape_requests request_ids
+  end
+
+  private
+
+  def scrape_requests(request_ids)
+    FileUtils.mkdir_p @output_dir
+    @parallel_request_scraper.scrape(request_ids) { |request| write_request_file request }
+  end
+
+  def write_request_file(request)
+    File.write request_file_path(request), request.to_json
+    print '.'
+  end
+
+  def request_file_path(request)
+    File.join @output_dir, "#{request[:id]}.json"
+  end
+
+  def default_output_dir
+    File.join 'output', Time.now.strftime('%Y-%m-%d-%H-%M')
+  end
+
+end

data/lib/trackit_scraper/2003.10.1.rb

@@ -0,0 +1 @@
+require_relative '2003.10.1/trackit'

metadata

@@ -0,0 +1,75 @@
+--- !ruby/object:Gem::Specification
+name: trackit_scraper
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Matthew Riley
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &70292605876980 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.7
+  type: :runtime
+  prerelease: false
+  version_requirements: *70292605876980
+- !ruby/object:Gem::Dependency
+  name: watir-webdriver
+  requirement: &70292605876500 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.4
+  type: :runtime
+  prerelease: false
+  version_requirements: *70292605876500
+description: 
+email: matthew-github@matthewriley.name
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/trackit_scraper/2003.10.1/lib/login_command.rb
+- lib/trackit_scraper/2003.10.1/lib/navigator.rb
+- lib/trackit_scraper/2003.10.1/lib/parallel_request_scraper.rb
+- lib/trackit_scraper/2003.10.1/lib/request_finder.rb
+- lib/trackit_scraper/2003.10.1/lib/request_history_table.rb
+- lib/trackit_scraper/2003.10.1/lib/request_info_table.rb
+- lib/trackit_scraper/2003.10.1/lib/request_page.rb
+- lib/trackit_scraper/2003.10.1/trackit.rb
+- lib/trackit_scraper/2003.10.1.rb
+homepage: 
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Screen scrapes data from the Track-It help desk web application.
+test_files: []