ferrumwizard 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2940cdfbf2aff12f2f0f70d219b46483f1561a3bf3a900299837644e1752a66f
-  data.tar.gz: f747b47b9ebddb10046b559de95952b48c39dc3375d617f2d246c9bcb4cc697c
+  metadata.gz: 4b012bc3e8cb9b91311c91eecef6b816cae7bc3d179dc037ac0011373e1d54d2
+  data.tar.gz: f0421333194ed8b204ea87beab02314a44f978bbafa28809622b89d1db26bd4f
 SHA512:
-  metadata.gz: 5e9bdf0508b48921682828926525b914b276269380c8871ea0ce70263878948fc27aab93d1ab1bf9c53b906b9cbebd2b3d19548b3595405b238b3c5935428174
-  data.tar.gz: f6137500af8d41c75f077219ada94581cd0a5fc8313315a7fa92c0a883ec055613578b42e37831810d83d44215dae09f64ff31b29c6569e8aac88ca8362a9f61
+  metadata.gz: b96dd21840cbed0d1253f4795ddf4a4cf36250d5d3736b918c70c7c48ef63d4e8ba31592395cce6de691049a076f41186337a92643e3e1748e122f58edc51905
+  data.tar.gz: 39d7c5cc96e6900db1bbfa668ba9b972d49f09147d0fc5b079c34fd40e7235bb256a1713c1bf5725a16c0087dca959d5df8b12b7f10e093891478bf0a33f53f2

data/lib/ferrumwizard.rb

@@ -2,6 +2,7 @@
 
 # file: ferrumwizard.rb
 
+require 'yaml'
 require 'rexle'
 require 'ferrum'
 
@@ -10,67 +11,197 @@ class FerrumWizard
 
   attr_reader :browser, :links, :radio, :buttons, :js_methods
 
-  def initialize(url, headless: true, debug: false)
+  def initialize(url=nil, headless: true, timeout: 10, debug: false)
 
     @url, @debug = url, debug
-    @browser = Ferrum::Browser.new headless: headless
-    sleep 2
+    @browser = Ferrum::Browser.new headless: headless, timeout: timeout
+    sleep 3
+
+    if url
+      @browser.goto(@url)
+      @browser.network.wait_for_idle
+      sleep 4
+    end
   end
-  
+
   def inspect()
     "#<FerrumWizard>"
   end
-  
+
+  # Intended to load all the cookies for a user to login automatically
+  #
+  # Follow these steps to load the cookies file:
+  #
+  #  1. launch the Ferrum browser
+  #  fw = FerrumWizard.new( headless: false, debug: false)
+  #
+  #  2. load the cookies before you visit the website
+  #  fw.load_cookies('/tmp/indeed2.txt')
+  #
+  #  3. visit the website
+  #  url='https://somewebsite.com'
+  #  fw.browser.goto(url)
+  #
+  def load_cookies(filepath)
+
+    rawcookies = YAML.load(File.read(filepath))
+
+    rawcookies.each do |h|
+
+      if @debug then
+        puts 'name: ' + h['name']
+        puts 'h: ' + h.inspect
+        sleep 0.7
+      end
+
+      browser.cookies.set(name: h['name'], value: h['value'],
+                          domain: h['domain'], expires: h['expires'],
+                          httponly: h['httpOnly'])
+    end
+
+  end
+
   def login(usernamex=nil, passwordx=nil, username: usernamex, password: passwordx)
-    
-    puts 'username: ' + username.inspect if @debug
 
-    b = @browser
-    b.goto(@url)
-    @browser.network.wait_for_idle
-    sleep 3
+    puts 'username: ' + username.inspect if @debug
 
     # search for the username input box
-    e_username = b.at_xpath('//input[@type="email"]')
+    e_username = @browser.at_xpath('//input[@type="email"]')
     puts 'e_username: ' + e_username.inspect if @debug
     sleep 1
     # search for the password input box
-    e_password  = b.at_xpath('//input[@type="password"]')
+    found  = @browser.at_xpath('//input[@type="password"]')
+
+    e_password = if found then
+      found
+    else
+      @browser.xpath('//input').find {|x| x.property(:id) =~ /password/i}
+    end
+
     sleep 1
-    
+
     if username and e_username then
       puts 'entering the username' if @debug
-      e_username.focus.type(username)       
+      e_username.focus.type(username)
       sleep 1
     end
-    
+
     e_password.focus.type(password, :Enter) if e_password
-    @browser.network.wait_for_idle
-    
-    sleep 4
 
-    scan_page()
-    
-  end    
-  
+    after_login()
+
+  end
+
+  # login2 is used for websites where the user is presented with the username
+  # input box on the first page and the password input box on the next page.
+  #
+  def login2(usernamex=nil, passwordx=nil, username: usernamex, password: passwordx)
+
+    puts 'username: ' + username.inspect if @debug
+
+    # search for the username input box
+    e_username = @browser.at_xpath('//input[@type="email"]')
+    puts 'e_username: ' + e_username.inspect if @debug
+    sleep 1
+    # search for the password input box
+
+    if username and e_username then
+      puts 'entering the username' if @debug
+      e_username.focus.type(username, :Enter)
+      sleep 2
+    end
+
+    e_password  = @browser.at_xpath('//input[@type="password"]')
+    sleep 1
+
+    e_password.focus.type(password, :Enter) if e_password
+
+    after_login()
+
+
+  end
+
   def quit
     @browser.quit
   end
-  
+
   def scan_page()
-    
-    @doc = Rexle.new @browser.body       
+
+    @doc = Rexle.new @browser.body
     fetch_links()
-    scan_form_elements()    
+    scan_form_elements()
     scan_js_links()
+    @browser.mouse.scroll_to(0, 800)
     self
   end
 
+  # Saves all cookies for a given website into a YAML file
+  # see also load_cookies()
+  #
+  # To use this method follow these steps:
+  #
+  #   1. launch the web browser through Ferrum
+  #   fw = FerrumWizard.new(url, headless: false, debug: false)
+  #
+  #   2. go to the browser and login using your credentials
+  #   fw.save_cookies(filepath)
+  #
+  #   3. exit the IRB session
+  #
+  def save_cookies(filepath=Tempfile.new('ferrum').path)
+
+    rawcookies = @browser.cookies.all.keys.map do |key|
+
+      if @debug then
+        puts 'key: ' + key.inspect
+        sleep 0.5
+      end
+
+      s = @browser.cookies[key].inspect
+      a = s.scan(/"([^"]+)"=\>/)
+      s2 = s[/(?<=@attributes=).*(?=>)/]
+      eval(s2)
+
+    end
+
+    File.write filepath, rawcookies.to_yaml
+
+  end
+
+  def submit(h)
+
+    e = nil
+
+    h.each do |key, value|
+      e = @browser.xpath('//input').find {|x| x.attribute('name') == key.to_s}
+      e.focus.type(value)
+    end
+
+    e.focus.type('', :Enter)
+
+    sleep 4
+    scan_page()
+
+  end
+
   def to_rb()
   end
-  
+
   private
-  
+
+  def after_login()
+
+    @browser.network.wait_for_idle
+    sleep 4
+    scan_page()
+
+    @browser.base_url = File.dirname(@browser.url)
+    @browser.mouse.scroll_to(0, 800)
+    self
+
+  end
+
+
   def fetch_buttons()
 
     a2 = @browser.xpath('//input[@type="button"]')
@@ -86,24 +217,24 @@ class FerrumWizard
     buttons = @buttons
 
     names.each do |name|
-      
+
       define_singleton_method name.to_sym do
         buttons[name].click
         @browser.network.wait_for_idle
         sleep = 1
         self
       end
-      
+
     end
 
   end
-  
+
   def fetch_links()
-     
-    all_links = @doc.root.xpath('//a')
-    
+
+    all_links = @doc.root.xpath('//a[@href]')
+
     all_links.each do |x|
-      
+
       if x.plaintext.empty? then
         x.text = x.attributes[:href].sub(/\.\w+$/,'')[/([^\/]+)$/].split(/[_]|(?=[A-Z])/).join(' ')
       else
@@ -111,90 +242,94 @@ class FerrumWizard
       end
 
     end
-                                                                                                        
+
     valid_links = all_links.reject do |x|
-      
+
       puts 'x: ' + x.inspect if @debug
       r = (x.attributes[:target] == '_blank')
 
       puts 'r: ' + r.inspect if @debug
       r
-      
+
     end
+
     indices = valid_links.map {|x| all_links.index x}
 
-    active_links = @browser.xpath('//a')
+    active_links = @browser.xpath('//a[@href]')
     valid_active_links = indices.map {|n| active_links[n]}
-    
 
-    @links = valid_active_links.flat_map.with_index do |x, i| 
+
+    @links = valid_active_links.flat_map.with_index do |x, i|
 
       a = valid_links[i].text.split(/\W+/).map {|label| [label, x]}
       a << [valid_links[i].text, x]
-      
+
       puts 'a: ' + a.inspect if @debug
-      a + a.map {|x, obj| [x.downcase, obj]}
+      a + a.map {|x2, obj| [x2.downcase, obj]}
+
     end.to_h
-    
+
     names = @links.keys.map(&:downcase).uniq.select {|x| x =~ /^[\w ]+$/}
     links = @links
-    
+
     names.each do |name|
-      
+
       define_singleton_method name.gsub(/ +/,'_').to_sym do
+
         links[name].click
+        @browser.network.wait_for_idle
+
         sleep 1
         scan_page()
         self
+
       end
-      
+
     end
-    
+
   end
 
   def scan_form_elements()
-    
+
     # find radio buttons
-    
-    #a = doc.root.xpath('//input[@type="radio"]')
+
     a = @browser.xpath('//input[@type="radio"]')
-    #h = a.group_by {|x| x.attributes[:name]}
     h = a.group_by {|x| x.attribute('name')}
     @radio = h.values
     define_singleton_method(:on) { @radio[0][0].click; self }
     define_singleton_method(:off) { @radio[0][1].click; self }
-    
+
     fetch_buttons()
-  
+
   end
-  
+
   def scan_js_links()
-    
+
     @js_methods = {}
     b = @browser
-    
+
     b.xpath('//a').select {|x| x.attribute('href') =~ /^javascript/}.each do |e|
-    
+
 
       s = e.attribute('href')[/(?<=^javascript:)[^\(]+/]
-      puts 's: ' + s.inspect  
+      puts 's: ' + s.inspect  if @debug
       a = s.split(/\W+|(?=[A-Z])/).map {|label| [label, s]}
       a << [s, s]
       a << [s.split(/\W+|(?=[A-Z])/).join('_'), s]
       a << [s.split(/\W+|(?=[A-Z])/).join('_').downcase, s]
       #@js_methods[s] = a
 
-      a.concat a.map {|x, name| [x.downcase, name] }      
+      a.concat a.map {|x, name| [x.downcase, name] }
 
-      puts 'a: ' + a.inspect
+      puts 'a: ' + a.inspect if @debug
 
       a.uniq.select {|x, _| x =~ /^[a-z0-9_]+$/}.each do |x, name|
-        
+
         if @debug then
           puts 'x: ' + x.inspect
           puts 'name: ' + name.inspect
         end
-        
+
         define_singleton_method(x.to_sym) do |*args|
           #args = raw_args.map {|x| x[/^[0-9]+$/] ? x.to_i : x}
           js_method = "%s(%s)" % [name, args.map(&:inspect).join(', ')]
@@ -203,9 +338,9 @@ class FerrumWizard
           sleep 4
           self.scan_page()
         end
-        
+
       end
-      
+
     end
   end
 
@@ -214,7 +349,7 @@ class FerrumWizard
     puts 'method_missing: ' + method_name.inspect if @debug
     node = @browser.at_css '.' + method_name.to_s
     node.text if node
-    
-  end  
-  
+
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ferrumwizard
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - James Robertson
@@ -11,31 +11,31 @@ cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
   MIIEXjCCAsagAwIBAgIBATANBgkqhkiG9w0BAQsFADAsMSowKAYDVQQDDCFnZW1t
-  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMjAwNzMwMTYxOTE5WhcN
-  MjEwNzMwMTYxOTE5WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
-  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDhNEIG
-  /Ab4nneih/AQFMcYk76JCiy26Xcy5uxd9ib7Emkj/9sZo6nxuSBaH03+Ixv3jgJs
-  TxZyaIKRsESFFmupYmKsyatCGGaBEsDb210ZBm313rP2Pk2fGrUtON0CjwJljWxR
-  8pHuglEXrGN/XhVicy7sZLJ2nVnvRtyiKi92XmY0S9LaCkWlOx2f3D7yiazkmHh5
-  59nHiGNlZ/SOzFrRMdBvkWZYHgqUEBv0KxEuMqW65U4HdlQImcwqu8XOWH9kutof
-  yyisv03kPqMvrOC8ptG/TieKYK0JuY23gS9MrVxkrf0gX3IQLY21JWG9t9uRImc/
-  kHC+EJ2rI8HQqcq/v6dndJb6MhYEhj7R5XsZqlfsLFo21FFBAyaPrqPRUstnW5U0
-  /tCpcuFyZJeRPqQ8LSlRGDuB/TdmV9dF+P5aGS32k9Okf9L6E6x3OGV29eMHSdDt
-  LOOB8l0EJbNXzpvYW+htziU8TbuzRQU8K7uTeAfpMUg4auPxdVyQpJcQWXcCAwEA
-  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU2+nN7PCw
-  js3NFmK8b17Ji/t+dvwwJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
+  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMjIwMzMwMTI1ODU1WhcN
+  MjMwMzMwMTI1ODU1WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
+  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDROL32
+  3LQKGcDR6x6XFa1US/Vq98DVnMeZHCSKdf471I4gJIOA7sQnrQTB6IZKTxb94Wjr
+  OSeGzlJpVq6pa7ltxvb9T7YQVVRrXYMC+u0gD9ukolnkpV/4Rh2/IIMxSNKncoZB
+  LKPseizGKlli4gs134gAu3wuWdCC7/UWPG/XyocdJC8tLtf/zi4JuRJTojKqYLOp
+  KsP9jHPmGVr81cW8HePmhQ/+LiYlKDE4Fwj4yl16XqhF7/5YOz9e5LOHsMUEord4
+  JscQ3GnhMfEXGJpwqCwNEpM3xAwcHp2DDdrwtT36ujSfnTJ3UpUIQUKVehA2i9rm
+  uDcDTr1PATGcOMPpExvLZu3a9uC81mj9z+axH5mWQ7jZ92sze79oAQTsMiMyBavJ
+  djSpnVBo71PFk8QekgIVVBIzG0iN5zoNUrSthvL/xUWXM6ea015HEDCCIEL417ID
+  humVWZyzKf7ITCdZWcxTgTgFfuPMctcICT5u7va+FrycYpdtt8kXvtD3VnkCAwEA
+  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUDipvmNU2
+  WydgAK8QPGb0vhhoGl0wJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
   c29uLmV1MCYGA1UdEgQfMB2BG2dlbW1hc3RlckBqYW1lc3JvYmVydHNvbi5ldTAN
-  BgkqhkiG9w0BAQsFAAOCAYEAlaYpTQ2vLuKU/nJl1inw9iE9XCwnTmIhmA9lnu1q
-  QKKCd7Z2PwtkahbDvMVQ347DQZQAanuZmtTPFMc4FDA530qJtwoYk03FTQXBh12M
-  d4C27VP9BOrUQcxkqtnTo+4Z60taszXqyPsPYU+Fd8AZUPeS5TOYG52OXTQ+q+pO
-  vNxkRP9oEka81ZrN1y3r3YaFHATZzf4pJo0HupZvMsQwa33/vA+xxxpDeTuWytNN
-  O0mYbo8Em2LnPnE8ehOnniDGXIIaDO9B1Qbbr0GhNCIWq3JIcbI2IBCKFWA6HyNF
-  yCdr7ZqPrnxXlhhnTPLFkzR/0+XxpbrdW4zb6uQqX92/tiUqP9uKf5dBEVoCWax/
-  IWPJE5JXx2iMvE9cWe4bFCUi7cZT7HsL6jkdUWxeTvsfc7XMbE8eWtHHiG6NjeFJ
-  7e24hNRMt3t/JE9ogEO4JzFUH2vq2zzR5X9JQqEclWfwHi4cf8bZFJ7spjZQPjSZ
-  Ok3rs0A+kW4ixAj1rDYuoyG/
+  BgkqhkiG9w0BAQsFAAOCAYEAHqRvm6iqjJ+bpEzSSgVmOMOMcgIoN6px1LMVAOmY
+  BJpF5F0fJr99thc1EYZJoRTwEcXJYhCTqKg+3xhNKpCzk2qHsaLKYEygPeBpyJOg
+  LyfHLrj98QLPYyFzqhWsqZAAAGC9WSF/kBJazpuotU2ec/Xw/e3NPopedV/Zvuhs
+  +/OKZWwRez/hg97ckaCYAp/7OdrVhJvR87MaQnN52Uk8OQbuPSyUNQUJ044HWHtu
+  lEJjsDetEFhNB69j3wAIMjMEZao29/dZhALbUDp+9ewK7uYbrX9Bo68NX+H+XcCZ
+  VFrdrjkyJUOHwSmvjYXN1V0Yz8kVVFU7E+Q4RHL8yAwBv+ynd927HtZVjs+455Pc
+  z+9gNpBQVr6LLXLJgJF2pTaIoYhgG6pcoMQHGVoxWdKzvOcl0h1epeJSp/aynX/r
+  FK+cyrQNA9DLJYJuz6uO7Z+gXZWjwAO38LUwF01w49asSv/5ZH2HH/EauX5xWpe+
+  ry6lYQlb8j50Iys5elAy1p0i
   -----END CERTIFICATE-----
-date: 2020-08-01 00:00:00.000000000 Z
+date: 2022-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rexle
@@ -46,7 +46,7 @@ dependencies:
         version: '1.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.7
+        version: 1.5.14
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -56,29 +56,29 @@ dependencies:
         version: '1.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.7
+        version: 1.5.14
 - !ruby/object:Gem::Dependency
   name: ferrum
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.9.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0.11'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.9.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0.11'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
 description: 
-email: james@jamesrobertson.eu
+email: digital.robertson@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -103,7 +103,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: Makes web scraping easier using the Ferrum gem.