ferrumwizard 0.1.4 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96cb226b3acf38c7c4b4f8c82665175025effc7d2cbe25d0ae6026e92109c899
-  data.tar.gz: e31dbbc763e88c553881660d7579117781934e92c2eae10f32fff329da670e09
+  metadata.gz: 3b11fa25652b962c446a8022e3d9cfce067110ac2c6c6509995533449942ad70
+  data.tar.gz: e487524c0b613f6a1aea2872faccc520106cd57bbfbee5f9725504467c219e51
 SHA512:
-  metadata.gz: 0d6f8bd47a33d4d8d3224cc231a993a36c47fa297c91e7b83a3888094de658cf129cc289b001935b442fd96f93103bfe5e214551b5d365de18971330d0e380e2
-  data.tar.gz: 7fce37cf15f5a493b0261b36eb843ad8087a19b040c14176a0488b0ab0c84220b5e70c12fdb5a962443608092e148b7ecff2832311d7395209115e5cfec5e268
+  metadata.gz: fd67b1f378008cfb07e86fec087a1ce3344f5df905bde152d447b731d9936d3e26240932cf68a1a837acf10338ffbe22f689af378939e19cb2f42a47d112f7ab
+  data.tar.gz: a098e1b6ae39a3c5dc0dcfc35250a14993036ec4907e28cc9e3027d0ab3c259043086f5d177424abca3517cb58b59649a15487d68d93098abc6305d4459e3fee

data/lib/ferrumwizard.rb

@@ -2,6 +2,7 @@
 
 # file: ferrumwizard.rb
 
+require 'yaml'
 require 'rexle'
 require 'ferrum'
 
@@ -10,25 +11,66 @@ class FerrumWizard
 
   attr_reader :browser, :links, :radio, :buttons, :js_methods
 
-  def initialize(url, headless: true, timeout: 10, debug: false)
+  def initialize(url=nil, headless: true, timeout: 10, cookies: nil,
+                 debug: false)
 
     @url, @debug = url, debug
     @browser = Ferrum::Browser.new headless: headless, timeout: timeout
-    sleep 3
-    
-    if url
+
+    loadx(cookies) if cookies
+
+    if url then
+
+      sleep 3
+
       @browser.goto(@url)
       @browser.network.wait_for_idle
-      sleep 4      
+      sleep 4
+
     end
   end
-  
+
   def inspect()
     "#<FerrumWizard>"
   end
-  
+
+  # Intended to load all the cookies for a user to login automatically
+  #
+  # Follow these steps to load the cookies file:
+  #
+  #  1. launch the Ferrum browser
+  #  fw = FerrumWizard.new( headless: false, debug: false)
+  #
+  #  2. load the cookies before you visit the website
+  #  fw.load_cookies('/tmp/indeed2.txt')
+  #
+  #  3. visit the website
+  #  url='https://somewebsite.com'
+  #  fw.browser.goto(url)
+  #
+  def load_cookies(filepath)
+
+    rawcookies = YAML.load(File.read(filepath))
+
+    rawcookies.each do |h|
+
+      if @debug then
+        puts 'name: ' + h['name']
+        puts 'h: ' + h.inspect
+        sleep 0.7
+      end
+
+      browser.cookies.set(name: h['name'], value: h['value'],
+                          domain: h['domain'], expires: h['expires'],
+                          httponly: h['httpOnly'])
+    end
+
+  end
+
+  alias loadx load_cookies
+
   def login(usernamex=nil, passwordx=nil, username: usernamex, password: passwordx)
-    
+
     puts 'username: ' + username.inspect if @debug
 
     # search for the username input box
@@ -36,23 +78,33 @@ class FerrumWizard
     puts 'e_username: ' + e_username.inspect if @debug
     sleep 1
     # search for the password input box
-    e_password  = @browser.at_xpath('//input[@type="password"]')
+    found  = @browser.at_xpath('//input[@type="password"]')
+
+    e_password = if found then
+      found
+    else
+      @browser.xpath('//input').find {|x| x.property(:id) =~ /password/i}
+    end
+
     sleep 1
-    
+
     if username and e_username then
       puts 'entering the username' if @debug
-      e_username.focus.type(username)       
+      e_username.focus.type(username)
       sleep 1
     end
-    
+
     e_password.focus.type(password, :Enter) if e_password
-    
+
     after_login()
-    
-  end    
-  
+
+  end
+
+  # login2 is used for websites where the user is presented with the username
+  # input box on the first page and the password input box on the next page.
+  #
   def login2(usernamex=nil, passwordx=nil, username: usernamex, password: passwordx)
-    
+
     puts 'username: ' + username.inspect if @debug
 
     # search for the username input box
@@ -60,70 +112,104 @@ class FerrumWizard
     puts 'e_username: ' + e_username.inspect if @debug
     sleep 1
     # search for the password input box
-    
+
     if username and e_username then
       puts 'entering the username' if @debug
-      e_username.focus.type(username, :Enter)       
+      e_username.focus.type(username, :Enter)
       sleep 2
     end
 
-    e_password  = b.at_xpath('//input[@type="password"]')
+    e_password  = @browser.at_xpath('//input[@type="password"]')
     sleep 1
-    
+
     e_password.focus.type(password, :Enter) if e_password
-    
+
     after_login()
 
-    
-  end     
-  
+
+  end
+
   def quit
     @browser.quit
   end
-  
+
   def scan_page()
-    
-    @doc = Rexle.new @browser.body       
+
+    @doc = Rexle.new @browser.body
     fetch_links()
-    scan_form_elements()    
+    scan_form_elements()
     scan_js_links()
-    self    
+    @browser.mouse.scroll_to(0, 800)
+    self
+  end
+
+  # Saves all cookies for a given website into a YAML file
+  # see also load_cookies()
+  #
+  # To use this method follow these steps:
+  #
+  #   1. launch the web browser through Ferrum
+  #   fw = FerrumWizard.new(url, headless: false, debug: false)
+  #
+  #   2. go to the browser and login using your credentials
+  #   fw.save_cookies(filepath)
+  #
+  #   3. exit the IRB session
+  #
+  def save_cookies(filepath=Tempfile.new('ferrum').path)
+
+    rawcookies = @browser.cookies.all.keys.map do |key|
+
+      if @debug then
+        puts 'key: ' + key.inspect
+        sleep 0.5
+      end
+
+      s = @browser.cookies[key].inspect
+      a = s.scan(/"([^"]+)"=\>/)
+      s2 = s[/(?<=@attributes=).*(?=>)/]
+      eval(s2)
+
+    end
+
+    File.write filepath, rawcookies.to_yaml
+
   end
-  
+
   def submit(h)
 
     e = nil
-    
+
     h.each do |key, value|
-      e = @browser.xpath('//input').find {|x| x.attribute('name') == key.to_s}      
-      e.focus.type(value)      
+      e = @browser.xpath('//input').find {|x| x.attribute('name') == key.to_s}
+      e.focus.type(value)
     end
-    
-    e.focus.type('', :Enter)    
-    
-    sleep 4    
+
+    e.focus.type('', :Enter)
+
+    sleep 4
     scan_page()
-    
+
   end
 
   def to_rb()
   end
-  
+
   private
-  
+
   def after_login()
-    
-    @browser.network.wait_for_idle    
+
+    @browser.network.wait_for_idle
     sleep 4
     scan_page()
-    
+
     @browser.base_url = File.dirname(@browser.url)
     @browser.mouse.scroll_to(0, 800)
     self
-    
+
   end
-  
-  
+
+
   def fetch_buttons()
 
     a2 = @browser.xpath('//input[@type="button"]')
@@ -139,24 +225,24 @@ class FerrumWizard
     buttons = @buttons
 
     names.each do |name|
-      
+
       define_singleton_method name.to_sym do
         buttons[name].click
         @browser.network.wait_for_idle
         sleep = 1
         self
       end
-      
+
     end
 
   end
-  
+
   def fetch_links()
-     
+
     all_links = @doc.root.xpath('//a[@href]')
-    
+
     all_links.each do |x|
-      
+
       if x.plaintext.empty? then
         x.text = x.attributes[:href].sub(/\.\w+$/,'')[/([^\/]+)$/].split(/[_]|(?=[A-Z])/).join(' ')
       else
@@ -164,74 +250,74 @@ class FerrumWizard
       end
 
     end
-                                                                                                        
+
     valid_links = all_links.reject do |x|
-      
+
       puts 'x: ' + x.inspect if @debug
       r = (x.attributes[:target] == '_blank')
 
       puts 'r: ' + r.inspect if @debug
       r
-      
+
     end
-    
+
     indices = valid_links.map {|x| all_links.index x}
 
     active_links = @browser.xpath('//a[@href]')
     valid_active_links = indices.map {|n| active_links[n]}
-    
 
-    @links = valid_active_links.flat_map.with_index do |x, i| 
+
+    @links = valid_active_links.flat_map.with_index do |x, i|
 
       a = valid_links[i].text.split(/\W+/).map {|label| [label, x]}
       a << [valid_links[i].text, x]
-      
+
       puts 'a: ' + a.inspect if @debug
       a + a.map {|x2, obj| [x2.downcase, obj]}
-      
+
     end.to_h
-    
+
     names = @links.keys.map(&:downcase).uniq.select {|x| x =~ /^[\w ]+$/}
     links = @links
-    
+
     names.each do |name|
-      
+
       define_singleton_method name.gsub(/ +/,'_').to_sym do
-        
+
         links[name].click
         @browser.network.wait_for_idle
-        
+
         sleep 1
         scan_page()
         self
-        
+
       end
-      
+
     end
-    
+
   end
 
   def scan_form_elements()
-    
+
     # find radio buttons
-    
+
     a = @browser.xpath('//input[@type="radio"]')
     h = a.group_by {|x| x.attribute('name')}
     @radio = h.values
     define_singleton_method(:on) { @radio[0][0].click; self }
     define_singleton_method(:off) { @radio[0][1].click; self }
-    
+
     fetch_buttons()
-  
+
   end
-  
+
   def scan_js_links()
-    
+
     @js_methods = {}
     b = @browser
-    
+
     b.xpath('//a').select {|x| x.attribute('href') =~ /^javascript/}.each do |e|
-    
+
 
       s = e.attribute('href')[/(?<=^javascript:)[^\(]+/]
       puts 's: ' + s.inspect  if @debug
@@ -241,17 +327,17 @@ class FerrumWizard
       a << [s.split(/\W+|(?=[A-Z])/).join('_').downcase, s]
       #@js_methods[s] = a
 
-      a.concat a.map {|x, name| [x.downcase, name] }      
+      a.concat a.map {|x, name| [x.downcase, name] }
 
       puts 'a: ' + a.inspect if @debug
 
       a.uniq.select {|x, _| x =~ /^[a-z0-9_]+$/}.each do |x, name|
-        
+
         if @debug then
           puts 'x: ' + x.inspect
           puts 'name: ' + name.inspect
         end
-        
+
         define_singleton_method(x.to_sym) do |*args|
           #args = raw_args.map {|x| x[/^[0-9]+$/] ? x.to_i : x}
           js_method = "%s(%s)" % [name, args.map(&:inspect).join(', ')]
@@ -260,9 +346,9 @@ class FerrumWizard
           sleep 4
           self.scan_page()
         end
-        
+
       end
-      
+
     end
   end
 
@@ -271,7 +357,7 @@ class FerrumWizard
     puts 'method_missing: ' + method_name.inspect if @debug
     node = @browser.at_css '.' + method_name.to_s
     node.text if node
-    
-  end  
-  
+
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ferrumwizard
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.2
 platform: ruby
 authors:
 - James Robertson
@@ -11,31 +11,31 @@ cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
   MIIEXjCCAsagAwIBAgIBATANBgkqhkiG9w0BAQsFADAsMSowKAYDVQQDDCFnZW1t
-  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMjAwNzMwMTYxOTE5WhcN
-  MjEwNzMwMTYxOTE5WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
-  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDhNEIG
-  /Ab4nneih/AQFMcYk76JCiy26Xcy5uxd9ib7Emkj/9sZo6nxuSBaH03+Ixv3jgJs
-  TxZyaIKRsESFFmupYmKsyatCGGaBEsDb210ZBm313rP2Pk2fGrUtON0CjwJljWxR
-  8pHuglEXrGN/XhVicy7sZLJ2nVnvRtyiKi92XmY0S9LaCkWlOx2f3D7yiazkmHh5
-  59nHiGNlZ/SOzFrRMdBvkWZYHgqUEBv0KxEuMqW65U4HdlQImcwqu8XOWH9kutof
-  yyisv03kPqMvrOC8ptG/TieKYK0JuY23gS9MrVxkrf0gX3IQLY21JWG9t9uRImc/
-  kHC+EJ2rI8HQqcq/v6dndJb6MhYEhj7R5XsZqlfsLFo21FFBAyaPrqPRUstnW5U0
-  /tCpcuFyZJeRPqQ8LSlRGDuB/TdmV9dF+P5aGS32k9Okf9L6E6x3OGV29eMHSdDt
-  LOOB8l0EJbNXzpvYW+htziU8TbuzRQU8K7uTeAfpMUg4auPxdVyQpJcQWXcCAwEA
-  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU2+nN7PCw
-  js3NFmK8b17Ji/t+dvwwJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
+  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMjIwMzMwMTI1ODU1WhcN
+  MjMwMzMwMTI1ODU1WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
+  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDROL32
+  3LQKGcDR6x6XFa1US/Vq98DVnMeZHCSKdf471I4gJIOA7sQnrQTB6IZKTxb94Wjr
+  OSeGzlJpVq6pa7ltxvb9T7YQVVRrXYMC+u0gD9ukolnkpV/4Rh2/IIMxSNKncoZB
+  LKPseizGKlli4gs134gAu3wuWdCC7/UWPG/XyocdJC8tLtf/zi4JuRJTojKqYLOp
+  KsP9jHPmGVr81cW8HePmhQ/+LiYlKDE4Fwj4yl16XqhF7/5YOz9e5LOHsMUEord4
+  JscQ3GnhMfEXGJpwqCwNEpM3xAwcHp2DDdrwtT36ujSfnTJ3UpUIQUKVehA2i9rm
+  uDcDTr1PATGcOMPpExvLZu3a9uC81mj9z+axH5mWQ7jZ92sze79oAQTsMiMyBavJ
+  djSpnVBo71PFk8QekgIVVBIzG0iN5zoNUrSthvL/xUWXM6ea015HEDCCIEL417ID
+  humVWZyzKf7ITCdZWcxTgTgFfuPMctcICT5u7va+FrycYpdtt8kXvtD3VnkCAwEA
+  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUDipvmNU2
+  WydgAK8QPGb0vhhoGl0wJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
   c29uLmV1MCYGA1UdEgQfMB2BG2dlbW1hc3RlckBqYW1lc3JvYmVydHNvbi5ldTAN
-  BgkqhkiG9w0BAQsFAAOCAYEAlaYpTQ2vLuKU/nJl1inw9iE9XCwnTmIhmA9lnu1q
-  QKKCd7Z2PwtkahbDvMVQ347DQZQAanuZmtTPFMc4FDA530qJtwoYk03FTQXBh12M
-  d4C27VP9BOrUQcxkqtnTo+4Z60taszXqyPsPYU+Fd8AZUPeS5TOYG52OXTQ+q+pO
-  vNxkRP9oEka81ZrN1y3r3YaFHATZzf4pJo0HupZvMsQwa33/vA+xxxpDeTuWytNN
-  O0mYbo8Em2LnPnE8ehOnniDGXIIaDO9B1Qbbr0GhNCIWq3JIcbI2IBCKFWA6HyNF
-  yCdr7ZqPrnxXlhhnTPLFkzR/0+XxpbrdW4zb6uQqX92/tiUqP9uKf5dBEVoCWax/
-  IWPJE5JXx2iMvE9cWe4bFCUi7cZT7HsL6jkdUWxeTvsfc7XMbE8eWtHHiG6NjeFJ
-  7e24hNRMt3t/JE9ogEO4JzFUH2vq2zzR5X9JQqEclWfwHi4cf8bZFJ7spjZQPjSZ
-  Ok3rs0A+kW4ixAj1rDYuoyG/
+  BgkqhkiG9w0BAQsFAAOCAYEAHqRvm6iqjJ+bpEzSSgVmOMOMcgIoN6px1LMVAOmY
+  BJpF5F0fJr99thc1EYZJoRTwEcXJYhCTqKg+3xhNKpCzk2qHsaLKYEygPeBpyJOg
+  LyfHLrj98QLPYyFzqhWsqZAAAGC9WSF/kBJazpuotU2ec/Xw/e3NPopedV/Zvuhs
+  +/OKZWwRez/hg97ckaCYAp/7OdrVhJvR87MaQnN52Uk8OQbuPSyUNQUJ044HWHtu
+  lEJjsDetEFhNB69j3wAIMjMEZao29/dZhALbUDp+9ewK7uYbrX9Bo68NX+H+XcCZ
+  VFrdrjkyJUOHwSmvjYXN1V0Yz8kVVFU7E+Q4RHL8yAwBv+ynd927HtZVjs+455Pc
+  z+9gNpBQVr6LLXLJgJF2pTaIoYhgG6pcoMQHGVoxWdKzvOcl0h1epeJSp/aynX/r
+  FK+cyrQNA9DLJYJuz6uO7Z+gXZWjwAO38LUwF01w49asSv/5ZH2HH/EauX5xWpe+
+  ry6lYQlb8j50Iys5elAy1p0i
   -----END CERTIFICATE-----
-date: 2020-08-01 00:00:00.000000000 Z
+date: 2022-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rexle
@@ -46,7 +46,7 @@ dependencies:
         version: '1.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.7
+        version: 1.5.14
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -56,29 +56,29 @@ dependencies:
         version: '1.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.7
+        version: 1.5.14
 - !ruby/object:Gem::Dependency
   name: ferrum
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.9.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0.11'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.9.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0.11'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
 description: 
-email: james@jamesrobertson.eu
+email: digital.robertson@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -103,7 +103,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: Makes web scraping easier using the Ferrum gem.