formscraper_helper 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b4f9be47b4c46d2e2823161ac508b4748afab4f2f6a03e2617bb91ae6ababe9
-  data.tar.gz: bea40e151206763162b372bb2163010f65ce021fbd795b82757780eba32ccde2
+  metadata.gz: 402c3ab5b633cce3e4852deba197a388a30d267e81595829cbc05deb6509f07e
+  data.tar.gz: c3867d152abfa6d910efd7f24b119c413c8ca8368229290583704bba1018108d
 SHA512:
-  metadata.gz: 8c7809564deb15b0b6a1ce109a676ceed9f77211dcf30fe14894a638ad75609421de74cc6863064d7c4fdb428b99fdffbb9ae777598bd3c32c99e0ca542f4860
-  data.tar.gz: a61eaf497175a3bb4aac7eed0e3e54038640697a90484c05adc11fa8342f1c668f94255488d1a6a6e932c8a42e0143b821af1bd98f4256605e86da92a9c4f218
+  metadata.gz: a3c5c2217823d8734d7e273a07c166012191f0196a3c039cbb7bb55a0ee2ab3593295b9e985c049ebcfdddaa768696091fa49352cb8af3c7782ac56a25d86dd2
+  data.tar.gz: 15b49ce36dd5517099e04436153f3e63c1f9138a383ddb22033e4896eb9b0203c32bcca28ec18ab1883bc452a175182e21700cbca4cd812cdb618ad0b571abcd

data/lib/formscraper_helper.rb

@@ -4,17 +4,21 @@
 
 require 'ferrum'
 require 'nokorexi'
+require 'clipboard'
 
 
 class FormScraperHelper
 
   attr_reader :browser
 
-  def initialize(url, headless: false, debug: false)
+  # note: fd corresponds to FakeDataGenerator22 which is optional
+  #
+  def initialize(url, headless: false, clipb: true, fd: nil, debug: false)
 
-    @url, @debug = url, debug
+    @url, @clipb, @fd, @debug = url, clipb, fd, debug
     @browser = Ferrum::Browser.new  headless: headless
     @browser.goto(url)
+
     sleep 2
     scrape()
 
@@ -40,6 +44,7 @@ doc = Nokorexi.new(browser.body).to_doc
 
 # load the YAML document containing the inputs
 #filepath = ''
+filepath = '/tmp/tmp.yaml'
 h = YAML.load(File.read(filepath))
 EOF
 
@@ -51,29 +56,26 @@ EOF
 
       if h[:type] == 'text' or h[:type] == 'password' then
 
-        var1 = if h[:title].length > 1 then
-          h[:title].downcase.gsub(/ +/,'_')
-        else 
-          key.downcase
-        end
+        var1, s2 = format_var1(h[:title], key)
+        s += s2
         s += var1 + " = h['#{var1}']\n"
-        s += "r.focus.type #{var1}\n\n"
+        s += "r.focus.type #{var1}\n"
+        s += "sleep 0.5\n\n"
 
       elsif h[:type] == 'select'
 
-        var1 = if h[:title].length > 1 then
-          h[:title].downcase.gsub(/ +/,'_').gsub(/\W/,'')
-        else 
-          key.downcase
-        end
+        var1, s2 = format_var1(h[:title], key)
+        s += s2
 
         s += "# options: #{h[:options].join(', ')}\n"
         s += "#{var1} = h['#{var1}']\n"
-        s += 'r = titles.grep /#{' + var1 + '}/i' + "\n"
-        s += "n = titles.index(r.first) + 1\n"
+        s += 'titles = %w(' + h[:options].join(' ') + ')' + "\n"
+        s += 'found = titles.grep /#{' + var1 + '}/i' + "\n"
+        s += "n = titles.index(found.first) + 1\n"
         s += "r.focus\n"
         s += "n.times { r.type(:down); sleep 1}\n"
-        s += "r.click\n\n"
+        s += "r.click\n"
+        s += "sleep 0.5\n\n"
 
       elsif h[:type] == 'checkbox'
         s += "r.focus.click\n\n"
@@ -81,6 +83,9 @@ EOF
 
     end
 
+    Clipboard.copy s if @clipb
+    puts 'generated code copied to clipboard'
+
     return s
 
   end
@@ -97,22 +102,30 @@ EOF
 
       if h[:type] == 'text' or h[:type] == 'password' then
 
-        var1 = if h[:title].length > 1 then
-          h[:title].downcase.gsub(/ +/,'_')
-        else 
-          key.downcase
+        var1, s2 = format_var1(h[:title], key)
+
+        s += s2
+
+        if h[:type] == 'password' then
+          @pwd ||= @fd ? @fd.password : 'xxx'
+          s += var1 + ": #{@pwd}\n"
+        elsif @fd
+
+          found = @fd.lookup var1
+          val = found.is_a?(String) ? found : 'xxx'
+          s += var1 + ": #{val}\n"
+        else
+          s += var1 + ": xxx\n"
         end
-        s += var1 + ": xxx\n"
 
       elsif h[:type] == 'select'
 
-        var1 = if h[:title].length > 1 then
-          h[:title].downcase.gsub(/ +/,'_').gsub(/\W/,'')
-        else 
-          key.downcase
-        end
+        var1, s2 = format_var1(h[:title], key)
 
-        s += "#{var1}: xxx\n"
+        s += s2
+        s += "# options: #{h[:options].join(', ')}\n"
+        val = h[:options][1..-1].sample
+        s += "#{var1}: #{val}\n"
 
       elsif h[:type] == 'checkbox'
 
@@ -120,20 +133,49 @@ EOF
 
     end
 
+    Clipboard.copy s if @clipb
+    puts 'generated YAML copied to clipboard'
+
     return s
 
   end
 
   private
 
-  def scrape()  
+  # returns var1 using arguments rawtitle or key
+  # note: argument s is passed by reference
+  #
+  def format_var1(rawtitle, key)
+
+    var1 = if rawtitle.length > 1 then
+
+      s = "\n# " + rawtitle + "\n"
+      title = rawtitle.scan(/[A-Z][^A-Z]+/).join(' ').gsub(/[^\w ]/,'')
+      words = title.downcase.scan(/\w+/)
+
+      if words.count > 2 then
+        words.take(5).map {|x| x[0]}.join
+      else
+        title.downcase.gsub(/ +/,'_')
+      end
+
+    else
+      newtitle = key.scan(/[A-Z][^A-Z]+/).join(' ')
+      s = "\n# " + newtitle + "\n"
+      newtitle.gsub(/[^\w ]/,'').downcase\
+          .gsub(/ +/,'_')
+    end
+
+    [var1, s]
+
+  end
+
+  def scrape()
 
     doc = Nokorexi.new(@browser.body).to_doc
 
     #a = doc.root.xpath('//input|//select')
-    a = doc.root.xpath('//*').select do |x|
-      x.name == 'input' or x.name == 'select'
-    end
+    a = doc.root.xpath('//*').select {|x| x.name == 'input' or x.name == 'select'}
     a.reject! do |x|
       x.attributes[:type] == 'hidden' or x.attributes[:style] =~ /display:none/
     end
@@ -160,3 +202,4 @@ EOF
 
 
 end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: formscraper_helper
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - James Robertson
@@ -35,7 +35,7 @@ cert_chain:
   hBw+4Vg30COBUGSGYs46Cy3vhis61poJJeWm/pLTMOH4lcl/Jz5fR//QP9ovEu3k
   3v0q89HVKLBtQzj+Dii/vHeI
   -----END CERTIFICATE-----
-date: 2022-05-25 00:00:00.000000000 Z
+date: 2022-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ferrum
@@ -106,5 +106,5 @@ requirements: []
 rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
-summary: Attempts to scrape the inputs required to complate a 1 page online form.
+summary: Attempts to scrape the inputs required to complete a 1 page online form.
 test_files: []