formscraper_helper 0.1.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b4f9be47b4c46d2e2823161ac508b4748afab4f2f6a03e2617bb91ae6ababe9
-  data.tar.gz: bea40e151206763162b372bb2163010f65ce021fbd795b82757780eba32ccde2
+  metadata.gz: 7fc2eba27c47cbfae52b020e76bf1771953bf4944ae5da17216bd513eff74067
+  data.tar.gz: 5200c30d48828c8d29dd3794e26d3779237ddc9da8da5afda854bc15968c33c1
 SHA512:
-  metadata.gz: 8c7809564deb15b0b6a1ce109a676ceed9f77211dcf30fe14894a638ad75609421de74cc6863064d7c4fdb428b99fdffbb9ae777598bd3c32c99e0ca542f4860
-  data.tar.gz: a61eaf497175a3bb4aac7eed0e3e54038640697a90484c05adc11fa8342f1c668f94255488d1a6a6e932c8a42e0143b821af1bd98f4256605e86da92a9c4f218
+  metadata.gz: 40d8f123a3000ff0072ba5d3bf71ccb0b983363c6298281edc94326af842c81688c67f25624662e5c02d4383b602392be9379ada75136a4205decebe1d843619
+  data.tar.gz: 516f582b35e1da445631d7f8f1daf1a22a86e62b4c5c4b4861d71f16a5918222701bb935d0e65547b1c4d12baf3a170993b761a0bc071ba85f78086893134668

data/lib/formscraper_helper.rb

@@ -2,21 +2,70 @@
 
 # file: formscraper_helper.rb
 
-require 'ferrum'
+require 'ferrumwizard'
 require 'nokorexi'
+require 'clipboard'
+require 'fdg22'
 
 
 class FormScraperHelper
 
   attr_reader :browser
 
-  def initialize(url, headless: false, debug: false)
+  # note: fd corresponds to FakeDataGenerator22 which is optional
+  #
+  def initialize(url=nil, browser: nil, headless: false, clipb: true,
+                 fd: nil, debug: false)
+
+    @url, @clipb, @fd, @debug = url, clipb, fd, debug
+
+    @browser = browser ? browser : FerrumWizard.new(url,  headless: headless)
+
+  end
+
+  def scrape(body=@browser.body)
+    puts 'body: '  + body.inspect if @debug
+    doc = Nokorexi.new(body).to_doc
+
+    #a = doc.root.xpath('//input|//select')
+    a = doc.root.xpath('//*').select do |x|
+      x.name == 'input' or x.name == 'select' or \
+          (x.name == 'button' and x.attributes[:type] == 'submit')
+    end
+    a.reject! do |x|
+      x.attributes[:type] == 'hidden' or x.attributes[:style] =~ /display:none/
+    end
+
+    a2 = a.map do |x|
+
+      key = x.attributes[:name]
+      name = x.name
+
+      h = {}
+      h[:type] = x.attributes[:type] || name
+
+      if key then
+        h[:xpath] = "//%s[@name=\"%s\"]" % [name, key]
+      else
+        h[:xpath] = "//%s[@type=\"%s\"]" % [name, h[:type]]
+      end
+
+      h[:title] = x.attributes[:title]
+
+      if name == 'select' then
+        h[:options] = x.xpath('option').map {|x| x.text.to_s}
+      end
+
+      [key || h[:type], h]
 
-    @url, @debug = url, debug
-    @browser = Ferrum::Browser.new  headless: headless
-    @browser.goto(url)
-    sleep 2
-    scrape()
+    end
+
+    # ensure submit appears at the end
+    submit = a2.assoc 'submit'
+    a2.delete submit
+    a2 << submit
+
+    @h = a2.to_h
 
   end
 
@@ -31,15 +80,13 @@ require 'yaml'
 require 'ferrum'
 require 'nokorexi'
 
-browser = Ferrum::Browser.new  headless: false
 url = '#{@url}'
-browser.goto(url)
-sleep 2
-
+browser = FerrumWizard.new(url,  headless: false)
 doc = Nokorexi.new(browser.body).to_doc
 
 # load the YAML document containing the inputs
 #filepath = ''
+filepath = '/tmp/data.yaml'
 h = YAML.load(File.read(filepath))
 EOF
 
@@ -51,36 +98,45 @@ EOF
 
       if h[:type] == 'text' or h[:type] == 'password' then
 
-        var1 = if h[:title].length > 1 then
-          h[:title].downcase.gsub(/ +/,'_')
-        else 
-          key.downcase
-        end
+        var1, s2 = format_var1(h[:title], key)
+        s += s2
         s += var1 + " = h['#{var1}']\n"
-        s += "r.focus.type #{var1}\n\n"
+        s += "r.focus.type #{var1}\n"
+        s += "sleep 0.5\n\n"
 
       elsif h[:type] == 'select'
 
-        var1 = if h[:title].length > 1 then
-          h[:title].downcase.gsub(/ +/,'_').gsub(/\W/,'')
-        else 
-          key.downcase
-        end
+        var1, s2 = format_var1(h[:title], key)
+        s += s2
 
         s += "# options: #{h[:options].join(', ')}\n"
         s += "#{var1} = h['#{var1}']\n"
-        s += 'r = titles.grep /#{' + var1 + '}/i' + "\n"
-        s += "n = titles.index(r.first) + 1\n"
+        s += 'titles = %w(' + h[:options].join(' ') + ')' + "\n"
+        s += 'found = titles.grep /#{' + var1 + '}/i' + "\n"
+        s += "n = titles.index(found.first) + 1\n"
         s += "r.focus\n"
         s += "n.times { r.type(:down); sleep 1}\n"
-        s += "r.click\n\n"
+        s += "r.click\n"
+        s += "sleep 0.5\n\n"
 
       elsif h[:type] == 'checkbox'
-        s += "r.focus.click\n\n"
+
+        s += "r.focus.click\n"
+        s += "sleep 0.5\n\n"
+
+      elsif h[:type] == 'submit'
+
+        s += "r.focus.click\n"
+        s += "sleep 4\n"
+        s += "browser.save_cookies('/tmp/cookies.yaml')\n"
+
       end
 
     end
 
+    Clipboard.copy s if @clipb
+    puts 'generated code copied to clipboard'
+
     return s
 
   end
@@ -97,22 +153,30 @@ EOF
 
       if h[:type] == 'text' or h[:type] == 'password' then
 
-        var1 = if h[:title].length > 1 then
-          h[:title].downcase.gsub(/ +/,'_')
-        else 
-          key.downcase
+        var1, s2 = format_var1(h[:title], key)
+
+        s += s2
+
+        if h[:type] == 'password' then
+          @pwd ||= @fd ? @fd.password : 'xxx'
+          s += var1 + ": #{@pwd}\n"
+        elsif @fd
+
+          found = @fd.lookup var1
+          val = found.is_a?(String) ? found : 'xxx'
+          s += var1 + ": '#{val}'\n"
+        else
+          s += var1 + ": xxx\n"
         end
-        s += var1 + ": xxx\n"
 
       elsif h[:type] == 'select'
 
-        var1 = if h[:title].length > 1 then
-          h[:title].downcase.gsub(/ +/,'_').gsub(/\W/,'')
-        else 
-          key.downcase
-        end
+        var1, s2 = format_var1(h[:title], key)
 
-        s += "#{var1}: xxx\n"
+        s += s2
+        s += "# options: #{h[:options].join(', ')}\n"
+        val = h[:options][1..-1].sample
+        s += "#{var1}: '#{val}'\n"
 
       elsif h[:type] == 'checkbox'
 
@@ -120,43 +184,70 @@ EOF
 
     end
 
+    Clipboard.copy s if @clipb
+    puts 'generated YAML copied to clipboard'
+
     return s
 
   end
 
   private
 
-  def scrape()  
+  # returns var1 using arguments rawtitle or key
+  #
+  def format_var1(rawtitle, key)
+
+    var1 = if rawtitle.length > 1 then
 
-    doc = Nokorexi.new(@browser.body).to_doc
+      s = "\n# " + rawtitle + "\n"
+      title = rawtitle.scan(/[A-Z][^A-Z]+/).join(' ').gsub(/[^\w ]/,'')
+      words = title.downcase.scan(/\w+/)
 
-    #a = doc.root.xpath('//input|//select')
-    a = doc.root.xpath('//*').select do |x|
-      x.name == 'input' or x.name == 'select'
-    end
-    a.reject! do |x|
-      x.attributes[:type] == 'hidden' or x.attributes[:style] =~ /display:none/
+      if words.count > 2 then
+        words.take(5).map {|x| x[0]}.join
+      else
+        title.downcase.gsub(/ +/,'_')
+      end
+
+    else
+      newtitle = key.scan(/[A-Z][^A-Z]+/).join(' ')
+      s = "\n# " + newtitle + "\n"
+      newtitle.gsub(/[^\w ]/,'').downcase\
+          .gsub(/ +/,'_')
     end
 
-    @h = a.map do |x|
+    [var1, s]
 
-      key = x.attributes[:name]
-      type = x.name
+  end
 
-      h = {}
-      h[:type] = x.attributes[:type] || type
-      h[:xpath] = "//%s[@name=\"%s\"]" % [type, key]
-      h[:title] = x.attributes[:title]
+end
 
-      if type == 'select' then
-        h[:options] = x.xpath('option').map {|x| x.text.to_s}
-      end
+class FormDataTool
 
-      [key, h]
+  def initialize(fd: nil)
+
+    @fd = fd
 
-    end.to_h
 
   end
 
+  def regen(yml='/tmp/data.yaml')
+
+    s = File.read(yml)
+    h = YAML.load(s)
+
+    h2 = h.map do |key, value|
+      v = @fd.lookup key
+      [key, (v || value)]
+    end.to_h
+
+    h2.each do |key, value|
+      puts 'scanning key: ' + key.inspect
+      s.sub!(/#{key}: [^\n]+/, "%s: '%s'" % [key, value])
+    end
+
+    return s
+
+  end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: formscraper_helper
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.4.0
 platform: ruby
 authors:
 - James Robertson
@@ -35,28 +35,28 @@ cert_chain:
   hBw+4Vg30COBUGSGYs46Cy3vhis61poJJeWm/pLTMOH4lcl/Jz5fR//QP9ovEu3k
   3v0q89HVKLBtQzj+Dii/vHeI
   -----END CERTIFICATE-----
-date: 2022-05-25 00:00:00.000000000 Z
+date: 2022-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: ferrum
+  name: ferrumwizard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: 0.3.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: 0.3.3
 - !ruby/object:Gem::Dependency
   name: nokorexi
   requirement: !ruby/object:Gem::Requirement
@@ -77,6 +77,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.0
+- !ruby/object:Gem::Dependency
+  name: fdg22
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 description: 
 email: digital.robertson@gmail.com
 executables: []
@@ -106,5 +126,5 @@ requirements: []
 rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
-summary: Attempts to scrape the inputs required to complate a 1 page online form.
+summary: Attempts to scrape the inputs required to complete a 1 page online form.
 test_files: []