formscraper_helper 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 402c3ab5b633cce3e4852deba197a388a30d267e81595829cbc05deb6509f07e
-  data.tar.gz: c3867d152abfa6d910efd7f24b119c413c8ca8368229290583704bba1018108d
+  metadata.gz: cb75f7060b87ea5800e4aa4ddc1b8b9cc54e5f77b7b4e7037031f3f4a714ba36
+  data.tar.gz: 39a21217d1c7f39f3e6e29cac3671481117a187aa49d1ea76c2802a608667608
 SHA512:
-  metadata.gz: a3c5c2217823d8734d7e273a07c166012191f0196a3c039cbb7bb55a0ee2ab3593295b9e985c049ebcfdddaa768696091fa49352cb8af3c7782ac56a25d86dd2
-  data.tar.gz: 15b49ce36dd5517099e04436153f3e63c1f9138a383ddb22033e4896eb9b0203c32bcca28ec18ab1883bc452a175182e21700cbca4cd812cdb618ad0b571abcd
+  metadata.gz: 489e4acefdc092393f0530656e894b2c5e896e530e9ed76a73ff751be1591777c23212b28bc0d01b429e2d79333d0a24373729b1d998c4754d959b79d47ed990
+  data.tar.gz: 3dbdb3bb6abf2579d58621babf7dbd8b1f0320542503bbf3f858122f5d7e3bd8d17f0876ea1e9fbdc32efc3d72f11d35ee429e33ac1489dc2d57fb5c9f879a4a

data/lib/formscraper_helper.rb

@@ -2,7 +2,7 @@
 
 # file: formscraper_helper.rb
 
-require 'ferrum'
+require 'ferrumwizard'
 require 'nokorexi'
 require 'clipboard'
 
@@ -13,14 +13,42 @@ class FormScraperHelper
 
   # note: fd corresponds to FakeDataGenerator22 which is optional
   #
-  def initialize(url, headless: false, clipb: true, fd: nil, debug: false)
+  def initialize(url=nil, browser: nil, headless: false, clipb: true,
+                 fd: nil, debug: false)
 
     @url, @clipb, @fd, @debug = url, clipb, fd, debug
-    @browser = Ferrum::Browser.new  headless: headless
-    @browser.goto(url)
 
-    sleep 2
-    scrape()
+    @browser = browser ? browser : FerrumWizard.new(url,  headless: headless)
+
+  end
+
+  def scrape(body=@browser.body)
+    puts 'body: '  + body.inspect if @debug
+    doc = Nokorexi.new(body).to_doc
+
+    #a = doc.root.xpath('//input|//select')
+    a = doc.root.xpath('//*').select {|x| x.name == 'input' or x.name == 'select'}
+    a.reject! do |x|
+      x.attributes[:type] == 'hidden' or x.attributes[:style] =~ /display:none/
+    end
+
+    @h = a.map do |x|
+
+      key = x.attributes[:name]
+      type = x.name
+
+      h = {}
+      h[:type] = x.attributes[:type] || type
+      h[:xpath] = "//%s[@name=\"%s\"]" % [type, key]
+      h[:title] = x.attributes[:title]
+
+      if type == 'select' then
+        h[:options] = x.xpath('option').map {|x| x.text.to_s}
+      end
+
+      [key, h]
+
+    end.to_h
 
   end
 
@@ -113,7 +141,7 @@ EOF
 
           found = @fd.lookup var1
           val = found.is_a?(String) ? found : 'xxx'
-          s += var1 + ": #{val}\n"
+          s += var1 + ": '#{val}'\n"
         else
           s += var1 + ": xxx\n"
         end
@@ -125,7 +153,7 @@ EOF
         s += s2
         s += "# options: #{h[:options].join(', ')}\n"
         val = h[:options][1..-1].sample
-        s += "#{var1}: #{val}\n"
+        s += "#{var1}: '#{val}'\n"
 
       elsif h[:type] == 'checkbox'
 
@@ -143,7 +171,6 @@ EOF
   private
 
   # returns var1 using arguments rawtitle or key
-  # note: argument s is passed by reference
   #
   def format_var1(rawtitle, key)
 
@@ -170,36 +197,5 @@ EOF
 
   end
 
-  def scrape()
-
-    doc = Nokorexi.new(@browser.body).to_doc
-
-    #a = doc.root.xpath('//input|//select')
-    a = doc.root.xpath('//*').select {|x| x.name == 'input' or x.name == 'select'}
-    a.reject! do |x|
-      x.attributes[:type] == 'hidden' or x.attributes[:style] =~ /display:none/
-    end
-
-    @h = a.map do |x|
-
-      key = x.attributes[:name]
-      type = x.name
-
-      h = {}
-      h[:type] = x.attributes[:type] || type
-      h[:xpath] = "//%s[@name=\"%s\"]" % [type, key]
-      h[:title] = x.attributes[:title]
-
-      if type == 'select' then
-        h[:options] = x.xpath('option').map {|x| x.text.to_s}
-      end
-
-      [key, h]
-
-    end.to_h
-
-  end
-
-
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: formscraper_helper
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - James Robertson
@@ -38,25 +38,25 @@ cert_chain:
 date: 2022-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: ferrum
+  name: ferrumwizard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: 0.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: 0.3.2
 - !ruby/object:Gem::Dependency
   name: nokorexi
   requirement: !ruby/object:Gem::Requirement