formscraper_helper 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb75f7060b87ea5800e4aa4ddc1b8b9cc54e5f77b7b4e7037031f3f4a714ba36
-  data.tar.gz: 39a21217d1c7f39f3e6e29cac3671481117a187aa49d1ea76c2802a608667608
+  metadata.gz: 7fc2eba27c47cbfae52b020e76bf1771953bf4944ae5da17216bd513eff74067
+  data.tar.gz: 5200c30d48828c8d29dd3794e26d3779237ddc9da8da5afda854bc15968c33c1
 SHA512:
-  metadata.gz: 489e4acefdc092393f0530656e894b2c5e896e530e9ed76a73ff751be1591777c23212b28bc0d01b429e2d79333d0a24373729b1d998c4754d959b79d47ed990
-  data.tar.gz: 3dbdb3bb6abf2579d58621babf7dbd8b1f0320542503bbf3f858122f5d7e3bd8d17f0876ea1e9fbdc32efc3d72f11d35ee429e33ac1489dc2d57fb5c9f879a4a
+  metadata.gz: 40d8f123a3000ff0072ba5d3bf71ccb0b983363c6298281edc94326af842c81688c67f25624662e5c02d4383b602392be9379ada75136a4205decebe1d843619
+  data.tar.gz: 516f582b35e1da445631d7f8f1daf1a22a86e62b4c5c4b4861d71f16a5918222701bb935d0e65547b1c4d12baf3a170993b761a0bc071ba85f78086893134668

data/lib/formscraper_helper.rb

@@ -5,6 +5,7 @@
 require 'ferrumwizard'
 require 'nokorexi'
 require 'clipboard'
+require 'fdg22'
 
 
 class FormScraperHelper
@@ -27,28 +28,44 @@ class FormScraperHelper
     doc = Nokorexi.new(body).to_doc
 
     #a = doc.root.xpath('//input|//select')
-    a = doc.root.xpath('//*').select {|x| x.name == 'input' or x.name == 'select'}
+    a = doc.root.xpath('//*').select do |x|
+      x.name == 'input' or x.name == 'select' or \
+          (x.name == 'button' and x.attributes[:type] == 'submit')
+    end
     a.reject! do |x|
       x.attributes[:type] == 'hidden' or x.attributes[:style] =~ /display:none/
     end
 
-    @h = a.map do |x|
+    a2 = a.map do |x|
 
       key = x.attributes[:name]
-      type = x.name
+      name = x.name
 
       h = {}
-      h[:type] = x.attributes[:type] || type
-      h[:xpath] = "//%s[@name=\"%s\"]" % [type, key]
+      h[:type] = x.attributes[:type] || name
+
+      if key then
+        h[:xpath] = "//%s[@name=\"%s\"]" % [name, key]
+      else
+        h[:xpath] = "//%s[@type=\"%s\"]" % [name, h[:type]]
+      end
+
       h[:title] = x.attributes[:title]
 
-      if type == 'select' then
+      if name == 'select' then
         h[:options] = x.xpath('option').map {|x| x.text.to_s}
       end
 
-      [key, h]
+      [key || h[:type], h]
 
-    end.to_h
+    end
+
+    # ensure submit appears at the end
+    submit = a2.assoc 'submit'
+    a2.delete submit
+    a2 << submit
+
+    @h = a2.to_h
 
   end
 
@@ -63,16 +80,13 @@ require 'yaml'
 require 'ferrum'
 require 'nokorexi'
 
-browser = Ferrum::Browser.new  headless: false
 url = '#{@url}'
-browser.goto(url)
-sleep 2
-
+browser = FerrumWizard.new(url,  headless: false)
 doc = Nokorexi.new(browser.body).to_doc
 
 # load the YAML document containing the inputs
 #filepath = ''
-filepath = '/tmp/tmp.yaml'
+filepath = '/tmp/data.yaml'
 h = YAML.load(File.read(filepath))
 EOF
 
@@ -106,7 +120,16 @@ EOF
         s += "sleep 0.5\n\n"
 
       elsif h[:type] == 'checkbox'
-        s += "r.focus.click\n\n"
+
+        s += "r.focus.click\n"
+        s += "sleep 0.5\n\n"
+
+      elsif h[:type] == 'submit'
+
+        s += "r.focus.click\n"
+        s += "sleep 4\n"
+        s += "browser.save_cookies('/tmp/cookies.yaml')\n"
+
       end
 
     end
@@ -199,3 +222,32 @@ EOF
 
 end
 
+class FormDataTool
+
+  def initialize(fd: nil)
+
+    @fd = fd
+
+
+  end
+
+  def regen(yml='/tmp/data.yaml')
+
+    s = File.read(yml)
+    h = YAML.load(s)
+
+    h2 = h.map do |key, value|
+      v = @fd.lookup key
+      [key, (v || value)]
+    end.to_h
+
+    h2.each do |key, value|
+      puts 'scanning key: ' + key.inspect
+      s.sub!(/#{key}: [^\n]+/, "%s: '%s'" % [key, value])
+    end
+
+    return s
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: formscraper_helper
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - James Robertson
@@ -35,7 +35,7 @@ cert_chain:
   hBw+4Vg30COBUGSGYs46Cy3vhis61poJJeWm/pLTMOH4lcl/Jz5fR//QP9ovEu3k
   3v0q89HVKLBtQzj+Dii/vHeI
   -----END CERTIFICATE-----
-date: 2022-05-28 00:00:00.000000000 Z
+date: 2022-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ferrumwizard
@@ -46,7 +46,7 @@ dependencies:
         version: '0.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.3.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -56,7 +56,7 @@ dependencies:
         version: '0.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.3.3
 - !ruby/object:Gem::Dependency
   name: nokorexi
   requirement: !ruby/object:Gem::Requirement
@@ -77,6 +77,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.0
+- !ruby/object:Gem::Dependency
+  name: fdg22
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 description: 
 email: digital.robertson@gmail.com
 executables: []