formscraper_helper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5b4f9be47b4c46d2e2823161ac508b4748afab4f2f6a03e2617bb91ae6ababe9
+  data.tar.gz: bea40e151206763162b372bb2163010f65ce021fbd795b82757780eba32ccde2
+SHA512:
+  metadata.gz: 8c7809564deb15b0b6a1ce109a676ceed9f77211dcf30fe14894a638ad75609421de74cc6863064d7c4fdb428b99fdffbb9ae777598bd3c32c99e0ca542f4860
+  data.tar.gz: a61eaf497175a3bb4aac7eed0e3e54038640697a90484c05adc11fa8342f1c668f94255488d1a6a6e932c8a42e0143b821af1bd98f4256605e86da92a9c4f218

data/lib/formscraper_helper.rb

@@ -0,0 +1,162 @@
+#!/usr/bin/env ruby
+
+# file: formscraper_helper.rb
+
+require 'ferrum'
+require 'nokorexi'
+
+
+class FormScraperHelper
+
+  attr_reader :browser
+
+  def initialize(url, headless: false, debug: false)
+
+    @url, @debug = url, debug
+    @browser = Ferrum::Browser.new  headless: headless
+    @browser.goto(url)
+    sleep 2
+    scrape()
+
+  end
+
+  def to_h()
+    @h
+  end
+
+  def to_code()
+
+s=<<EOF
+require 'yaml'
+require 'ferrum'
+require 'nokorexi'
+
+browser = Ferrum::Browser.new  headless: false
+url = '#{@url}'
+browser.goto(url)
+sleep 2
+
+doc = Nokorexi.new(browser.body).to_doc
+
+# load the YAML document containing the inputs
+#filepath = ''
+h = YAML.load(File.read(filepath))
+EOF
+
+    @h.each do |key, h|
+
+      puts 'key: ' + key.inspect if @debug
+
+      s += "r = browser.at_xpath('#{h[:xpath]}')\n"
+
+      if h[:type] == 'text' or h[:type] == 'password' then
+
+        var1 = if h[:title].length > 1 then
+          h[:title].downcase.gsub(/ +/,'_')
+        else 
+          key.downcase
+        end
+        s += var1 + " = h['#{var1}']\n"
+        s += "r.focus.type #{var1}\n\n"
+
+      elsif h[:type] == 'select'
+
+        var1 = if h[:title].length > 1 then
+          h[:title].downcase.gsub(/ +/,'_').gsub(/\W/,'')
+        else 
+          key.downcase
+        end
+
+        s += "# options: #{h[:options].join(', ')}\n"
+        s += "#{var1} = h['#{var1}']\n"
+        s += 'r = titles.grep /#{' + var1 + '}/i' + "\n"
+        s += "n = titles.index(r.first) + 1\n"
+        s += "r.focus\n"
+        s += "n.times { r.type(:down); sleep 1}\n"
+        s += "r.click\n\n"
+
+      elsif h[:type] == 'checkbox'
+        s += "r.focus.click\n\n"
+      end
+
+    end
+
+    return s
+
+  end
+
+  # creates a YAML document for the inputs
+  #
+  def to_yaml()
+
+    s = '---' + "\n"
+
+    @h.each do |key, h|
+
+      puts 'key: ' + key.inspect if @debug
+
+      if h[:type] == 'text' or h[:type] == 'password' then
+
+        var1 = if h[:title].length > 1 then
+          h[:title].downcase.gsub(/ +/,'_')
+        else 
+          key.downcase
+        end
+        s += var1 + ": xxx\n"
+
+      elsif h[:type] == 'select'
+
+        var1 = if h[:title].length > 1 then
+          h[:title].downcase.gsub(/ +/,'_').gsub(/\W/,'')
+        else 
+          key.downcase
+        end
+
+        s += "#{var1}: xxx\n"
+
+      elsif h[:type] == 'checkbox'
+
+      end
+
+    end
+
+    return s
+
+  end
+
+  private
+
+  def scrape()  
+
+    doc = Nokorexi.new(@browser.body).to_doc
+
+    #a = doc.root.xpath('//input|//select')
+    a = doc.root.xpath('//*').select do |x|
+      x.name == 'input' or x.name == 'select'
+    end
+    a.reject! do |x|
+      x.attributes[:type] == 'hidden' or x.attributes[:style] =~ /display:none/
+    end
+
+    @h = a.map do |x|
+
+      key = x.attributes[:name]
+      type = x.name
+
+      h = {}
+      h[:type] = x.attributes[:type] || type
+      h[:xpath] = "//%s[@name=\"%s\"]" % [type, key]
+      h[:title] = x.attributes[:title]
+
+      if type == 'select' then
+        h[:options] = x.xpath('option').map {|x| x.text.to_s}
+      end
+
+      [key, h]
+
+    end.to_h
+
+  end
+
+
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: formscraper_helper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- James Robertson
+autorequire: 
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIEXjCCAsagAwIBAgIBATANBgkqhkiG9w0BAQsFADAsMSowKAYDVQQDDCFnZW1t
+  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMjIwNTI1MjA0NzIxWhcN
+  MjMwNTI1MjA0NzIxWjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
+  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCxEJVm
+  2J0b5OWsG7dGVaATl+OOZOM3vvtID8nZdmkI750Z/5dsKrUDXRID1/tN7OIXsD4C
+  kf/gQTu1ZhDs0d2ZAFF++HhGBHsRpWbJFodCUXlxlSrsOouwsb9QU9wwrjSf2ROh
+  CewjBtBu5elGVxg88eCpHm5WJdTt1niiMCnl8ci2QzUkjqyoD7mujUTnJm0vbR2G
+  cAoU6A6/xraZb5HyVj+S3iU3tCcvZ7GsyfG5U60XLlee073tEbrhA+1Veu6jD2q+
+  445lGG247SYCEtWmu+S1ia7xWoTtmoHqoc0GsThK8FmCILTO0VouQwI1Em0erLM0
+  UWeUxzOIL3HiaxElI39RZwAo0AjsQs9btmIWnE6Qa0UlCc4aIZY4FPPeeOg58LXD
+  P3U1zu2dm9BFzJA9fanaKPIFWu9JaLi0h6VfRQ9HxRF0HUxyJz2vc91xhgg4cVv2
+  wIE2htNKt3KJrRoz4FF7abK371dYheDVPtgPrTZDRx3ftlIu5gscIwpCohsCAwEA
+  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUYBF7Fed1
+  r+l7945CzwRkfTiERfIwJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
+  c29uLmV1MCYGA1UdEgQfMB2BG2dlbW1hc3RlckBqYW1lc3JvYmVydHNvbi5ldTAN
+  BgkqhkiG9w0BAQsFAAOCAYEARmjroNgUuAeDY25EA0uXOWArJdBq+0Z95PHZweIa
+  Et38VT81ul/VnoEk7ehvRZNUU42XjhJkRXfnfSJdViEZYfuXFmr1GzN+Ib8Og1O/
+  3uC8eiDax+7RpFOeqIjAZ7/6lHrW6/h0oQ+66yLAc7kq4SbPZJETHkAj5JFPJijm
+  YanatYhSZ2P0/k42k38PLHBn8w8YBq7kcnntMyh/DAq7cr1G/5fLfZKLx4+Aim6S
+  wp+1pU+SnrUdzobFQ3Tq1N76CJp27iN3XpNsu5wlSUQDKbDT2hWSPZKO9XsV9/ch
+  rlJMGcoOwGdXlgipiQcc71tXQTxPgxZVw8nYlvHtoBexEupPxcPK54BS4LI88KFr
+  UTqj+Ktsfri/arY458I/Th+KLrzwetk+Z8xNOa1Pw8pTXyynbiVEPfhe80TYH/Zg
+  hBw+4Vg30COBUGSGYs46Cy3vhis61poJJeWm/pLTMOH4lcl/Jz5fR//QP9ovEu3k
+  3v0q89HVKLBtQzj+Dii/vHeI
+  -----END CERTIFICATE-----
+date: 2022-05-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ferrum
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: nokorexi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+description: 
+email: digital.robertson@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/formscraper_helper.rb
+homepage: https://github.com/jrobertson/formscraper_helper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.22
+signing_key: 
+specification_version: 4
+summary: Attempts to scrape the inputs required to complate a 1 page online form.
+test_files: []