scrap_kit 0.1.3 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd2924b0ba55e1d73238b17b4bf15b6fb36a8df61910e8af0a5876ed44ada36a
-  data.tar.gz: 897da113e002b93980a735beb87138f73b39cf2f3961d267c99bba291d6553b6
+  metadata.gz: 9125aec53d6a517aa7679bc67df8c6ffd366d77afea643fae757ab7862767c0d
+  data.tar.gz: bbfa2cae0560461e7fb1f5af1cb672a0b5c11686e75696b24578861e0d14aa1c
 SHA512:
-  metadata.gz: fb70c2d388c38e8f1b7a42f31ab661d8e277fffbba6750e774d89c34d7ef6ac277eaefe6b17a83924cc8f41e9deb61c280839f673388a7b55310d024b554df84
-  data.tar.gz: 9dfe5f761476abb11ac5e19e905f88694aade27d52e2cea7bbf92dd3cece6dda9cd62a41f0bdff25509dbad321435c907d005e276319b2e9dce05d3ab9b7daf1
+  metadata.gz: f2f8e6b7cb709ec2db86696adf27ec64e9c5faf2c339248ef34cb31764155d11de7904e1e881b7a21ba933469ebbccee998db35748030544fd699cb41a2cd7e1
+  data.tar.gz: 79fcdcf90aae48eefdf83ed6e285114221047f007a15e7e5d3f5b357091b37f7f51e0778c6ad03f03caa0fa291666831f02a1459d426e522aef1f6c02b4b509d

data/.github/workflows/run_tests.yml

@@ -0,0 +1,20 @@
+name: Run tests
+
+on: [push]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v1
+      - name: Setup Ruby
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: '2.7.1'
+      - name: Build and run tests
+        run: |
+          gem install bundler
+          bundle update --conservative --jobs 4 --retry 3
+          bundle exec rake spec

data/CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## [0.1.8] 2020-08-29
+
+### Changed/Added
+- Add new ways to declare selectors
+- Add steps
+
+## [0.1.7] 2020-08-28
+
+### Changed/Added
+- Match selector condition by regexp or exact value
+
+## [0.1.6] 2020-08-28
+
+### Changed/Added
+- Update activesupport requirement from = 6.0.2.1 to ~> 6.0
+- Fix bug when matching selector condition
+
+## [0.1.5] 2020-08-08
+
+### Changed/Added
+- Add `webdrivers` as dependency
+
+## [0.1.4] 2020-07-12
+
+### Changed/Added
+- Add support for `<input />` elements
+
+## [0.1.3] 2020-06-18
+
+### Changed/Added
+- Moved development dependencies as dependencies
+
 ## [0.1.2] 2020-06-18
 
 ### Changed/Added
@@ -9,6 +41,7 @@
 
 ### Changed/Added
 - Update activesupport requirement from = 6.0.2.1 to = 6.0.3.1
+- Update rake requirement from ~> 10.0 to ~> 13.0
 
 ## [0.1.0] 2020-02-09
 

data/Gemfile.lock

@@ -1,25 +1,29 @@
 PATH
   remote: .
   specs:
-    scrap_kit (0.1.3)
-      activesupport (= 6.0.3.1)
+    scrap_kit (0.1.8)
+      activesupport (~> 6.0)
       watir (~> 6.16.5)
+      webdrivers (~> 4.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.1)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
       zeitwerk (~> 2.2, >= 2.2.2)
     childprocess (3.0.0)
-    concurrent-ruby (1.1.6)
+    concurrent-ruby (1.1.7)
     diff-lcs (1.3)
-    i18n (1.8.3)
+    i18n (1.8.5)
       concurrent-ruby (~> 1.0)
+    mini_portile2 (2.4.0)
     minitest (5.14.1)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
     rake (13.0.1)
     regexp_parser (1.7.1)
     rspec (3.9.0)
@@ -45,7 +49,11 @@ GEM
     watir (6.16.5)
       regexp_parser (~> 1.2)
       selenium-webdriver (~> 3.6)
-    zeitwerk (2.3.0)
+    webdrivers (4.4.1)
+      nokogiri (~> 1.6)
+      rubyzip (>= 1.3.0)
+      selenium-webdriver (>= 3.0, < 4.0)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby

data/README.md

@@ -81,6 +81,66 @@ output = recipe.run
 #=> {:posts=>[{:title=>"APIs de Internacionalización en JavaScript"}, {:title=>"Ejecutando comandos desde Ruby"}, {:title=>"Usando Higher-Order Components"}]}
 ```
 
+### Working with selectors
+
+Each attribute can be mapped to a selector, which can be any of the following types:
+
+* A string, which represents a CSS selector.
+
+```ruby
+".subnav__inner .ember-view:nth-child(1) > .status-summary__description"
+```
+
+* A hash, which can have any of the following options:
+  * `xpath: [String]`
+  * `css: [String]`
+  * `index: [Integer]`
+  * `tag_name: [String]`
+  * `text: [String]`
+
+```ruby
+{ text: "View Archive" }
+```
+
+* An array, which represents a path of selectors, where its last item must be a hash that matches a selector with an expected value.
+
+```ruby
+[".up-time-chart", { ".region-header .u-margin-Tm": "REGION" }]
+```
+
+Use any of them as it suits you best.
+
+### Writing steps
+
+Recipes can have a `steps` entry. This entry defines previous actions the scraper have to follow before extract the attributes. The following steps are supported:
+
+* **`goto`**: It instructs the scraper to go to a link inside the current page. Its value can be a hash or array selector, or a URL:
+
+```ruby
+{
+  goto: { text: "View Archive" }
+}
+```
+
+* **`click`**: It instructs the scraper to click on an element inside the current page. Its value can be a hash or array selector:
+
+```ruby
+{
+  click: { css: "[type=submit]" }
+}
+```
+
+* **`fill_form`**: It instructs the scraper to fill a form or any form field inside the current page. Its value is a hash where the keys are either a input's name or a CSS selector, and the values are the values to be entered into those fields:
+
+```ruby
+{
+  fill_form: {
+    gem_name: "ScrapKit",
+    author: "hpneo",
+  }
+}
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/scrap_kit/recipe.rb

@@ -1,8 +1,23 @@
 require "active_support/core_ext/hash"
+require "webdrivers/chromedriver"
 require "watir"
 
 module ScrapKit
   class Recipe
+    class << self
+      def load(source)
+        input = if source.is_a?(Hash)
+          source
+        elsif source.is_a?(IO)
+          JSON.parse(source.read)
+        else
+          JSON.parse(File.read(source))
+        end
+
+        new(input.deep_symbolize_keys)
+      end
+    end
+
     def initialize(url: nil, steps: [], attributes: {})
       @url = url
       @steps = steps
@@ -12,76 +27,169 @@ module ScrapKit
     def run
       output = {}
 
-      browser = Watir::Browser.new(:chrome, headless: true)
-      browser.goto @url
+      @browser = Watir::Browser.new(:chrome, headless: true)
+      @browser.goto @url
 
       @steps.each do |step|
-        run_step(browser, step)
+        run_step(step)
       end
 
       @attributes.each do |attribute_name, selector|
-        output[attribute_name] = extract_attribute(browser, selector)
+        output[attribute_name] = extract_attribute(@browser, selector)
       end
 
-      browser.close
-      browser = nil
+      @browser.close
+      @browser = nil
 
       output
     end
 
-    def run_step(browser, step)
+    def run_step(step)
+      return goto(step[:goto]) if step[:goto]
+      return click(step[:click]) if step[:click]
+      return fill_form(step[:fill_form]) if step[:fill_form]
+
+      nil
+    end
+
+    def find_element_by_name_or_selector(browser_or_element, name_or_selector)
+      element = browser_or_element.element(name: name_or_selector.to_s)
+      return element if element.exists?
+
+      element = browser_or_element.element(css: name_or_selector.to_s)
+      return element if element.exists?
+
+      nil
     end
 
     def elements_from_selector(browser_or_element, selector)
       if selector.is_a?(String)
         browser_or_element.elements(css: selector)
+      elsif selector.is_a?(Hash)
+        browser_or_element.elements(selector)
       elsif selector.is_a?(Array)
         *remainder, condition = selector
+        condition_key, condition_value = condition.first
         elements = browser_or_element
 
-        remainder.each do |item|
-          elements = elements.elements(css: item)
+        if remainder.empty?
+          elements = elements.elements(css: condition_key.to_s)
+        else
+          remainder.each do |item|
+            elements = elements.elements(css: item)
+          end
         end
 
         elements.filter do |element|
-          condition_key = condition.keys[0].to_s
-          condition_value = condition.values[0]
-          found_element = element.element(css: condition_key)
-          found_element&.text_content&.match(condition_value)
+          found_element = element.element(css: condition_key.to_s)
+          extracted_value = extract_value_from_element(found_element)
+          extracted_value.match(condition_value) || extracted_value == condition_value
+        end
+      end
+    end
+
+    def extract_value_from_element(element)
+      if element&.respond_to?(:tag_name)
+        if element.tag_name.downcase == "input"
+          return element.attribute_value(:value)
         end
       end
+
+      element&.text_content
     end
 
-    def extract_attribute(browser_or_element, selector_or_hash)
-      if selector_or_hash.is_a?(String)
-        browser_or_element.element(css: selector_or_hash)&.text_content
-      elsif selector_or_hash.is_a?(Hash)
-        selector = selector_or_hash[:selector]
-        selector_for_children_attributes = selector_or_hash[:children_attributes]
+    def extract_attribute(browser_or_element, selector_or_object)
+      if selector_or_object.is_a?(String)
+        extract_value_from_element(browser_or_element.element(css: selector_or_object))
+      elsif selector_or_object.is_a?(Array)
+        found_elements = elements_from_selector(browser_or_element, selector_or_object)
+
+        if found_elements.size === 1
+          extract_value_from_element(found_elements.first)
+        else
+          found_elements.map do |element|
+            extract_value_from_element(element)
+          end
+        end
+      elsif selector_or_object.is_a?(Hash)
+        if selector_or_object[:selector] && selector_or_object[:children_attributes]
+          selector = selector_or_object[:selector]
+          selector_for_children_attributes = selector_or_object[:children_attributes]
+
+          elements_from_selector(browser_or_element, selector).map do |element|
+            output = {}
 
-        elements_from_selector(browser_or_element, selector).map do |element|
-          output = {}
+            selector_for_children_attributes.each do |child_attribute_name, child_selector|
+              output[child_attribute_name] = extract_attribute(element, child_selector)
+            end
 
-          selector_for_children_attributes.each do |child_attribute_name, child_selector|
-            output[child_attribute_name] = extract_attribute(element, child_selector)
+            output
+          end
+        else
+          found_elements = elements_from_selector(browser_or_element, selector_or_object)
+
+          if found_elements.size === 1
+            extract_value_from_element(found_elements.first)
+          else
+            found_elements.map do |element|
+              extract_value_from_element(element)
+            end
           end
+        end
+      end
+    end
+
+    private
 
-          output
+    def goto(link_or_selector)
+      if link_or_selector.is_a?(String)
+        @browser.goto(link_or_selector)
+      elsif link_or_selector.is_a?(Array) || link_or_selector.is_a?(Hash)
+        if found_element = elements_from_selector(@browser, link_or_selector).first
+          found_element.click
         end
       end
+
+      sleep 0.5
+      @browser.wait_until do
+        @browser.ready_state == "complete"
+      end
+    rescue
+      nil
     end
 
-    class << self
-      def load(source)
-        input = if source.is_a?(Hash)
-          source
-        elsif source.is_a?(IO)
-          JSON.parse(source.read)
-        else
-          JSON.parse(File.read(source))
+    def click(selector)
+      if selector.is_a?(Array) || selector.is_a?(Hash)
+        if found_element = elements_from_selector(@browser, selector).first
+          found_element.click
         end
+      end
 
-        new(input.deep_symbolize_keys)
+      sleep 1
+      @browser.wait_until do
+        @browser.ready_state == "complete"
+      end
+
+    rescue
+      nil
+    end
+
+    def fill_form(form_data)
+      form_data.each do |name, value|
+        if element = find_element_by_name_or_selector(@browser.body, name.to_s)
+          element = element.to_subtype
+
+          if element.respond_to?(:set)
+            element.set(value)
+          elsif element.respond_to?(:select)
+            element.select(value)
+          end
+        end
+      end
+
+      sleep 0.25
+      @browser.wait_until do
+        @browser.ready_state == "complete"
       end
     end
   end

data/lib/scrap_kit/version.rb

@@ -1,3 +1,3 @@
 module ScrapKit
-  VERSION = "0.1.3"
+  VERSION = "0.1.8"
 end

data/scrap_kit.gemspec

@@ -32,5 +32,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_dependency "watir", "~> 6.16.5"
-  spec.add_dependency "activesupport", "6.0.3.1"
+  spec.add_dependency "webdrivers", "~> 4.0"
+  spec.add_dependency "activesupport", "~> 6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scrap_kit
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.8
 platform: ruby
 authors:
 - Gustavo Leon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-06-19 00:00:00.000000000 Z
+date: 2020-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,20 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 6.16.5
+- !ruby/object:Gem::Dependency
+  name: webdrivers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 6.0.3.1
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 6.0.3.1
+        version: '6.0'
 description: Run JSON-based recipes to scrap web sites.
 email:
 - hpneo@hotmail.com
@@ -87,7 +101,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/publish_gem.yml"
+- ".github/workflows/run_tests.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"

data/.github/workflows/publish_gem.yml

@@ -1,17 +0,0 @@
-name: Publish gem
-
-on:
-  push:
-    tags:
-      - '*'
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Publish gem
-        uses: dawidd6/action-publish-gem@v1.0.0
-        with:
-          api_key: ${{secrets.RUBYGEMS_API_KEY}}