scrap_kit 0.1.1 → 0.1.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/run_tests.yml +20 -0
  3. data/.gitignore +1 -0
  4. data/CHANGELOG.md +30 -0
  5. data/Gemfile.lock +21 -13
  6. data/lib/scrap_kit/recipe.rb +33 -2
  7. data/lib/scrap_kit/version.rb +1 -1
  8. data/scrap_kit.gemspec +3 -2
  9. metadata +27 -13
  10. data/.github/workflows/publish_gem.yml +0 -17
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d30e6aebb388e92e31b33c920619fefec3a1a5ac2e6c44d1dd14c7f865cdc816
4
- data.tar.gz: 17ad0fdd2fcff8f8f11f6000cfa4a34c90243d14012b919cbc243dddde6b7495
3
+ metadata.gz: ffb35087e7e374b2e5fa15d2f9c2d41374d0242e4f2b5a986db195929a530dc6
4
+ data.tar.gz: 904246fc2061a3ec1686a70d6755fd81f13b215b3dce7816177f0774af666b89
5
5
  SHA512:
6
- metadata.gz: 9c427bc98598612c53cb75e355d101c37f998642d75fc4daf11de4cacdc337b6099a588b884c43813d268d38fcffd78178114d0a07fe0589ec4f5e670a06d81a
7
- data.tar.gz: 7b825d7e22ac8bcbb8b1cd3131f9c4a893f86ae6ae8874bf6bfc6ab1bb0499d33da1906632a8c811cb726c60e6d8c5f3e76035aecb93745632759fe19531667c
6
+ metadata.gz: c55754d5f772f7bfd9c9bd4fdda802f77a898f6c86f0956d9f94d9c81e7151fdaed07ec45f56d0b7010eb56dc7a1a94e58079bcb1da4753586766eefd4bd4954
7
+ data.tar.gz: b20fc3e3b8852361e707d4597ec510d44cb3d76a9260f6d719e2547adf5a68abafeb7423f02c47ba853a4f92ee43df3ccfc88bb60dd21913249ef8b62613a53c
data/.github/workflows/run_tests.yml ADDED
@@ -0,0 +1,20 @@
1
+ name: Run tests
2
+
3
+ on: [push]
4
+
5
+ jobs:
6
+ test:
7
+
8
+ runs-on: ubuntu-latest
9
+
10
+ steps:
11
+ - uses: actions/checkout@v1
12
+ - name: Setup Ruby
13
+ uses: actions/setup-ruby@v1
14
+ with:
15
+ ruby-version: '2.7.1'
16
+ - name: Build and run tests
17
+ run: |
18
+ gem install bundler
19
+ bundle update --conservative --jobs 4 --retry 3
20
+ bundle exec rake spec
data/.gitignore CHANGED
@@ -9,4 +9,5 @@
9
9
 
10
10
  # rspec failure tracking
11
11
  .rspec_status
12
+ *.gem
12
13
  .DS_Store
data/CHANGELOG.md CHANGED
@@ -1,5 +1,35 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.1.6] 2020-08-28
4
+ - Update activesupport requirement from = 6.0.2.1 to ~> 6.0
5
+ - Fix bug when matching selector condition
6
+
7
+ ## [0.1.5] 2020-08-08
8
+
9
+ ### Changed/Added
10
+ - Add `webdrivers` as dependency
11
+
12
+ ## [0.1.4] 2020-07-12
13
+
14
+ ### Changed/Added
15
+ - Add support for `<input />` elements
16
+
17
+ ## [0.1.3] 2020-06-18
18
+
19
+ ### Changed/Added
20
+ - Moved development dependencies as dependencies
21
+
22
+ ## [0.1.2] 2020-06-18
23
+
24
+ ### Changed/Added
25
+ - Add support for selector arrays to filter elements based on conditions
26
+
27
+ ## [0.1.1] 2020-05-18
28
+
29
+ ### Changed/Added
30
+ - Update activesupport requirement from = 6.0.2.1 to = 6.0.3.1
31
+ - Update rake requirement from ~> 10.0 to ~> 13.0
32
+
3
33
  ## [0.1.0] 2020-02-09
4
34
 
5
35
  ### Changed/Added
data/Gemfile.lock CHANGED
@@ -1,25 +1,31 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- scrap_kit (0.1.0)
4
+ scrap_kit (0.1.6)
5
+ activesupport (~> 6.0)
6
+ watir (~> 6.16.5)
7
+ webdrivers (~> 4.0)
5
8
 
6
9
  GEM
7
10
  remote: https://rubygems.org/
8
11
  specs:
9
- activesupport (6.0.2.1)
12
+ activesupport (6.0.3.2)
10
13
  concurrent-ruby (~> 1.0, >= 1.0.2)
11
14
  i18n (>= 0.7, < 2)
12
15
  minitest (~> 5.1)
13
16
  tzinfo (~> 1.1)
14
- zeitwerk (~> 2.2)
17
+ zeitwerk (~> 2.2, >= 2.2.2)
15
18
  childprocess (3.0.0)
16
- concurrent-ruby (1.1.5)
19
+ concurrent-ruby (1.1.7)
17
20
  diff-lcs (1.3)
18
- i18n (1.8.2)
21
+ i18n (1.8.5)
19
22
  concurrent-ruby (~> 1.0)
20
- minitest (5.14.0)
23
+ mini_portile2 (2.4.0)
24
+ minitest (5.14.1)
25
+ nokogiri (1.10.10)
26
+ mini_portile2 (~> 2.4.0)
21
27
  rake (13.0.1)
22
- regexp_parser (1.6.0)
28
+ regexp_parser (1.7.1)
23
29
  rspec (3.9.0)
24
30
  rspec-core (~> 3.9.0)
25
31
  rspec-expectations (~> 3.9.0)
@@ -33,28 +39,30 @@ GEM
33
39
  diff-lcs (>= 1.2.0, < 2.0)
34
40
  rspec-support (~> 3.9.0)
35
41
  rspec-support (3.9.2)
36
- rubyzip (2.2.0)
42
+ rubyzip (2.3.0)
37
43
  selenium-webdriver (3.142.7)
38
44
  childprocess (>= 0.5, < 4.0)
39
45
  rubyzip (>= 1.2.2)
40
46
  thread_safe (0.3.6)
41
- tzinfo (1.2.6)
47
+ tzinfo (1.2.7)
42
48
  thread_safe (~> 0.1)
43
49
  watir (6.16.5)
44
50
  regexp_parser (~> 1.2)
45
51
  selenium-webdriver (~> 3.6)
46
- zeitwerk (2.2.2)
52
+ webdrivers (4.4.1)
53
+ nokogiri (~> 1.6)
54
+ rubyzip (>= 1.3.0)
55
+ selenium-webdriver (>= 3.0, < 4.0)
56
+ zeitwerk (2.4.0)
47
57
 
48
58
  PLATFORMS
49
59
  ruby
50
60
 
51
61
  DEPENDENCIES
52
- activesupport (= 6.0.2.1)
53
62
  bundler (~> 2.0)
54
63
  rake (~> 13.0)
55
64
  rspec (~> 3.0)
56
65
  scrap_kit!
57
- watir (~> 6.16.5)
58
66
 
59
67
  BUNDLED WITH
60
- 2.0.2
68
+ 2.1.4
data/lib/scrap_kit/recipe.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require "active_support/core_ext/hash"
2
+ require "webdrivers/chromedriver"
2
3
  require "watir"
3
4
 
4
5
  module ScrapKit
@@ -32,14 +33,44 @@ module ScrapKit
32
33
  def run_step(browser, step)
33
34
  end
34
35
 
36
+ def elements_from_selector(browser_or_element, selector)
37
+ if selector.is_a?(String)
38
+ browser_or_element.elements(css: selector)
39
+ elsif selector.is_a?(Array)
40
+ *remainder, condition = selector
41
+ elements = browser_or_element
42
+
43
+ remainder.each do |item|
44
+ elements = elements.elements(css: item)
45
+ end
46
+
47
+ elements.filter do |element|
48
+ condition_key = condition.keys.first.to_s
49
+ condition_value = condition.values.first
50
+ found_element = element.element(css: condition_key)
51
+ extract_value_from_element(found_element) == condition_value
52
+ end
53
+ end
54
+ end
55
+
56
+ def extract_value_from_element(element)
57
+ if element&.respond_to?(:tag_name)
58
+ if element.tag_name.downcase == "input"
59
+ return element.attribute_value(:value)
60
+ end
61
+ end
62
+
63
+ element&.text_content
64
+ end
65
+
35
66
  def extract_attribute(browser_or_element, selector_or_hash)
36
67
  if selector_or_hash.is_a?(String)
37
- browser_or_element.element(css: selector_or_hash)&.text_content
68
+ extract_value_from_element(browser_or_element.element(css: selector_or_hash))
38
69
  elsif selector_or_hash.is_a?(Hash)
39
70
  selector = selector_or_hash[:selector]
40
71
  selector_for_children_attributes = selector_or_hash[:children_attributes]
41
72
 
42
- browser_or_element.elements(css: selector).map do |element|
73
+ elements_from_selector(browser_or_element, selector).map do |element|
43
74
  output = {}
44
75
 
45
76
  selector_for_children_attributes.each do |child_attribute_name, child_selector|
data/lib/scrap_kit/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module ScrapKit
2
- VERSION = "0.1.1"
2
+ VERSION = "0.1.6"
3
3
  end
data/scrap_kit.gemspec CHANGED
@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
31
31
  spec.add_development_dependency "bundler", "~> 2.0"
32
32
  spec.add_development_dependency "rake", "~> 13.0"
33
33
  spec.add_development_dependency "rspec", "~> 3.0"
34
- spec.add_development_dependency "activesupport", "6.0.2.1"
35
- spec.add_development_dependency "watir", "~> 6.16.5"
34
+ spec.add_dependency "watir", "~> 6.16.5"
35
+ spec.add_dependency "webdrivers", "~> 4.0"
36
+ spec.add_dependency "activesupport", "~> 6.0"
36
37
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: scrap_kit
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Gustavo Leon
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-05-19 00:00:00.000000000 Z
11
+ date: 2020-08-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -53,33 +53,47 @@ dependencies:
53
53
  - !ruby/object:Gem::Version
54
54
  version: '3.0'
55
55
  - !ruby/object:Gem::Dependency
56
- name: activesupport
56
+ name: watir
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - '='
59
+ - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: 6.0.2.1
62
- type: :development
61
+ version: 6.16.5
62
+ type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - '='
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: 6.0.2.1
68
+ version: 6.16.5
69
69
  - !ruby/object:Gem::Dependency
70
- name: watir
70
+ name: webdrivers
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: 6.16.5
76
- type: :development
75
+ version: '4.0'
76
+ type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: 6.16.5
82
+ version: '4.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: activesupport
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '6.0'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '6.0'
83
97
  description: Run JSON-based recipes to scrap web sites.
84
98
  email:
85
99
  - hpneo@hotmail.com
@@ -87,7 +101,7 @@ executables: []
87
101
  extensions: []
88
102
  extra_rdoc_files: []
89
103
  files:
90
- - ".github/workflows/publish_gem.yml"
104
+ - ".github/workflows/run_tests.yml"
91
105
  - ".gitignore"
92
106
  - ".rspec"
93
107
  - ".rubocop.yml"
data/.github/workflows/publish_gem.yml DELETED
@@ -1,17 +0,0 @@
1
- name: Publish gem
2
-
3
- on:
4
- push:
5
- tags:
6
- - '*'
7
-
8
- jobs:
9
- build:
10
-
11
- runs-on: ubuntu-latest
12
-
13
- steps:
14
- - name: Publish gem
15
- uses: dawidd6/action-publish-gem@v1.0.0
16
- with:
17
- api_key: ${{secrets.RUBYGEMS_API_KEY}}