newegg_scraper_chsbr 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/Gemfile +9 -0
- data/Gemfile.lock +27 -0
- data/LICENSE.txt +21 -0
- data/README.md +43 -0
- data/Rakefile +3 -0
- data/bin/newegg_scraper +4 -0
- data/class_diagram.drawio +1 -0
- data/lib/environment.rb +11 -0
- data/lib/newegg_scraper_chsbr/cli.rb +67 -0
- data/lib/newegg_scraper_chsbr/cpu.rb +67 -0
- data/lib/newegg_scraper_chsbr/data_grabber.rb +158 -0
- data/lib/newegg_scraper_chsbr/descriptors.rb +3 -0
- data/lib/newegg_scraper_chsbr/name.rb +10 -0
- data/lib/newegg_scraper_chsbr/price.rb +12 -0
- data/lib/newegg_scraper_chsbr/scraper.rb +13 -0
- data/lib/newegg_scraper_chsbr/user.rb +20 -0
- data/lib/newegg_scraper_chsbr/version.rb +5 -0
- data/newegg_scraper_chsbr.gemspec +39 -0
- metadata +93 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: c0d338e6de4d37ec3b6e4d803a5ad5072690cd7602df34397ffc982027075c84
|
|
4
|
+
data.tar.gz: c3fd4a5570d47b5076f445777e6044b5c5b3da94f5c4a18bc8104f68eaf37efb
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: a86cd954f40f9833d453ae9cb22334bf7e0453706a0f9d62eb09725df9a6506de4d8bee6af669825bda14fb6fc3ca7b5197edda1738dc71557536374b6d7c404
|
|
7
|
+
data.tar.gz: 3a2f2fc4723c764fd0df8a4b605c06415429f000b978522a8e6ea301c3c9a819302b8d72dfc43d56ad308b465670ba2bc9e7093a3e0472bcc733f73e5f3aa264
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
newegg_scraper_chsbr (0.1.0)
|
|
5
|
+
nokogiri
|
|
6
|
+
open-uri
|
|
7
|
+
|
|
8
|
+
GEM
|
|
9
|
+
remote: https://rubygems.org/
|
|
10
|
+
specs:
|
|
11
|
+
nokogiri (1.11.3-x86_64-linux)
|
|
12
|
+
racc (~> 1.4)
|
|
13
|
+
open-uri (0.1.0)
|
|
14
|
+
racc (1.5.2)
|
|
15
|
+
rake (13.0.3)
|
|
16
|
+
|
|
17
|
+
PLATFORMS
|
|
18
|
+
x86_64-linux
|
|
19
|
+
|
|
20
|
+
DEPENDENCIES
|
|
21
|
+
newegg_scraper_chsbr!
|
|
22
|
+
nokogiri
|
|
23
|
+
open-uri
|
|
24
|
+
rake (~> 13.0)
|
|
25
|
+
|
|
26
|
+
BUNDLED WITH
|
|
27
|
+
2.2.13
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2021 Lrd134
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# NeweggScraperChsbr
|
|
2
|
+
|
|
3
|
+
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/newegg_scraper_chsbr`. To experiment with that code, run `bin/console` for an interactive prompt.
|
|
4
|
+
|
|
5
|
+
TODO: Delete this and the text above, and describe your gem
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
Add this line to your application's Gemfile:
|
|
10
|
+
|
|
11
|
+
```ruby
|
|
12
|
+
gem 'newegg_scraper_chsbr'
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
And then execute:
|
|
16
|
+
|
|
17
|
+
$ bundle install
|
|
18
|
+
|
|
19
|
+
Or install it yourself as:
|
|
20
|
+
|
|
21
|
+
$ gem install newegg_scraper_chsbr
|
|
22
|
+
|
|
23
|
+
## Usage
|
|
24
|
+
|
|
25
|
+
TODO: Write usage instructions here
|
|
26
|
+
|
|
27
|
+
## Development
|
|
28
|
+
|
|
29
|
+
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
30
|
+
|
|
31
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
32
|
+
|
|
33
|
+
## Contributing
|
|
34
|
+
|
|
35
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/newegg_scraper_chsbr. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/newegg_scraper_chsbr/blob/master/CODE_OF_CONDUCT.md).
|
|
36
|
+
|
|
37
|
+
## License
|
|
38
|
+
|
|
39
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
|
40
|
+
|
|
41
|
+
## Code of Conduct
|
|
42
|
+
|
|
43
|
+
Everyone interacting in the NeweggScraperChsbr project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/newegg_scraper_chsbr/blob/master/CODE_OF_CONDUCT.md).
|
data/Rakefile
ADDED
data/bin/newegg_scraper
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<mxfile host="app.diagrams.net" modified="2021-04-11T18:44:34.960Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36" etag="iBiGEqASrCRcf-A13k5W" version="14.5.10" type="device"><diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">7Vtbc9o4FP41zHQfsmPZ5pLHQC5NN82mJW32rSOwMCqyxcoihPz6HtkythEX0+CBEmaYwTqWZel8n6TvHNs1pxO83Ag8Hn7mHmE12/Jeas5lzbaRa1vwpyyzxHJedxODL6inK2WGLn0l2qiv8yfUI1GhouScSTouGvs8DElfFmxYCD4tVhtwVrzrGPvEMHT7mJnWJ+rJYWJt2c3M/pFQf5jeGTXOkzMBTivrkURD7PFpzuRc1ZyO4FwmR8FLhzDlvNQvT7ezJ3Y3atx8+hL9j7+1/3m8/36WNHa9zSXzIQgSyt9u+nU0uP743f353/jh+svUurnofTrTl1jPmE20vzrjSaTHK2epE6MpDRgOodQe8FB29RlwQRsz6odw3Ie+EQGGZyIkBf9f6BOSj8HaH1Lm3eEZn6gRRBL3R2mpPeSCvkKzmMEpBAY4LaSmkt0o1OiqK8FsgVWQCOo8pG5Bc9MdjqSu0+eM4XFEe3GHVZUAC5+GbS4lD9KG+CT0iKdLc5zjghR8NGeOur4kGBo05Q3ykqOiBueG8IBIMYMq+uycZ3qiobQ8zWiLGto2zFPWaenpoqeKP297fruvMLVw6IMTsvs5C/dzS94PECncDjMAPsSStJUbozwJ4SA31MwUU3MLmiKDpg+C9onBU3C1zHGSkYFcychojPs09O/iOpduZvmqh6pMHK4dsJgNQ+p5JIzZIrHECaEURcachjL2Rb0NP/BYx/q7XqtDhzpQRlkZfqq6kB0eArEwjRlEgK1Tohi7hFtrZ+1mbs2KkG0LbZ5JBUy3BdA2ALyVJPhxj4PjA3HNajKUAdOHVUFdt/cMtWPOVR7FC/CPzpAf46zdL+DN1p4Bdw3ADYQZjbWD9gZauq9ugD8AIFVzKd6Pig6XZ8jghGNywlmCP8M9woCZVFKu2hdJ3QVe7GvBRpZbDtU1W/+bQK0boP47GIC+U+pdqfUh55ESE52Hb92jm9CVoYr2vTg3DFi7cuKpYZaU/NZJ8u9O8qMFye80qpX8aEHyO63DlfzN1Uy17idBT61F73PZaZSm2aGo/5aB5QX0XiVubOszFqOjQ3KninBrvPceApyfFOHOQS0r86sShMhMFd7Cwm9dARrxVmtbj0oZXoWCq9aObEbnNZBbFcatfS/UyEy03RC143ZJQEMsFOCPeATOPjZ8q4J0/5I/7cC61ZiE3oV61gKlHuNKbbfBpOU4spLiNWXpzrVBLxe3P+L5JJ06hPX49CoztGMDnEiZsrXajvhE9MlmyCC+8EmJaE71di2wy4AUhGFJn4uPnNZo8wdF3EyX21ZRl9uWU2wiGaS+Kv+MZ9uGEi8YDe0sT78k+Sf4gEQRN+X6KcKsPsJ0FiO+shFm09pI4iUBpuOWCDCbhxFgIjNv2cUM6xX3He5t88n75wSWyExTGui990hje1jLxo+VhRpmlvJdKZYUsYOWLGhhpW/+pmRxFhqymwt9qVqyLEk09gUeL8kvngRL9YKlbrUKdCj9FoyN1mwyqxVLvf4HvQVj5lH1azDQFwdlKcV3qF2apYl2MNrFzJKe3ompCOy9Z8RtMzA+CdU3w7rvnLhthpA0BF8BSq/kgx9nTj1w6F9HN58rg3TvKXB7u+CDj2NXbxdezCMTdADBhoZoZ8GGhlkxKS+sXN2hN8Yd9YUMEzq3i02UjjtaGxpaEXcYDZ0t5rwW2uGDQUQqCV1sU0Ew8Kx6SIPNF3r0kpPbXuIwYeX6oJaAMpHNfM+JN4s2BCZ+TPcOZxzqX4Y83ta2ii7i8ErH8nb55cZe9Xhm/nmJHm0t/wVHKfae74S9Z04xunGtt3MFitk3KEn17Ese5+oX</diagram></mxfile>
|
data/lib/environment.rb
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
require 'open-uri'
|
|
2
|
+
require 'nokogiri'
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require_relative "./newegg_scraper_chsbr/version.rb"
|
|
5
|
+
require_relative './newegg_scraper_chsbr/data_grabber.rb'
|
|
6
|
+
require_relative './newegg_scraper_chsbr/scraper.rb'
|
|
7
|
+
require_relative './newegg_scraper_chsbr/user.rb'
|
|
8
|
+
require_relative './newegg_scraper_chsbr/name.rb'
|
|
9
|
+
require_relative './newegg_scraper_chsbr/price.rb'
|
|
10
|
+
require_relative './newegg_scraper_chsbr/cli.rb'
|
|
11
|
+
require_relative './newegg_scraper_chsbr/cpu.rb'
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
|
|
2
|
+
class NeweggScraperChsbr::CLI
|
|
3
|
+
attr_reader :user
|
|
4
|
+
def initialize
|
|
5
|
+
NeweggScraperChsbr::DataGrabber.new
|
|
6
|
+
end
|
|
7
|
+
def start
|
|
8
|
+
puts "Hello! I heard you were interested in purchasing some CPU's."
|
|
9
|
+
puts "Please enter y to browse."
|
|
10
|
+
@counter = gets.strip
|
|
11
|
+
while @counter == 'y' do
|
|
12
|
+
get_init_data_for_user
|
|
13
|
+
NeweggScraperChsbr::Cpu.display_cpu(@user)
|
|
14
|
+
@user.choose_cpu
|
|
15
|
+
NeweggScraperChsbr::Cpu.display_cpu_with_extras(@user)
|
|
16
|
+
puts "If you would like to continue please enter : #{@counter}"
|
|
17
|
+
@counter = gets.strip
|
|
18
|
+
end
|
|
19
|
+
puts "Thanks for using Chesbro's Scraper!"
|
|
20
|
+
end
|
|
21
|
+
def get_init_data_for_user
|
|
22
|
+
|
|
23
|
+
if @counter == 'y'
|
|
24
|
+
get_price_range
|
|
25
|
+
puts "Are you searching for Intel or AMD?"
|
|
26
|
+
puts "Please put i for Intel, or a for AMD."
|
|
27
|
+
puts "Please put x for either."
|
|
28
|
+
@cpu_make = gets.strip
|
|
29
|
+
puts "\n\nYou want to search for (an) #{NeweggScraperChsbr::Cpu.cpu_maker(@cpu_make)} CPU."
|
|
30
|
+
if @max_price.to_i != 0 && @min_price.to_i != 0
|
|
31
|
+
puts "You entered a maximum price of $#{@max_price}, and a minimum price of $#{@min_price}."
|
|
32
|
+
elsif @max_price.to_i != 0
|
|
33
|
+
puts "You entered a maximum price of $#{@max_price}."
|
|
34
|
+
else
|
|
35
|
+
puts "You did not set a price to search for."
|
|
36
|
+
end
|
|
37
|
+
if_correct_info
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
def if_correct_info
|
|
43
|
+
puts "\n IF this data is correct please enter y"
|
|
44
|
+
@counter = gets.strip
|
|
45
|
+
if @counter == 'y'
|
|
46
|
+
@user = NeweggScraperChsbr::User.new(@cpu_make, @min_price, @max_price)
|
|
47
|
+
else
|
|
48
|
+
get_init_data_for_user
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
def get_price_range
|
|
52
|
+
puts "Hello! Would you like to set a budget? If no, set your maximum budget to 0"
|
|
53
|
+
puts "Please enter the integer value of your maximum price."
|
|
54
|
+
puts "$500.50, please enter as 501, or 500 to stay below budget."
|
|
55
|
+
@max_price = gets.strip
|
|
56
|
+
if @max_price.to_i == 0
|
|
57
|
+
@min_price = "0"
|
|
58
|
+
else
|
|
59
|
+
puts "Please enter the minimum you would spend on a CPU."
|
|
60
|
+
puts "Please use the same format as above."
|
|
61
|
+
@min_price = gets.strip
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
end
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
|
|
2
|
+
class NeweggScraperChsbr::Cpu
|
|
3
|
+
attr_reader :name, :price, :desc, :shipping
|
|
4
|
+
@@all = []
|
|
5
|
+
def initialize(name, price, shipping, desc)
|
|
6
|
+
@name = NeweggScraperChsbr::Name.new(name)
|
|
7
|
+
@price = NeweggScraperChsbr::Price.new(price)
|
|
8
|
+
@desc = desc
|
|
9
|
+
@shipping = shipping
|
|
10
|
+
@@all << self
|
|
11
|
+
end
|
|
12
|
+
def self.all
|
|
13
|
+
@@all
|
|
14
|
+
end
|
|
15
|
+
def self.cpu_maker(cpu_make)
|
|
16
|
+
if cpu_make == 'i'
|
|
17
|
+
"Intel"
|
|
18
|
+
elsif cpu_make == 'a'
|
|
19
|
+
"AMD"
|
|
20
|
+
else
|
|
21
|
+
"either Intel or AMD"
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
def self.display_cpu(user)
|
|
25
|
+
counter = 1
|
|
26
|
+
@@all.each do | computer |
|
|
27
|
+
if user.max_price.to_i != 0 && user.min_price.to_i == 0
|
|
28
|
+
binding.pry
|
|
29
|
+
|
|
30
|
+
user.min_price = "1"
|
|
31
|
+
end
|
|
32
|
+
if user.max_price.to_i == 0 &&
|
|
33
|
+
user.min_price.to_i == 0 &&
|
|
34
|
+
(NeweggScraperChsbr::Cpu.cpu_maker(user.cpu_make) == "Intel" || NeweggScraperChsbr::Cpu.cpu_maker(user.cpu_make) == "AMD")
|
|
35
|
+
computer.name.name.include?(NeweggScraperChsbr::Cpu.cpu_maker(user.cpu_make)) ? puts("\n #{counter}:\n Name: #{computer.name.name}\n Price: #{computer.price.price}") : nil
|
|
36
|
+
elsif user.max_price.to_i == 0 &&
|
|
37
|
+
user.min_price.to_i == 0 &&
|
|
38
|
+
user.cpu_make == 'x'
|
|
39
|
+
puts("\n #{counter}:\n Name: #{computer.name.name}\n Price: #{computer.price.price}")
|
|
40
|
+
elsif user.max_price.to_i >= computer.price.convert_price &&
|
|
41
|
+
user.min_price.to_i <= computer.price.convert_price &&
|
|
42
|
+
(NeweggScraperChsbr::Cpu.cpu_maker(user.cpu_make) == "Intel" || NeweggScraperChsbr::Cpu.cpu_maker(user.cpu_make) == "AMD")
|
|
43
|
+
computer.name.name.include?(NeweggScraperChsbr::Cpu.cpu_maker(user.cpu_make)) ? puts("\n #{counter}:\n Name: #{computer.name.name}\n Price: #{computer.price.price}") : nil
|
|
44
|
+
|
|
45
|
+
elsif user.max_price.to_i >= computer.price.convert_price &&
|
|
46
|
+
user.min_price.to_i <= computer.price.convert_price &&
|
|
47
|
+
user.cpu_make == 'x'
|
|
48
|
+
puts("\n #{counter}:\n Name: #{computer.name.name}\n Price: #{computer.price.price}")
|
|
49
|
+
end
|
|
50
|
+
counter += 1
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
end
|
|
54
|
+
def self.display_cpu_with_extras(user)
|
|
55
|
+
|
|
56
|
+
user.chosen_cpu.each do | chosen_cpu |
|
|
57
|
+
puts "\n\n\n#{@@all[chosen_cpu - 1].name.name}\nHas a price of: #{@@all[chosen_cpu - 1].price.price}\nShipping: #{@@all[chosen_cpu - 1].shipping}\n\n Descriptive Points:\n"
|
|
58
|
+
@@all[chosen_cpu - 1].printDesc
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
def printDesc
|
|
62
|
+
@desc.each do | key, value |
|
|
63
|
+
puts " #{key + 1}: #{value}"
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
end
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
|
|
2
|
+
class NeweggScraperChsbr::DataGrabber
|
|
3
|
+
attr_reader :cpus
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def initialize()
|
|
7
|
+
shipping_price = getShipping
|
|
8
|
+
prices = getPrice
|
|
9
|
+
names = getNames
|
|
10
|
+
descHash = getCpuDesc
|
|
11
|
+
@cpus = makeCpus(prices, names, shipping_price, descHash)
|
|
12
|
+
end #end initialize
|
|
13
|
+
|
|
14
|
+
def getCpuDesc
|
|
15
|
+
scraped = NeweggScraperChsbr::Scraper.new
|
|
16
|
+
pages = []
|
|
17
|
+
description = []
|
|
18
|
+
css_next_link = scraped.xml_obj.css ".item-title"
|
|
19
|
+
css_next_link.each_with_index do | element, index |
|
|
20
|
+
if index != 0
|
|
21
|
+
pages << element.attributes["href"].value
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
description = {}
|
|
25
|
+
counter = 1
|
|
26
|
+
puts "Getting data.. Please wait.\n"
|
|
27
|
+
pages.each_with_index do | url, index_of_pages |
|
|
28
|
+
description[index_of_pages] = {}
|
|
29
|
+
scraped_info = NeweggScraperChsbr::Scraper.new(url)
|
|
30
|
+
descriptors = scraped_info.xml_obj.css(".product-bullets")
|
|
31
|
+
descriptors.children.children.each do |info |
|
|
32
|
+
info.children.each do | more_info |
|
|
33
|
+
|
|
34
|
+
description[index_of_pages][counter] = more_info.text
|
|
35
|
+
counter += 1
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
counter = 0
|
|
39
|
+
|
|
40
|
+
end
|
|
41
|
+
puts "Done!\n"
|
|
42
|
+
description
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
end
|
|
47
|
+
def split_price(html_element) # This method checks each individual text element, this is to make sure the prices are accurate.
|
|
48
|
+
counter = 0
|
|
49
|
+
until counter == 100
|
|
50
|
+
if counter < 10 # If the counter is less than 10, is necessary because the computer doesn't know we are working with money.
|
|
51
|
+
if html_element.text.include?(".0#{counter.to_s}") #If the text element includes any dollar ammount below .10 is a neccessary check.
|
|
52
|
+
#Otherwise the computer would only check [.0.. .9], and we need it to check [.00.. .09]
|
|
53
|
+
price = html_element.text.split ".0#{counter.to_s}" #Once we know it's the correct element we can assign it to the price variable.
|
|
54
|
+
#This is only necessary to be able to delete the data we are splitting off.
|
|
55
|
+
# (We are splitting anything after the cents amount because it was uneccessary.)
|
|
56
|
+
price.delete_at 1
|
|
57
|
+
return price, ".0#{counter.to_s}" # Return the dollar amount, and the cents amount in an array.
|
|
58
|
+
end
|
|
59
|
+
elsif counter >= 10 # This does the same as above, but for any number over 9.
|
|
60
|
+
if html_element.text.include?(".#{counter.to_s}")
|
|
61
|
+
|
|
62
|
+
price = (html_element.text.split ".#{counter.to_s}")
|
|
63
|
+
price.delete_at 1
|
|
64
|
+
return price, (".#{counter.to_s}")
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
counter += 1
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
def isCoolerOrMB?(name) # This methods name means is it a cooler or a motherboard?
|
|
71
|
+
name.include?("Water") || name.include?("Air") || name.include?("Motherboard") # this will return true if
|
|
72
|
+
# any of the products include
|
|
73
|
+
# words that describe motherboards and/or
|
|
74
|
+
# cooling products.
|
|
75
|
+
end
|
|
76
|
+
def getPrice
|
|
77
|
+
scraped_info = NeweggScraperChsbr::Scraper.new
|
|
78
|
+
css_price = scraped_info.xml_obj.css ".price-current" # This will set the variable equal to the
|
|
79
|
+
# html that contains what I need to find the price.
|
|
80
|
+
|
|
81
|
+
prices = [] # This will hold all of the prices in an array.
|
|
82
|
+
|
|
83
|
+
css_price.each do | piece | # Going 1 level deep into the html obj that contains the price text element.
|
|
84
|
+
|
|
85
|
+
temp_price = split_price piece # split_price will split the price, and return
|
|
86
|
+
# two variables in an array. The first element of
|
|
87
|
+
# the array will be the dollar amount.
|
|
88
|
+
# The second element will be the cent ammount because
|
|
89
|
+
# I deemed it easier to remove anything after the
|
|
90
|
+
# floating point, it was the only pattern that was
|
|
91
|
+
# consistent.
|
|
92
|
+
|
|
93
|
+
if temp_price != nil
|
|
94
|
+
temp_price.flatten! # When returned by #split_price the dollar amount is
|
|
95
|
+
# nested inside of another array. I use flatten!
|
|
96
|
+
# to return the flattened array to temp_price.
|
|
97
|
+
|
|
98
|
+
prices << "#{temp_price[0]}#{temp_price[1]}" # Here prices is being shoveled the
|
|
99
|
+
# two elements, concatenated to create
|
|
100
|
+
# a string similar to "$00.00"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
end #end prices.each
|
|
104
|
+
prices
|
|
105
|
+
end
|
|
106
|
+
def getNames
|
|
107
|
+
scraped = NeweggScraperChsbr::Scraper.new
|
|
108
|
+
names = [] # Names will hold the names of
|
|
109
|
+
# objects found on the page.
|
|
110
|
+
|
|
111
|
+
css_name = scraped.xml_obj.css ".item-title" # css_name will hold the html
|
|
112
|
+
# object to be enumerated upon.
|
|
113
|
+
|
|
114
|
+
css_name.each_with_index do | name, index | # css_name will be enumerated upon
|
|
115
|
+
# to retrieve the strings.
|
|
116
|
+
|
|
117
|
+
index != 0 ? names << name.text : nil # I have to skip the first name
|
|
118
|
+
# because it does not have a price.
|
|
119
|
+
# After I skip the first name
|
|
120
|
+
# I will shovel the text from the
|
|
121
|
+
# html object into the names array.
|
|
122
|
+
end #end names.each_with_index
|
|
123
|
+
names
|
|
124
|
+
end
|
|
125
|
+
def getShipping
|
|
126
|
+
scraped = NeweggScraperChsbr::Scraper.new
|
|
127
|
+
shipping = []
|
|
128
|
+
css_shipping = scraped.xml_obj.css ".price-ship"
|
|
129
|
+
|
|
130
|
+
css_shipping.each do | name |
|
|
131
|
+
shipping << name.text
|
|
132
|
+
end
|
|
133
|
+
shipping
|
|
134
|
+
end
|
|
135
|
+
def makeCpus(prices, names, shipping, desc_hash)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
cpus = []
|
|
139
|
+
prices.each_with_index do | price, index | # I will then enumerate on
|
|
140
|
+
# each of the prices with their
|
|
141
|
+
# index so that the enumation can
|
|
142
|
+
# access the names array at the
|
|
143
|
+
# same time.
|
|
144
|
+
|
|
145
|
+
if !isCoolerOrMB?(names[index]) # Here the program check if
|
|
146
|
+
# the name is a cooler
|
|
147
|
+
# or motherboard because
|
|
148
|
+
# there is a price associated
|
|
149
|
+
# with the coolers on the
|
|
150
|
+
# website.
|
|
151
|
+
|
|
152
|
+
cpus << NeweggScraperChsbr::Cpu.new(names[index], price, shipping[index], desc_hash[index])
|
|
153
|
+
|
|
154
|
+
end #end if
|
|
155
|
+
end #end prices.each_with_index
|
|
156
|
+
cpus
|
|
157
|
+
end
|
|
158
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
class NeweggScraperChsbr::Scraper
|
|
4
|
+
attr_reader :xml_obj
|
|
5
|
+
|
|
6
|
+
URL = "https://www.newegg.com/p/pl?d=cpu"
|
|
7
|
+
def initialize(url = URL)
|
|
8
|
+
@xml_obj = Nokogiri::HTML(open(url)) # This opens the URL with open-uri, then parses it as an XML Object
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
|
|
2
|
+
class NeweggScraperChsbr::User
|
|
3
|
+
attr_accessor :cpu_make, :min_price, :max_price, :chosen_cpu
|
|
4
|
+
def initialize(cpu_make = 'x', min_price = 0, max_price = 0)
|
|
5
|
+
@cpu_make = cpu_make
|
|
6
|
+
@min_price = min_price
|
|
7
|
+
@max_price = max_price
|
|
8
|
+
@chosen_cpu = []
|
|
9
|
+
end
|
|
10
|
+
def choose_cpu
|
|
11
|
+
puts "Please enter the number of the CPU you'd like to choose."
|
|
12
|
+
chosen = gets.strip.to_i
|
|
13
|
+
@chosen_cpu << chosen
|
|
14
|
+
puts "Would you like to add another CPU?\nEnter y to do so."
|
|
15
|
+
if gets.strip == 'y'
|
|
16
|
+
choose_cpu
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "lib/newegg_scraper_chsbr/version"
|
|
4
|
+
|
|
5
|
+
Gem::Specification.new do |spec|
|
|
6
|
+
spec.name = "newegg_scraper_chsbr"
|
|
7
|
+
spec.version = NeweggScraperChsbr::VERSION
|
|
8
|
+
spec.authors = ["Lrd134"]
|
|
9
|
+
spec.email = ["larryc3200@gmail.com"]
|
|
10
|
+
|
|
11
|
+
spec.summary = "Cpu Shopping Data from Newegg"
|
|
12
|
+
spec.description = "To scrape CPU's off of Newegg and display them nicely."
|
|
13
|
+
spec.homepage = "https://github.com/Lrd134/aa_phase_1_proj"
|
|
14
|
+
spec.license = "MIT"
|
|
15
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
|
|
16
|
+
|
|
17
|
+
#spec.metadata["allowed_push_host"] = "https://rubygems.org/"
|
|
18
|
+
|
|
19
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
|
20
|
+
spec.metadata["source_code_uri"] = "https://github.com/Lrd134/aa_phase_1_proj"
|
|
21
|
+
spec.metadata["changelog_uri"] = "https://github.com/Lrd134/aa_phase_1_proj/blob/master/CHANGELOG.md"
|
|
22
|
+
|
|
23
|
+
# Specify which files should be added to the gem when it is released.
|
|
24
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
25
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
|
26
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
|
27
|
+
end
|
|
28
|
+
spec.bindir = "exe"
|
|
29
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
|
30
|
+
spec.require_paths = ["lib"]
|
|
31
|
+
|
|
32
|
+
# Uncomment to register a new dependency of your gem
|
|
33
|
+
|
|
34
|
+
spec.add_dependency "nokogiri"
|
|
35
|
+
spec.add_dependency "open-uri"
|
|
36
|
+
|
|
37
|
+
# For more information and examples about making a new gem, checkout our
|
|
38
|
+
# guide at: https://bundler.io/guides/creating_gem.html
|
|
39
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: newegg_scraper_chsbr
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Lrd134
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: exe
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2021-04-15 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: nokogiri
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '0'
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - ">="
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: open-uri
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - ">="
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '0'
|
|
34
|
+
type: :runtime
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - ">="
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '0'
|
|
41
|
+
description: To scrape CPU's off of Newegg and display them nicely.
|
|
42
|
+
email:
|
|
43
|
+
- larryc3200@gmail.com
|
|
44
|
+
executables: []
|
|
45
|
+
extensions: []
|
|
46
|
+
extra_rdoc_files: []
|
|
47
|
+
files:
|
|
48
|
+
- ".gitignore"
|
|
49
|
+
- Gemfile
|
|
50
|
+
- Gemfile.lock
|
|
51
|
+
- LICENSE.txt
|
|
52
|
+
- README.md
|
|
53
|
+
- Rakefile
|
|
54
|
+
- bin/newegg_scraper
|
|
55
|
+
- class_diagram.drawio
|
|
56
|
+
- lib/environment.rb
|
|
57
|
+
- lib/newegg_scraper_chsbr/cli.rb
|
|
58
|
+
- lib/newegg_scraper_chsbr/cpu.rb
|
|
59
|
+
- lib/newegg_scraper_chsbr/data_grabber.rb
|
|
60
|
+
- lib/newegg_scraper_chsbr/descriptors.rb
|
|
61
|
+
- lib/newegg_scraper_chsbr/name.rb
|
|
62
|
+
- lib/newegg_scraper_chsbr/price.rb
|
|
63
|
+
- lib/newegg_scraper_chsbr/scraper.rb
|
|
64
|
+
- lib/newegg_scraper_chsbr/user.rb
|
|
65
|
+
- lib/newegg_scraper_chsbr/version.rb
|
|
66
|
+
- newegg_scraper_chsbr.gemspec
|
|
67
|
+
homepage: https://github.com/Lrd134/aa_phase_1_proj
|
|
68
|
+
licenses:
|
|
69
|
+
- MIT
|
|
70
|
+
metadata:
|
|
71
|
+
homepage_uri: https://github.com/Lrd134/aa_phase_1_proj
|
|
72
|
+
source_code_uri: https://github.com/Lrd134/aa_phase_1_proj
|
|
73
|
+
changelog_uri: https://github.com/Lrd134/aa_phase_1_proj/blob/master/CHANGELOG.md
|
|
74
|
+
post_install_message:
|
|
75
|
+
rdoc_options: []
|
|
76
|
+
require_paths:
|
|
77
|
+
- lib
|
|
78
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - ">="
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: 2.4.0
|
|
83
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
84
|
+
requirements:
|
|
85
|
+
- - ">="
|
|
86
|
+
- !ruby/object:Gem::Version
|
|
87
|
+
version: '0'
|
|
88
|
+
requirements: []
|
|
89
|
+
rubygems_version: 3.2.13
|
|
90
|
+
signing_key:
|
|
91
|
+
specification_version: 4
|
|
92
|
+
summary: Cpu Shopping Data from Newegg
|
|
93
|
+
test_files: []
|