ebookdealinfo 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b2b469d9f1996f015d320bbf4948873186bea439
+  data.tar.gz: 3ba1dad6388407f5fff202de99e69afa8b5e0f95
+SHA512:
+  metadata.gz: 231a456cddc2468a41eb3fbb05f28a417c669b830d38100389a1883a4ca7cfe9b027bafc578328010a2105d6c3eb40f262ce0372765e68c687de64e015e85930
+  data.tar.gz: db62e7b8dd07b00465778ee2d435aa9d988738e184e90f7764b6c31fa43ce25e4b0402546fd5fdc5370c9f27e46b9dd8ee6a14157b1fbc039a9f274d886d521e

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.3
+before_install: gem install bundler -v 1.15.4

data/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+#git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in EbookDealInfo.gemspec
+#gemspec
+
+gem 'ebookdealinfo'

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 kylek
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,25 @@
+# ebookdealinfo
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/EbookDealInfo`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+## Installation
+
+    $ gem install ebookdealinfo
+
+## Usage
+
+Reddit /r/Ebookdeals will be scraped for a list of ebooks with current deals and then goodreads will be scraped to get more information about those books. You can then select a book from the menu to get detailed information about it.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/<github username>/EbookDealInfo.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "ebookdealinfo"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/ebookdealinfo

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/ebookdealinfo.rb"
+
+Controller.new.call

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/ebookdealinfo.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "ebookdealinfo/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "ebookdealinfo"
+  spec.version       = EbookDealInfo::VERSION
+  spec.authors       = ["kylek"]
+  spec.email         = ["27875726+kylekinnear@users.noreply.github.com"]
+
+  spec.summary       = "Grabs daily ebook deals and gets goodreads info about them."
+  spec.homepage      = "https://github.com/kylekinnear/ebook-deal-info"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  #  if spec.respond_to?(:metadata)
+  #    spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+  #  else
+  #    raise "RubyGems 2.0 or newer is required to protect against "
+  #      "public gem pushes."
+  #  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  #spec.bindir        = "exe"
+  spec.executables   = ["ebookdealinfo"]#spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.15"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_dependency "nokogiri", "~> 1.8"
+end

data/lib/ebookdealinfo.rb

@@ -0,0 +1,8 @@
+require 'open-uri'
+require 'nokogiri'
+
+require_relative "./ebookdealinfo/version"
+require_relative "./ebookdealinfo/book.rb"
+require_relative "./ebookdealinfo/controller.rb"
+require_relative "./ebookdealinfo/deal_scraper.rb"
+require_relative "./ebookdealinfo/info_scraper.rb"

data/lib/ebookdealinfo/book.rb

@@ -0,0 +1,47 @@
+#this class holds the data for each book
+#require_relative "EbookDealInfo"
+
+class Book
+  attr_accessor :author, :title, :price, :genre_one, :genre_two, :series, :rating, :rates, :blurb, :completable
+  @@all = [] #collection to push all new books into
+
+  def initialize(author, title, price, good_scrape=1)
+    if good_scrape == 1
+      @@all << self #push new books
+      @author = author
+      @title = title
+      @price = price
+      @completable = true
+      InfoScraper.new.info_scrape(self)
+    else
+      @@all << self
+      @completable = false
+    end
+  end
+
+  def self.create(author, title, price, good_scrape=1) #deals scrape will pass in hash with title, author, and price
+    self.new(author, title, price, good_scrape)
+  end
+
+  def self.all #we'll use this to grab the collection, which the goodreads scraper will use
+    @@all
+  end
+
+  def wrap_blurb(width=78) #code from other projects to make blurbs line wrap; handle unicode replacement elsewhere
+  lines = []
+  line = ""
+  @blurb.split(/\s+/).each do |word|
+    if line.size + word.size >= width
+      lines << line
+      line = word
+    elsif line.empty?
+      line = word
+    else
+      line << " " << word
+    end
+  end
+  lines << line if line
+  return (lines.join "\n")
+  end
+
+end

data/lib/ebookdealinfo/controller.rb

@@ -0,0 +1,72 @@
+#this class is called by the cli and handles outputs, menus
+#require_relative "EbookDealInfo"
+
+class Controller
+  attr_accessor :completed_books
+
+  def call
+    welcome #call the intro message
+  end
+
+  def welcome
+    #intro message
+    puts "Welcome to the Ebook Recent Deal Info Getter"
+    puts "Getting the latest deals (this may take a while)"
+    puts "----------------------------------------------------------------------------"
+    DealScraper.new.scrape #call DealScraper, and through it instantiate books and call info_scraper on the books
+    @completed_books = [] #we'll use this to avoid interacting with bad scrapes
+    Book.all.each {|book| @completed_books << book if book.completable == true}
+    list_books #main menu
+  end
+
+  def list_books
+    #the main menu
+    puts "A list of the latest deals:"
+    puts "#{Book.all.select {|i| i.completable == false}.size} book(s) failed to load. Probably a spelling or selector error."
+    @completed_books.each_with_index do |book, index|
+      puts "#{index+1}. #{book.title} - #{book.author} - #{book.genre_one}"
+    end
+
+    interact
+    exeunt
+  end
+
+  def interact
+    input = nil
+    while %w[e exit q quit n no].include?(input) == false
+      puts "----------------------------------------------------------------------------\nEnter the number of the book you'd like more information about."
+      puts "You can type list to list the books again or type quit to leave."
+      input = gets.strip.downcase
+
+      if input.to_i > 0 && input.to_i <= completed_books.size
+        chosen_book = @completed_books[input.to_i-1]
+        puts "----------------------------------------------------------------------------\n#{chosen_book.title}"
+        puts "#{chosen_book.series}" if chosen_book.series.size > 0
+        puts "By #{chosen_book.author}"
+        if chosen_book.price.size > 0
+          puts "Price: #{chosen_book.price}"
+        else
+          puts "Couldn't find a price"
+        end
+        if chosen_book.genre_two.include?(chosen_book.genre_one)
+          puts "Shelved as #{chosen_book.genre_two}"
+        elsif chosen_book.genre_one == "No genre listed"
+          puts "This book doesn't have a genre listing"
+        else
+          puts "Shelved as #{chosen_book.genre_one} #{"and #{chosen_book.genre_two}" if chosen_book.genre_two.size > 0}"
+        end
+        puts "#{chosen_book.rates} people gave this book an average rating of #{chosen_book.rating}"
+        puts "----------------------------------------------------------------------------\n#{chosen_book.wrap_blurb}" #line wrap our blurb
+      elsif input == "list"
+        list_books
+      elsif %w[e exit q quit n no].include?(input) == false
+        puts "Sorry, I couldn't understand that. Please enter a number, list, or quit."
+      end
+    end
+  end
+
+  def exeunt
+    puts "----------------------------------------------------------------------------\nCheck back later for more ebook deals."
+  end
+
+end

data/lib/ebookdealinfo/deal_scraper.rb

@@ -0,0 +1,33 @@
+#this class grabs the daily deals from reddit /r/ebookdeals
+#require_relative "EbookDealInfo"
+
+class DealScraper
+
+  def scrape #this method scrapes the deals page, then for each item on that page, instantiates a new book with the relevant info; are we passing a hash or attributes?
+    deals = Nokogiri::HTML(open("https://www.reddit.com/r/ebookdeals/new/",'User-Agent' => 'Chrome'))
+
+    deals.search("div.link").each_with_index do |post,index|
+      if post.search("p.title").text.size > 0
+        if post.search("p.title").text.include?(";") #for posts formatted "#~Author~; ~Title; ~Price~"
+          author = post.search("p.title").text.split(";")[0].strip
+          #.search("p.title a").attribute("href").value
+          title = post.search("p.title").text.split(";")[1].strip
+          price = post.search("p.title").text.split(";")[2].strip #can't handle free books
+          Book.create(author, title, price)
+          puts "Loaded book ##{index+1} of 25"
+        else #for posts formatted "~Title~ by ~author~ (~Price~)"
+          if post.search("p.title").text.slice(/\s([Bb]y)/) != nil #ignore wacky formatted posts
+            author = post.search("p.title").text.split("by")[1].slice(/\A[^(,$\/]+/).split(". Kindle")[0].split("-- Kindle")[0].strip
+            title = post.search("p.title").text.split("by").first.gsub(/\W+\z/, "").strip
+            price = post.search("p.title").text.slice(/[$]\d+[.]\d+/).strip #can't handle free books
+            Book.create(author, title, price)
+            puts "Loaded book ##{index+1} of 25"
+          else
+            Book.create("","","",0)
+          end
+        end
+      end
+    end
+  end
+
+end

data/lib/ebookdealinfo/info_scraper.rb

@@ -0,0 +1,41 @@
+#this is the goodreads scraper
+#require_relative "EbookDealInfo"
+
+class InfoScraper
+
+  def info_scrape(book) #for each instance of book in the class collection, go get blurb, series, gr rating/rates and add them to that instance; also author to deal with last name only from scrape?
+    search_string = "#{book.title} #{book.author.gsub(".", ". ").gsub(/[^\w\s]/,"")}".gsub(/(\A|\s)\S\s/," ").gsub(/[^a-zA-Z0-9']+/, "+") #turns the author + title into a usable goodreads search string
+          #should remove anything joining multiple authors ("&", ",") that would break the search
+    search_page = Nokogiri::HTML(open("https://www.goodreads.com/search?q=#{search_string}&search_type=books",'User-Agent' => 'Ruby')) #uses the search string to pull an item's goodreads page
+    if search_page.css("table a").size != 0
+      determinant = search_page.css("span.minirating").map.with_index {|i,index| [index, i.text.strip.slice(/\s(\d|,)+/).strip.gsub(",","").to_i]}.sort! {|x,y| x[1].to_i <=> y[1].to_i}.last #the search result with the most rates (and presumably most legitimate) is an array [result_index, #rates]
+      item_page = Nokogiri::HTML(open("https://goodreads.com/#{search_page.css("table a.bookTitle")[determinant[0]].attribute("href").value}",'User-Agent' => 'Ruby').read)
+      book.author = item_page.search("div#bookAuthors.stacked span :not(.greyText) :not(.smallText)").text #gets the complete author name since reddit might not provide it
+      book.title = item_page.search("h1#bookTitle.bookTitle").text.slice(/^(\n).+(\S\n)/).strip #goodreads provides better titles
+      book.series = item_page.search("h1#bookTitle.bookTitle :first-child").text.strip.gsub(/[()]/, "") #provides series
+      book.rating = item_page.search("span.average").text #average rating
+      book.rates = item_page.search("span.votes.value-title").text.strip #number of ratings
+      #blurb needs work
+      book.blurb = item_page.xpath('//span[starts-with(@id, "freeText")]')[1].text#grab the blurb
+      #we will scrape the top two genre entries, but when we want to check if one is a more specific form of the other
+      if item_page.search("div.bigBoxContent div.elementList div.left").empty?
+        book.genre_one = "No genre listed"
+        book.genre_two = ""
+      else
+        genre_one = item_page.search("div.bigBoxContent div.elementList div.left")[0].text.split("\n").map {|i| i.strip if i.strip.size > 0}.reject {|i| i==nil} # turn the first genre into a stripped array of actual content
+        book.genre_one = ""
+        genre_one.each {|i| book.genre_one << "#{i} "}
+        if item_page.search("div.bigBoxContent div.elementList div.left")[1] != nil
+          genre_two = item_page.search("div.bigBoxContent div.elementList div.left")[1].text.split("\n").map {|i| i.strip if i.strip.size > 0}.reject {|i| i==nil}
+          book.genre_two = ""
+          genre_two.each {|i| book.genre_two << "#{i} "}
+        else
+          book.genre_two = ""
+        end
+      end
+    else #instead of a raising a nobook error that will break the looping, let's flag the book as incomplete and not display it at the end
+      book.completable = false
+    end
+  end
+
+end

data/lib/ebookdealinfo/version.rb

@@ -0,0 +1,3 @@
+module EbookDealInfo
+  VERSION = "1.0.0"
+end

data/spec.md

@@ -0,0 +1,6 @@
+# Specifications for the CLI Assessment
+
+Specs:
+- [X] Have a CLI for interfacing with the application - The program begins scraping automatically, than prompts the user with a list of books, each of which the user can use CLI to select and get more information about
+- [X] Pull data from an external source - reddit is scraped for a list of books that currently have available deals, then the goodreads page for each of those books is scraped to get detailed information
+- [X] Implement both list and detail views - The list is of the ~25 books (occasional weird reddit topics will produce cases the scraper will ignore) posted on reddit, and the books have individual details pages

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: ebookdealinfo
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- kylek
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-09-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+description: 
+email:
+- 27875726+kylekinnear@users.noreply.github.com
+executables:
+- ebookdealinfo
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/ebookdealinfo
+- bin/setup
+- ebookdealinfo.gemspec
+- lib/ebookdealinfo.rb
+- lib/ebookdealinfo/book.rb
+- lib/ebookdealinfo/controller.rb
+- lib/ebookdealinfo/deal_scraper.rb
+- lib/ebookdealinfo/info_scraper.rb
+- lib/ebookdealinfo/version.rb
+- spec.md
+homepage: https://github.com/kylekinnear/ebook-deal-info
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.7
+signing_key: 
+specification_version: 4
+summary: Grabs daily ebook deals and gets goodreads info about them.
+test_files: []