yamd 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a7cfa4dadcceadf400f49b0d0fc2e353b9a474a7
-  data.tar.gz: ae7c9eb249ca40a34515832316ca20069addd234
+  metadata.gz: 8a02e2f65df6e064441de364691ef2405e8104b2
+  data.tar.gz: d4e1c6e4aa6c322545e7d1e8684eb49e4dfedf96
 SHA512:
-  metadata.gz: 13e0fd6911898fe1eed2b82ef27e55bdda1cbc4df5020694bdbf5236145a5490a1d518e0bb4a1716977644bfcb8e3ce4774bcc6257a76459a8d08a96b1b26a3b
-  data.tar.gz: b9b3c932f1313b3d6909c88cb292a3be96f3d22b3246c8dc190853152b9897845ecc4621e12d46d2180f1af709c854d4743c17c1a60dfd33b093a7bc38f09725
+  metadata.gz: 322808ece20606194fff0ac606fe313d33d7ce5cbd942cb64475d0ca4f540c33dbdf03db87137594ae2960066a9873b4885dd4d7c44161cd850c7cac3eff94f2
+  data.tar.gz: 233cfec564e2985af9facf3c77617c221b4b823e740fd98fd00deb4faadec7a04124e49fb53528245430f11c2afe4bea8d31db8dc46f3078df6245d7871a186e

data/bin/yamd

@@ -3,11 +3,12 @@
 require 'yamd/mangahere'
 require 'yamd/mangafox'
 require 'yamd/fakku'
+require 'yamd/hentaicafe'
 
 unless ARGV.size > 0
   puts 'USAGE: yamd <manga main page url>'
   puts 'EXAMPLE: yamd http://www.mangahere.com/manga/asagao_to_kase_san/'
-  puts 'Support mangahere and mangafox sites so far.'
+  #puts 'Support mangahere and mangafox sites so far.'
   exit
 end
 
@@ -18,6 +19,8 @@ elsif /mangahere/.match(manga_main_page_url)
   manga = MangahereCrawler.new(manga_main_page_url)
 elsif /fakku/.match(manga_main_page_url)
   manga = FakkuCrawler.new(manga_main_page_url)
+elsif /hentai\.cafe/.match(manga_main_page_url)
+  manga = HentaiCafeCrawler.new(manga_main_page_url)
 else
   puts "The argument (#{manga_main_page_url}) doesn't seem to be a URL of one of the supported sites."
 end

data/lib/yamd.rb

@@ -1,31 +1,52 @@
 require 'nokogiri'
 require 'open-uri'
 require 'addressable/uri'
+require 'resolv-replace'
 require 'pathname'
 
 require 'capybara'
 require 'capybara/poltergeist'
+#require 'phantomjs'
 
 Capybara.register_driver(:poltergeist) do | app |
-  Capybara::Poltergeist::Driver.new(app, js_errors: false)
+  #Capybara::Poltergeist::Driver.new(app, { js_errors: false})
+  #Capybara::Poltergeist::Driver.new(app, { phantomjs: Phantomjs.path, js_errors: false})
+  Capybara::Poltergeist::Driver.new(app, {
+    # this blacklist was needed to unbloat mangahere downloader,
+    # without it the mangahere downloader often timeout
+    url_blacklist: [
+      'googletagmanager.com',
+      'googleapis.com',
+      'facebook.net',
+      'facebook.com',
+      'adtrue.com',
+      'z6.com',
+      'sharethis.com',
+      'puserving.com'
+    ],
+    js_errors: false,
+    phantomjs_options: ['--ignore-ssl-errors=yes', '--load-images=false'] 
+  })
 end
 
 Capybara.default_driver = :poltergeist
 Capybara.run_server = false
-$internet = Capybara.current_session
+$session = Capybara.current_session
   
 def my_open(url)
-  $internet.visit url
-
-  $internet.html
+  puts "visiting " + url
+  $session.visit url
+  
+  $session.html
 end
 
 class PageCrawler
-  attr_reader :custom_data, :url, :parsed_html, :number, :chapter
+  attr_reader :custom_data, :url, :uri, :parsed_html, :number, :chapter
 
   def initialize(custom_data, parsed_html, number, chapter)
     @custom_data = custom_data
     @url = custom_data[:url]
+    @uri = Addressable::URI.heuristic_parse(url)
     @parsed_html = parsed_html
     @number = number
     @chapter = chapter
@@ -34,14 +55,19 @@ class PageCrawler
   def image_url
     fail 'This method is abstract and have to be defined in a subclass.'
   end
+
+  def clean_image_url
+    @uri.join(image_url).normalize.to_s
+  end
 end
 
 class ChapterCrawler
-  attr_reader :custom_data, :url, :parsed_html, :number, :manga
+  attr_reader :custom_data, :url, :uri, :parsed_html, :number, :manga
 
   def initialize(custom_data, chapter_page, number, manga)
     @custom_data = custom_data
     @url = custom_data[:url]
+    @uri = Addressable::URI.heuristic_parse(url)
     @number = number
     @parsed_html = chapter_page
     @manga = manga
@@ -59,7 +85,10 @@ class ChapterCrawler
     Enumerator.new do | yielder |
       number = 1
       pages_info.each do | page_info |
-        parsed_html = Nokogiri::HTML(my_open(page_info[:url]))
+        # fix the url to be absolute
+        full_url = @uri.join(page_info[:url]).to_s
+        page_info[:url] = full_url
+        parsed_html = Nokogiri::HTML(my_open(full_url))
         yielder.yield self.class.page_class.new(page_info, parsed_html, number, self)
         number += 1
       end
@@ -72,11 +101,12 @@ class ChapterCrawler
 end
 
 class MangaCrawler
-  attr_accessor :url, :parsed_html
+  attr_accessor :url, :uri, :parsed_html
 
   def initialize(manga_main_page_url)
     @url = manga_main_page_url
-    @parsed_html = Nokogiri::HTML(open(manga_main_page_url))
+    @uri = Addressable::URI.heuristic_parse(url)
+    @parsed_html = Nokogiri::HTML(my_open(manga_main_page_url))
   end
 
   def chapters_info
@@ -87,7 +117,10 @@ class MangaCrawler
     Enumerator.new do | yielder |
       number = 1
       chapters_info.each do | chapter_info |
-        page = Nokogiri::HTML(my_open(chapter_info[:url]))
+        # fix the url to be absolute
+        full_url = @uri.join(chapter_info[:url]).to_s
+        chapter_info[:url] = full_url
+        page = Nokogiri::HTML(my_open(full_url))
         yielder.yield self.class.chapter_class.new(chapter_info, page, number, self)
         number += 1
       end
@@ -143,9 +176,10 @@ class ImageDownloader
           page_name = self.class.format_page_name(page, chapter, manga)
           page_abs_path = chapter_dir.join(page_name).to_s
           File.open(page_abs_path, 'wb') do | f |
-            safe_uri = URI.encode(page.image_url, '[]')
-            open(safe_uri, 'rb') do | image |
-              f.write(image.read)
+            open(page.clean_image_url) do | image |
+              # TODO: check if copy_stream avoids alloacting the whole image in
+              # memory before starting to flush it
+              IO.copy_stream(image, f)
             end
           end
         end

data/lib/yamd/gehentai.rb

@@ -0,0 +1,44 @@
+require 'yamd'
+
+class MangaherePage < PageCrawler
+  def image_url
+    @parsed_html.at_css('#viewer a img')['src']
+  end
+end
+
+class MangahereChapter < ChapterCrawler
+  def self.page_class
+    MangaherePage
+  end
+
+  def pages_info
+    # there's no need of an lazy enumerator here, no IO action is taken
+    page_options = @parsed_html.at_css('.prew_page + select').css('option')
+    page_urls = []
+    page_options.each do | option |
+      page_urls << { url: option['value'] }
+    end
+    page_urls
+  end
+
+  def name
+    @custom_data[:name]
+  end
+end
+
+class MangahereCrawler < MangaCrawler
+  def self.chapter_class
+    MangahereChapter
+  end
+
+  def chapters_info
+    url = URI.join(self.url, @parsed_html.at_css('a.button.green')['href'])
+    [{ name: 'OnlyChapter',
+      url: url }]
+  end
+
+  def name
+    @parsed_html.at_css('h1').text.strip
+  end
+end
+

data/lib/yamd/hentaicafe.rb

@@ -0,0 +1,44 @@
+require 'yamd'
+
+class HentaiCafePage < PageCrawler
+  def image_url
+    @parsed_html.at_css('#page img')['src']
+  end
+end
+
+class HentaiCafeChapter < ChapterCrawler
+  def self.page_class
+    HentaiCafePage
+  end
+
+  def pages_info
+    # there's no need of an lazy enumerator here, no IO action is taken
+    page_list = @parsed_html.at_css('ul.dropdown').css('li')
+    page_urls = []
+    page_list.each do | li_el |
+      page_urls << { url: li_el.css('a').first['href'] }
+    end
+    page_urls
+  end
+
+  def name
+    @custom_data[:name]
+  end
+end
+
+class HentaiCafeCrawler < MangaCrawler
+  def self.chapter_class
+    HentaiCafeChapter
+  end
+
+  def chapters_info
+    css = 'a.x-btn.x-btn-flat.x-btn-rounded.x-btn-large'
+    [{ name: 'OnlyChapter',
+      url: @parsed_html.at_css(css)['href'] }]
+  end
+
+  def name
+    @parsed_html.at_css('h3').text.strip
+  end
+end
+

data/lib/yamd/mangahere.rb

@@ -2,7 +2,7 @@ require 'yamd'
 
 class MangaherePage < PageCrawler
   def image_url
-    @parsed_html.at_css('#viewer a img')['src']
+    @parsed_html.at_css('#viewer a img:not(.loadingImg)')['src']
   end
 end
 
@@ -18,7 +18,12 @@ class MangahereChapter < ChapterCrawler
     page_options.each do | option |
       page_urls << { url: option['value'] }
     end
-    page_urls
+    # drop the 'featured' page at end of each chapter
+    if /featured/.match(page_urls.last[:url]) then
+      page_urls[0...-1]
+    else
+      page_urls
+    end
   end
 
   def name
@@ -33,9 +38,9 @@ class MangahereCrawler < MangaCrawler
 
   def chapters_info
     @parsed_html.css('.detail_list ul li a').reverse.map do | chapter_link |
-        { name: chapter_link.text.strip,
-          url: chapter_link['href']
-        }
+      { name: chapter_link.text.strip,
+        url: chapter_link['href']
+      }
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yamd
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Henrique Becker
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-25 00:00:00.000000000 Z
+date: 2018-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '2.1'
 description: 'This gem offers: classes to subclass and create a manga site crawler;
   a dowloader to use with these classes; some site-specific scripts.'
 email: henriquebecker91@gmail.com
@@ -91,11 +91,13 @@ files:
 - bin/yamd
 - lib/yamd.rb
 - lib/yamd/fakku.rb
+- lib/yamd/gehentai.rb
+- lib/yamd/hentaicafe.rb
 - lib/yamd/mangafox.rb
 - lib/yamd/mangahere.rb
 homepage: http://rubygems.org/gems/yamd
 licenses:
-- Public domain
+- Unlicense
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -113,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: YAMD (Yet Another Manga Downloader) - A lazy interface for writting manga