manga-crawler 0.0.2 → 0.1.1

data/lib/manga-crawler.rb

@@ -1,6 +1,4 @@
 require "manga-crawler/version"
 require "manga-crawler/crawler"
-
-module MangaCrawler
-  # Your code goes here...
-end
+require "website/parameters"
+require "website/page"

data/lib/manga-crawler/crawler.rb

@@ -4,25 +4,18 @@ require 'open-uri'
 module MangaCrawler
   class Crawler
 
-    # Returns an array of pairs. The first position contains the 
-    # manga name and the second position the manga link
-    # Params:
-    # +index_link+:: string with the url containing the index of all mangas
-    # +css_path+:: string of a css path format of the links you want to collect
-    # +css_pagination+:: string with the css path to the next page link
-    # +html_field+:: simbol of the field that has the link
-    def get_mangas index_link, css_path, css_pagination, html_field
+    def get_mangas index_website
 
       result = Array.new
 
-      html_index = Nokogiri::HTML(open(index_link))
+      html_index = Nokogiri::HTML(open(index_website.params.current_url))
 
       #find all content that matches with the css_path
-      links = html_index.css(css_path)
+      links = html_index.css(index_website.params.css_path)
 
       #find all content from the anchor nodes found in last search
       links.each do |anchor|
-        result.push([anchor.content, anchor[html_field]])
+        result.push([anchor.content, anchor[index_website.params.html_field]])
       end
 
       #TODO
@@ -32,72 +25,51 @@ module MangaCrawler
       return result
     end
 
-    # Returns the chapters information of a manga. It uses the same
-    # logic of get_mangas.
-    # Params:
-    # +manga_link+:: string with the url containing the manga_link
-    # +css_path+:: string of the css path format of the links you want to collect
-    # +css_pagination+:: string with the css path to the next page link
-    # +html_field+:: simbol of the field that has the link 
-    def get_chapters manga_link, css_path, css_pagination, html_field
+    def get_chapters manga_website
       #TODO
       #uses the same logic of get_mangas
-      return get_mangas manga_link, css_path, css_pagination, html_field
+      return get_mangas manga_website
     end
 
-    # Returns the direct links of all pages from a specific chapter. It uses two
-    # methods: 'get_pages_links_from_chapter' and 'get_image_from_page'.
-    # Params:
-    # +chapter_link+:: string with the chapter
-    # +css_pages_path+:: string with the CSS path to the pages links
-    # +pages_html_field+:: HTML field with the page link value
-    # +css_image_path+:: CSS path to the image
-    # +image_html_field+:: HTML field with the direct's image url
-    # +url_base+:: Site´s base url
-    def get_pages chapter_link, css_pages_path, pages_html_field, css_image_path, image_html_field, url_base
+    def get_pages chapter_website, css_image_path
       
       result = Array.new
 
-      pages_links = get_pages_links_from_chapter url_base + chapter_link, css_pages_path, pages_html_field
+      pages_links = get_pages_links_from_chapter chapter_website
 
       pages_links.each do |page|
-        result.push( get_image_from_page url_base + page[1], css_image_path, image_html_field )
+        
+        current_url = chapter_website.params.base_url + page[1]
+
+        params = Website::Parameters.new(chapter_website.params.base_url, current_url, css_image_path, :src)
+
+        result.push( get_image_from_page Website::Page.new(params) )
       end
 
       return result
 
     end
 
-    # Returns all pages HTML links from a chapter
-    # Params:
-    # +chapter_link+:: Link of the chapter
-    # +css_path+:: CSS path to the block with the pages links
-    # +html_field+:: HTML field that contains the url
-    def get_pages_links_from_chapter chapter_link, css_path, html_field
+    def get_pages_links_from_chapter chapter_website
 
       result = Array.new
 
-      chapter_page = Nokogiri::HTML(open(chapter_link))
+      chapter_page = Nokogiri::HTML(open(chapter_website.params.current_url))
 
-      pages_links = chapter_page.css(css_path)
+      pages_links = chapter_page.css(chapter_website.params.css_path)
 
       pages_links.each do |option|
-        result.push([option.content, option[html_field]])
+        result.push([option.content, option[chapter_website.params.html_field]])
       end
 
       return result
     end
 
-    # Returns the image's direct url of a manga page
-    # Params:
-    # +page_link+:: HTML page who contains the image
-    # +css_path+:: CSS path to the image
-    # +html_field+:: field that contains the url
-    def get_image_from_page page_link, css_path, html_field
+    def get_image_from_page image_website
 
-      html_image = Nokogiri::HTML(open(page_link))
+      html_image = Nokogiri::HTML(open(image_website.params.current_url))
 
-      image_link = html_image.at_css(css_path)[html_field]
+      image_link = html_image.at_css(image_website.params.css_path)[image_website.params.html_field]
 
       return image_link
     end

data/lib/manga-crawler/version.rb

@@ -1,3 +1,3 @@
 module MangaCrawler
-  VERSION = "0.0.2"
+  VERSION = "0.1.1"
 end

data/lib/website/page.rb

@@ -0,0 +1,14 @@
+module Website
+  class Page
+
+    attr_reader :params
+
+    def initialize(params)
+      if params.instance_of?(Website::Parameters)
+        @params = params
+      else
+        raise "#{params} must be an instance of Website::Parameters"
+      end
+    end
+  end
+end

data/lib/website/parameters.rb

@@ -0,0 +1,13 @@
+module Website
+  class Parameters
+
+    attr_reader :base_url, :current_url, :css_path, :html_field
+
+    def initialize(base_url, current_url, css_path, html_field)
+      @base_url = base_url
+      @current_url = current_url
+      @css_path = css_path
+      @html_field = html_field
+    end
+  end
+end

data/manga-crawler.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["kimobr@gmail.com"]
   spec.description   = %q{ A gem that collects mangas from websites}
   spec.summary       = %q{ Retrieve basic manga information }
-  spec.homepage      = ""
+  spec.homepage      = "https://github.com/thiagokimo/manga-crawler"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files`.split($/)

data/test/fixtures/Bleach/bleach.html

@@ -0,0 +1,83 @@
+<html>
+  <head><title>Bleach</title></head>
+
+  <body>
+    <div class="c_h1">Information</div>
+
+    <div>
+    <table width="100%" border="0" cellspacing="0" cellpadding="0">
+    <tr>
+    <td valign="top" style="background: #f0f0f0; padding: 5px; border-right: 1px solid #fff;border-bottom-left-radius: 5px;">
+      <div><a class="olol" title="Bleach" href="#"><img class="a_img" src="../images/default_bleach.jpg" alt="Bleach"/></a></div>
+      
+    </td>
+    <td width="100%" style="background: #e0e0e0; border-bottom-right-radius: 5px;" valign="top">
+    <table width="100%" cellspacing="0" cellpadding="0" border="0">
+      <tr class="c_h2">
+        <td width="80" valign="top"><b>Title(s):</b></td>
+        <td>
+        Bleach<br/>
+            </td>
+      </tr>
+      <tr class="c_h2b">
+        <td valign="top"><b>Creator(s):</b></td>
+        <td>
+              <a href="#">Lorem Ipsum</a> 
+            </td>
+      </tr>
+      <tr class="c_h2">
+        <td valign="top"><b>Genres:</b></td>
+        <td>
+              <a href="#">Action</a>, 
+              <a href="#">Adventure</a>, 
+              <a href="#">Anime</a>, 
+              <a href="#">Comedy</a>, 
+              <a href="#">Drama</a>, 
+              <a href="#">Fantasy</a>, 
+              <a href="#">Shounen</a>, 
+              <a href="#">Supernatural</a> 
+            </td>
+      </tr>
+        <tr class="c_h2b">
+        <td valign="top"><b>Start Date:</b></td>
+        <td>
+          2001    </td>
+      </tr>
+      <tr class="c_h2">
+        <td valign="top"><b>Status:</b></td>
+        <td>
+          <span style="color: ">Ongoing</span>    </td>
+      </tr>
+        <tr class="c_h2b">
+        <td valign="top" style="border-bottom: 0;"><b>Summary:</b></td>
+        <td style="border-bottom: 0;">
+        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+        tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+        quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+        consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+        cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+        proident, sunt in culpa qui officia deserunt mollit anim id est laborum.<br />
+      </tr>
+            </table>
+    </td>
+    </tr>
+    </table>
+    </div>
+
+    <div class="c_h1" style="margin-top:1px;">Chapters</div>
+
+    <div class="episode c_h2">
+        <div><a class="download-link" href="chapters/1/1.html">Bleach <em>chapter</em> <strong>1</strong></a> </div>
+        <div class="clear"></div>
+    </div>
+    <div class="episode c_h2b">
+        <div><a class="download-link" href="chapters/2/1.html">Bleach <em>chapter</em> <strong>2</strong></a> </div>
+        <div class="clear"></div>
+    </div>
+    <div class="episode c_h2">
+        <div><a class="download-link" href="chapters/3/1.html">Bleach <em>chapter</em> <strong>3</strong></a> </div>
+        <div class="clear"></div>
+    </div>
+
+  </body>
+</html>

data/test/fixtures/Bleach/chapters/1/1.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="2.html">
+    <img id="img" src="mushroom_risotto.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/Bleach/chapters/1/2.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="../2/1.html">
+    <img id="img" src="vegetable_curry.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/Bleach/chapters/2/1.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="2.html">
+    <img id="img" src="angry_birds_cake.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/Bleach/chapters/2/2.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="../3/1.html">
+    <img id="img" src="thai_shrimp_cake.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/Bleach/chapters/3/1.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="../3/2.html">
+    <img id="img" src="instant_noodle_with_egg.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/Bleach/chapters/3/2.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="#">
+    <img id="img" src="noodle_with_bbq_pork.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/OnePiece/chapters/1/1.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="2.html">
+    <img id="img" src="full_breakfast.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/OnePiece/chapters/1/2.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="../2/1.html">
+    <img id="img" src="white_chocolate_donut.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/OnePiece/chapters/2/1.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="2.html">
+    <img id="img" src="green_tea.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/OnePiece/chapters/2/2.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="../3/1.html">
+    <img id="img" src="ham_and_egg_sandwich.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/OnePiece/chapters/3/1.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="../3/2.html">
+    <img id="img" src="starbucks_coffee.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/OnePiece/chapters/3/2.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="#">
+    <img id="img" src="japanese_noodle_with_pork.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/OnePiece/one_piece.html

@@ -0,0 +1,85 @@
+<html>
+  <head>
+    <title>One Piece</title>
+  </head>
+  <body>
+
+    <div class="c_h1">Information</div>
+
+    <div>
+    <table width="100%" border="0" cellspacing="0" cellpadding="0">
+    <tr>
+    <td valign="top" style="background: #f0f0f0; padding: 5px; border-right: 1px solid #fff;border-bottom-left-radius: 5px;">
+      <div><a class="olol" title="One Piece" href="#"><img class="a_img" src="../images/default_onepiece.jpg" alt="One Piece"/></a></div>
+    </td>
+    <td width="100%" style="background: #e0e0e0; border-bottom-right-radius: 5px;" valign="top">
+    <table width="100%" cellspacing="0" cellpadding="0" border="0">
+      <tr class="c_h2">
+        <td width="80" valign="top"><b>Title(s):</b></td>
+        <td>
+        One Piece<br/>
+            </td>
+      </tr>
+      <tr class="c_h2b">
+        <td valign="top"><b>Creator(s):</b></td>
+        <td>
+              <a href="/manga/search?c=321">Lorem Ipsum</a> 
+            </td>
+      </tr>
+      <tr class="c_h2">
+        <td valign="top"><b>Genres:</b></td>
+        <td>
+              <a href="#">Action</a>, 
+              <a href="#">Adventure</a>, 
+              <a href="#">Anime</a>, 
+              <a href="#">Comedy</a>, 
+              <a href="#">Drama</a>, 
+              <a href="#">Fantasy</a>, 
+              <a href="#">Shounen</a> 
+            </td>
+      </tr>
+        <tr class="c_h2b">
+        <td valign="top"><b>Start Date:</b></td>
+        <td>
+          1999    </td>
+      </tr>
+      <tr class="c_h2">
+        <td valign="top"><b>Status:</b></td>
+        <td>
+          <span style="color: ">Ongoing</span>    </td>
+      </tr>
+        <tr class="c_h2b">
+        <td valign="top" style="border-bottom: 0;"><b>Summary:</b></td>
+        <td style="border-bottom: 0;">
+        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+        tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+        quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+        consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+        cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+        proident, sunt in culpa qui officia deserunt mollit anim id est laborum.<br />
+    <br />
+      </td>
+      </tr>
+    </table>
+    </td>
+    </tr>
+    </table>
+    </div>
+
+    <div class="c_h1" style="margin-top:1px;">Chapters</div>
+
+    <div class="episode c_h2">
+        <div><a class="download-link" href="chapters/1/1.html">One Piece <em>chapter</em> <strong>1</strong></a> </div>
+        <div class="clear"></div>
+    </div>
+    <div class="episode c_h2b">
+        <div><a class="download-link" href="chapters/2/1.html">One Piece <em>chapter</em> <strong>2</strong></a> </div>
+        <div class="clear"></div>
+    </div>
+    <div class="episode c_h2">
+        <div><a class="download-link" href="chapters/3/1.html">One Piece <em>chapter</em> <strong>3</strong></a> </div>
+        <div class="clear"></div>
+    </div>
+
+  </body>
+</html>

data/test/fixtures/index.html

@@ -0,0 +1,14 @@
+<html>
+  <head>
+    <title>A sample manga website</title>
+  </head>
+  <body>
+    <div class="simple_div">
+      <ul class="simple_div">
+        <li><a href="Naruto/naruto.html">Naruto</a></li>
+        <li><a href="Bleach/bleach.html">Bleach</a></li>
+        <li><a href="OnePiece/one_piece.html">One Piece</a></li>
+      </ul>
+    </div>
+  </body>
+</html>

data/test/fixtures/naruto/chapters/1/1.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="2.html">
+    <img id="img" src="duck.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/naruto/chapters/1/2.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="../2/1.html">
+    <img id="img" src="hamburger.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/naruto/chapters/2/1.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="2.html">
+    <img id="img" src="egg_benedict.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/naruto/chapters/2/2.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="../3/1.html">
+    <img id="img" src="ham_and_cheese_panini.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/naruto/chapters/3/1.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="../3/2.html">
+    <img id="img" src="starbucks_coffee.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/naruto/chapters/3/2.html

@@ -0,0 +1,9 @@
+<select id="page_switch" class="bb_drop2" style="border: 1px solid #0b7eb5;">
+  <option selected value="1.html">1</option>
+  <option  value="2.html">2</option></select> of <strong>2</strong>
+
+<div id="imgholder">
+  <a href="#">
+    <img id="img" src="japanese_noodle_with_pork.jpg" alt="An image" name="img"/>
+  </a>
+</div>

data/test/fixtures/naruto/naruto.html

@@ -0,0 +1,85 @@
+<html>
+  <head>
+    <title>Naruto</title>
+  </head>
+  <body>
+
+    <div class="c_h1">Information</div>
+
+    <div>
+    <table width="100%" border="0" cellspacing="0" cellpadding="0">
+    <tr>
+    <td valign="top" style="background: #f0f0f0; padding: 5px; border-right: 1px solid #fff;border-bottom-left-radius: 5px;">
+      <div><a class="olol" title="Naruto" href="#"><img class="a_img" src="../images/default_naruto.jpg" alt="Naruto"/></a></div>
+    </td>
+    <td width="100%" style="background: #e0e0e0; border-bottom-right-radius: 5px;" valign="top">
+    <table width="100%" cellspacing="0" cellpadding="0" border="0">
+      <tr class="c_h2">
+        <td width="80" valign="top"><b>Title(s):</b></td>
+        <td>
+        Naruto<br/>
+            </td>
+      </tr>
+      <tr class="c_h2b">
+        <td valign="top"><b>Creator(s):</b></td>
+        <td>
+              <a href="#">Lorem Ipsum</a> 
+            </td>
+      </tr>
+      <tr class="c_h2">
+        <td valign="top"><b>Genres:</b></td>
+        <td>
+              <a href="#">Action</a>, 
+              <a href="#">Adventure</a>, 
+              <a href="#">Anime</a>, 
+              <a href="#">Comedy</a>, 
+              <a href="#">Drama</a>, 
+              <a href="#">Fantasy</a>, 
+              <a href="#">Shounen</a> 
+            </td>
+      </tr>
+        <tr class="c_h2b">
+        <td valign="top"><b>Start Date:</b></td>
+        <td>
+          1999    </td>
+      </tr>
+      <tr class="c_h2">
+        <td valign="top"><b>Status:</b></td>
+        <td>
+          <span style="color: ">Ongoing</span>    </td>
+      </tr>
+        <tr class="c_h2b">
+        <td valign="top" style="border-bottom: 0;"><b>Summary:</b></td>
+        <td style="border-bottom: 0;">
+        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+        tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+        quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+        consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+        cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+        proident, sunt in culpa qui officia deserunt mollit anim id est laborum.<br />
+    <br />
+      </td>
+      </tr>
+    </table>
+    </td>
+    </tr>
+    </table>
+    </div>
+
+    <div class="c_h1" style="margin-top:1px;">Chapters</div>
+
+    <div class="episode c_h2">
+        <div><a class="download-link" href="chapters/1/1.html">Naruto <em>chapter</em> <strong>1</strong></a> </div>
+        <div class="clear"></div>
+    </div>
+    <div class="episode c_h2b">
+        <div><a class="download-link" href="chapters/2/1.html">Naruto <em>chapter</em> <strong>2</strong></a> </div>
+        <div class="clear"></div>
+    </div>
+    <div class="episode c_h2">
+        <div><a class="download-link" href="chapters/3/1.html">Naruto <em>chapter</em> <strong>3</strong></a> </div>
+        <div class="clear"></div>
+    </div>
+
+  </body>
+</html>

data/test/lib/manga-crawler/crawler_test.rb

@@ -3,55 +3,77 @@ require "test_helper"
 describe MangaCrawler::Crawler do
 
   crawler = MangaCrawler::Crawler.new
+  base_url = "localhost"
 
   it "must retrieve mangas" do
-    
-    sample_index = File.open("test/samples/index-page.html")
+
+    sample_index_page = File.open("test/fixtures/index.html")
     css_path = "a"
     html_field = :href
+
+    params = Website::Parameters.new(base_url, sample_index_page, css_path, html_field)
+    index_page = Website::Page.new(params)
     
-    mangas = crawler.get_mangas sample_index, css_path, nil, html_field
+    mangas = crawler.get_mangas index_page
 
-    mangas.must_equal [ ["Naruto", "/first-manga"], 
-                        ["Bleach", "/second-manga"], 
-                        ["One Piece", "/third-manga"] ]
+    mangas.must_equal [ ["Naruto", "Naruto/naruto.html"], 
+                        ["Bleach", "Bleach/bleach.html"], 
+                        ["One Piece", "OnePiece/one_piece.html"] ]
   end
 
   it "must retrieve chapters" do
     
-    sample_manga_page = File.open("test/samples/manga-page.html")
-    css_path = "a"
+    sample_manga_page = File.open("test/fixtures/Bleach/bleach.html")
+    css_path = ".download-link"
     html_field = :href
 
-    chapters = crawler.get_chapters sample_manga_page, css_path, nil, html_field
+    params = Website::Parameters.new(base_url, sample_manga_page, css_path, html_field)
+    manga_page = Website::Page.new(params)
+
+    chapters = crawler.get_chapters manga_page
 
-    chapters.must_equal [ ["Chapter 1", "/first-manga/1"], 
-                          ["Chapter 2", "/second-manga/2"], 
-                          ["Chapter 3", "/third-manga/3"] ]
+    chapters.must_equal [ ["Bleach chapter 1", "chapters/1/1.html"], 
+                          ["Bleach chapter 2", "chapters/2/1.html"], 
+                          ["Bleach chapter 3", "chapters/3/1.html"] ]
   end
 
   it "must retrieve a direct image link from a page" do
     
-    sample_image_page = File.open("test/samples/image-page.html")
+    sample_image_page = File.open("test/fixtures/naruto/chapters/1/1.html")
     css_path = "#img"
     html_field = :src
 
-    image = crawler.get_image_from_page sample_image_page, css_path, html_field
+    params = Website::Parameters.new(base_url, sample_image_page, css_path, html_field)
+    image_page = Website::Page.new(params)
+
+    image = crawler.get_image_from_page image_page
 
-    image.must_equal "image.jpg"
+    image.must_equal "duck.jpg"
   end
 
   it "must retrieve all pages links from a chapter" do
 
-    sample_image_page = File.open("test/samples/image-page.html")
-    css_path = "#pageMenu option"
+    sample_image_page = File.open("test/fixtures/OnePiece/chapters/2/1.html")
+    css_path = "#page_switch option"
     html_field = :value
+
+    params = Website::Parameters.new(base_url, sample_image_page, css_path, html_field)
+    pages = Website::Page.new(params)
     
-    pages_links = crawler.get_pages_links_from_chapter sample_image_page, css_path, html_field
+    pages_links = crawler.get_pages_links_from_chapter pages
+
+    pages_links.must_equal [ ["1", "1.html"], ["2", "2.html"] ]
+
+  end
+
+  it "must collect all pages from a given chapter" do
+    link = "https://starkana.me/manga/0/A_Princess_and_a_Bum_(Manhwa)/chapter/7"
+    css_pages_path = "#page_switch option"
+    pages_html_field = :value
 
-    pages_links.must_equal [ ["1", "/first-manga/1/1"],
-                             ["2", "/first-manga/1/2"], 
-                             ["3", "/first-manga/1/3"] ]
+    params = Website::Parameters.new("https://starkana.me", link, css_pages_path, pages_html_field)
+    chapter_page = Website::Page.new(params)
 
+    crawler.get_pages chapter_page, "#pic img"
   end
 end

data/test/lib/website/page_test.rb

@@ -0,0 +1,17 @@
+require "test_helper"
+
+describe Website::Page do
+
+  params = Website::Parameters.new("","","","")
+  page = Website::Page.new(params)
+
+  it "must have parameters" do
+    page.params.wont_be_nil
+    lambda { Website::Page.new(nil) }.must_raise(RuntimeError)
+  end
+
+  it "params must be an instance of Website::Parameters" do
+    lambda { Website::Page.new("invalid params") }.must_raise(RuntimeError)
+    lambda { Website::Page.new(params) }.must_be_silent
+  end
+end

data/test/lib/website/parameters_test.rb

@@ -0,0 +1,30 @@
+require "test_helper"
+
+describe Website::Parameters do
+
+  describe "default attributes" do
+
+    base_url = "lorem.ipsum.dolor"
+    current_url = "/lorem"
+    css_path = "#lorem"
+    html_field = "src"
+    params = Website::Parameters.new(base_url, current_url, css_path, html_field)
+    
+    it "must have a base url" do
+      params.base_url.wont_be_nil
+    end
+
+    it "must have a current url" do
+      params.current_url.wont_be_nil
+    end
+
+    it "must have a css path" do
+      params.css_path.wont_be_nil
+    end
+
+    it "must have a html attribute" do
+      params.html_field.wont_be_nil
+    end
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: manga-crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-16 00:00:00.000000000 Z
+date: 2013-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -74,15 +74,58 @@ files:
 - lib/manga-crawler.rb
 - lib/manga-crawler/crawler.rb
 - lib/manga-crawler/version.rb
+- lib/website/page.rb
+- lib/website/parameters.rb
 - manga-crawler.gemspec
+- test/fixtures/Bleach/bleach.html
+- test/fixtures/Bleach/chapters/1/1.html
+- test/fixtures/Bleach/chapters/1/2.html
+- test/fixtures/Bleach/chapters/1/mushroom_risotto.jpg
+- test/fixtures/Bleach/chapters/1/vegetable_curry.jpg
+- test/fixtures/Bleach/chapters/2/1.html
+- test/fixtures/Bleach/chapters/2/2.html
+- test/fixtures/Bleach/chapters/2/angry_birds_cake.jpg
+- test/fixtures/Bleach/chapters/2/thai_shrimp_cake.jpg
+- test/fixtures/Bleach/chapters/3/1.html
+- test/fixtures/Bleach/chapters/3/2.html
+- test/fixtures/Bleach/chapters/3/instant_noodle_with_egg.jpg
+- test/fixtures/Bleach/chapters/3/noodle_with_bbq_pork.jpg
+- test/fixtures/OnePiece/chapters/1/1.html
+- test/fixtures/OnePiece/chapters/1/2.html
+- test/fixtures/OnePiece/chapters/1/full_breakfast.jpg
+- test/fixtures/OnePiece/chapters/1/white_chocolate_donut.jpg
+- test/fixtures/OnePiece/chapters/2/1.html
+- test/fixtures/OnePiece/chapters/2/2.html
+- test/fixtures/OnePiece/chapters/2/green_tea.jpg
+- test/fixtures/OnePiece/chapters/2/ham_and_egg_sandwich.jpg
+- test/fixtures/OnePiece/chapters/3/1.html
+- test/fixtures/OnePiece/chapters/3/2.html
+- test/fixtures/OnePiece/chapters/3/japanese_noodle_with_pork.jpg
+- test/fixtures/OnePiece/chapters/3/starbucks_coffee.jpg
+- test/fixtures/OnePiece/one_piece.html
+- test/fixtures/images/default_bleach.jpg
+- test/fixtures/images/default_naruto.jpg
+- test/fixtures/images/default_onepiece.jpg
+- test/fixtures/index.html
+- test/fixtures/naruto/chapters/1/1.html
+- test/fixtures/naruto/chapters/1/2.html
+- test/fixtures/naruto/chapters/1/duck.jpg
+- test/fixtures/naruto/chapters/1/hamburger.jpg
+- test/fixtures/naruto/chapters/2/1.html
+- test/fixtures/naruto/chapters/2/2.html
+- test/fixtures/naruto/chapters/2/egg_benedict.jpg
+- test/fixtures/naruto/chapters/2/ham_and_cheese_panini.jpg
+- test/fixtures/naruto/chapters/3/1.html
+- test/fixtures/naruto/chapters/3/2.html
+- test/fixtures/naruto/chapters/3/japanese_noodle_with_pork.jpg
+- test/fixtures/naruto/chapters/3/starbucks_coffee.jpg
+- test/fixtures/naruto/naruto.html
 - test/lib/manga-crawler/crawler_test.rb
 - test/lib/manga-crawler/version_test.rb
-- test/samples/image-page.html
-- test/samples/image.jpg
-- test/samples/index-page.html
-- test/samples/manga-page.html
+- test/lib/website/page_test.rb
+- test/lib/website/parameters_test.rb
 - test/test_helper.rb
-homepage: ''
+homepage: https://github.com/thiagokimo/manga-crawler
 licenses:
 - MIT
 post_install_message: 
@@ -108,10 +151,51 @@ signing_key:
 specification_version: 3
 summary: Retrieve basic manga information
 test_files:
+- test/fixtures/Bleach/bleach.html
+- test/fixtures/Bleach/chapters/1/1.html
+- test/fixtures/Bleach/chapters/1/2.html
+- test/fixtures/Bleach/chapters/1/mushroom_risotto.jpg
+- test/fixtures/Bleach/chapters/1/vegetable_curry.jpg
+- test/fixtures/Bleach/chapters/2/1.html
+- test/fixtures/Bleach/chapters/2/2.html
+- test/fixtures/Bleach/chapters/2/angry_birds_cake.jpg
+- test/fixtures/Bleach/chapters/2/thai_shrimp_cake.jpg
+- test/fixtures/Bleach/chapters/3/1.html
+- test/fixtures/Bleach/chapters/3/2.html
+- test/fixtures/Bleach/chapters/3/instant_noodle_with_egg.jpg
+- test/fixtures/Bleach/chapters/3/noodle_with_bbq_pork.jpg
+- test/fixtures/OnePiece/chapters/1/1.html
+- test/fixtures/OnePiece/chapters/1/2.html
+- test/fixtures/OnePiece/chapters/1/full_breakfast.jpg
+- test/fixtures/OnePiece/chapters/1/white_chocolate_donut.jpg
+- test/fixtures/OnePiece/chapters/2/1.html
+- test/fixtures/OnePiece/chapters/2/2.html
+- test/fixtures/OnePiece/chapters/2/green_tea.jpg
+- test/fixtures/OnePiece/chapters/2/ham_and_egg_sandwich.jpg
+- test/fixtures/OnePiece/chapters/3/1.html
+- test/fixtures/OnePiece/chapters/3/2.html
+- test/fixtures/OnePiece/chapters/3/japanese_noodle_with_pork.jpg
+- test/fixtures/OnePiece/chapters/3/starbucks_coffee.jpg
+- test/fixtures/OnePiece/one_piece.html
+- test/fixtures/images/default_bleach.jpg
+- test/fixtures/images/default_naruto.jpg
+- test/fixtures/images/default_onepiece.jpg
+- test/fixtures/index.html
+- test/fixtures/naruto/chapters/1/1.html
+- test/fixtures/naruto/chapters/1/2.html
+- test/fixtures/naruto/chapters/1/duck.jpg
+- test/fixtures/naruto/chapters/1/hamburger.jpg
+- test/fixtures/naruto/chapters/2/1.html
+- test/fixtures/naruto/chapters/2/2.html
+- test/fixtures/naruto/chapters/2/egg_benedict.jpg
+- test/fixtures/naruto/chapters/2/ham_and_cheese_panini.jpg
+- test/fixtures/naruto/chapters/3/1.html
+- test/fixtures/naruto/chapters/3/2.html
+- test/fixtures/naruto/chapters/3/japanese_noodle_with_pork.jpg
+- test/fixtures/naruto/chapters/3/starbucks_coffee.jpg
+- test/fixtures/naruto/naruto.html
 - test/lib/manga-crawler/crawler_test.rb
 - test/lib/manga-crawler/version_test.rb
-- test/samples/image-page.html
-- test/samples/image.jpg
-- test/samples/index-page.html
-- test/samples/manga-page.html
+- test/lib/website/page_test.rb
+- test/lib/website/parameters_test.rb
 - test/test_helper.rb

data/test/samples/image-page.html

@@ -1,21 +0,0 @@
-<html>
-  <head>
-    <title>A sample manga website</title>
-  </head>
-  <body>
-
-    <div id="selectpage">
-      <select id="pageMenu" name="pageMenu">
-        <option value="/first-manga/1/1" selected="selected">1</option>
-        <option value="/first-manga/1/2">2</option>
-        <option value="/first-manga/1/3">3</option>
-      </select>
-    </div>
-
-    <div id="imgholder">
-      <a href="/first-manga/1/2">
-        <img id="img" src="image.jpg" alt="An image" name="img"/>
-      </a>
-    </div>
-  </body>
-</html>

data/test/samples/index-page.html

@@ -1,14 +0,0 @@
-<html>
-  <head>
-    <title>A sample manga website</title>
-  </head>
-  <body>
-    <div class="simple_div">
-      <ul class="simple_div">
-        <li><a href="/first-manga">Naruto</a></li>
-        <li><a href="/second-manga">Bleach</a></li>
-        <li><a href="/third-manga">One Piece</a></li>
-      </ul>
-    </div>
-  </body>
-</html>

data/test/samples/manga-page.html

@@ -1,38 +0,0 @@
-<html>
-  <head>
-    <title>A sample manga website</title>
-  </head>
-  <body>
-    <div id="chapterlist">
-      <table id="listing">
-        
-        <tr class="table_head">
-          <th class="leftgap">Chapter Name</th>
-          <th>Date Added</th>
-        </tr>
-      
-        <tr>
-          <td>
-            <div class="chico_manga"></div>
-            <a href="/first-manga/1">Chapter 1</a> : First chapter </td>
-          <td>09/09/2011</td>
-        </tr>
-      
-        <tr>
-          <td>
-            <div class="chico_manga"></div>
-            <a href="/second-manga/2">Chapter 2</a> : Second chapter</td>
-          <td>09/09/2011</td>
-        </tr>
-
-        <tr>
-          <td>
-            <div class="chico_manga"></div>
-            <a href="/third-manga/3">Chapter 3</a> : Third chapter</td>
-          <td>09/09/2011</td>
-        </tr>
-        
-      </table>
-    </div>
-  </body>
-</html>