bookshark 1.0.0.alpha.2

data/lib/bookshark/crawlers/book_crawler.rb

@@ -0,0 +1,55 @@
+require 'rubygems'
+require 'nokogiri'
+require 'open-uri'
+require 'fileutils'
+
+require File.expand_path(File.join(File.dirname(__FILE__), '../extractors', 'base'))
+# page = Nokogiri::HTML(open("raw_html_pages/book_45454.html"))   
+# puts page.class   # => Nokogiri::HTML::Document
+# puts page
+
+FOLDER = 'html_book_pages'
+BASE_URL = 'http://www.biblionet.gr/book/'
+EXTENSION = '.html'
+
+301000.step(400000, 1000) do |last|  
+  # saved_pages = 0
+  # empty_pages = 0
+
+  first = last - 1000 + 1
+  subfolder = (last/1000 - 1).to_s
+  path = "#{FOLDER}/#{subfolder}/"
+
+  # Create a new directory (does nothing if directory exists)
+  FileUtils.mkdir_p path
+
+  first.upto(last) do |id|
+    file_to_save = "#{path}book_#{id}#{EXTENSION}"
+    url_to_download = "#{BASE_URL}#{id}/"
+
+    downloader = Biblionet::Core::Base.new(url_to_download)
+    downloader.save_page(file_to_save) unless downloader.page.nil?    
+
+    # open(url_to_parse) do |uri|
+    #   puts "Parsing page: #{url_to_parse}"
+    #   page = uri.read.gsub(/\s+/, " ")
+    #   # doc = Nokogiri::HTML(page)
+    #   # body = doc.at('title').inner_html
+    #   # puts body
+    #   if page.include? "</body>"
+    #     puts "Saving page: #{file_to_save}"
+    #     open(file_to_save, "w") do |file|
+    #       file.write(page)
+    #     end        
+    #     saved_pages += 1
+    #   else
+    #     puts "Page #{file_to_save} seems to be empty..."
+    #     empty_pages += 1
+    #   end
+    # end
+  end
+
+  # puts "Saved Pages: #{saved_pages}"
+  # puts "Empty Pages: #{empty_pages}"
+
+end

data/lib/bookshark/crawlers/category_crawler.rb

@@ -0,0 +1,55 @@
+require 'rubygems'
+require 'nokogiri'
+require 'open-uri'
+require 'fileutils'
+
+require File.expand_path(File.join(File.dirname(__FILE__), '../extractors', 'base'))
+# page = Nokogiri::HTML(open("raw_html_pages/book_45454.html"))   
+# puts page.class   # => Nokogiri::HTML::Document
+# puts page
+
+FOLDER = 'html_dcc_pages'
+BASE_URL = 'http://www.biblionet.gr/index/'
+EXTENSION = '.html'
+
+1000.step(6000, 1000) do |last|  
+  # saved_pages = 0
+  # empty_pages = 0
+
+  first = last - 1000 + 1
+  subfolder = (last/1000 - 1).to_s
+  path = "#{FOLDER}/#{subfolder}/"
+
+  # Create a new directory (does nothing if directory exists)
+  FileUtils.mkdir_p path
+
+  first.upto(last) do |id|
+    file_to_save = "#{path}dcc_#{id}#{EXTENSION}"
+    url_to_download = "#{BASE_URL}#{id}/"
+
+    downloader = Biblionet::Core::Base.new(url_to_download)
+    downloader.save_page(file_to_save) unless downloader.page.nil? 
+
+    # open(url_to_parse) do |uri|
+    #   puts "Parsing page: #{url_to_parse}"
+    #   page = uri.read.gsub(/\s+/, " ")
+    #   # doc = Nokogiri::HTML(page)
+    #   # body = doc.at('title').inner_html
+    #   # puts body
+    #   if page.include? "</body>"
+    #     puts "Saving page: #{file_to_save}"
+    #     open(file_to_save, "w") do |file|
+    #       file.write(page)
+    #     end        
+    #     saved_pages += 1
+    #   else
+    #     puts "Page #{file_to_save} seems to be empty..."
+    #     empty_pages += 1
+    #   end
+    # end
+  end
+
+  # puts "Saved Pages: #{saved_pages}"
+  # puts "Empty Pages: #{empty_pages}"
+
+end

data/lib/bookshark/crawlers/publisher_crawler.rb

@@ -0,0 +1,35 @@
+require_relative 'base'
+
+module Biblionet
+  module Crawlers
+
+    class PublisherCrawler < Base
+      def initialize(options = {})
+        options[:folder]    ||= 'lib/bookshark/storage/html_publisher_pages'
+        options[:base_url]  ||= 'http://www.biblionet.gr/com/'
+        options[:page_type] ||= 'publisher'
+        options[:extension] ||= '.html'
+        options[:start]     ||= 1
+        options[:finish]    ||= 800
+        options[:step]      ||= 100        
+        super(options)
+      end
+
+      def crawl_and_save 
+        downloader = Extractors::Base.new
+
+        spider do |url_to_download, file_to_save|                   
+          downloader.load_page(url_to_download)
+
+          # Create a new directory (does nothing if directory exists) 
+          path = File.dirname(file_to_save)
+          FileUtils.mkdir_p path unless File.directory?(path)
+
+          downloader.save_page(file_to_save) unless downloader.page.nil? or downloader.page.length < 1024
+
+        end
+      end
+    end
+
+  end
+end

data/lib/bookshark/extractors/author_extractor.rb

@@ -0,0 +1,116 @@
+require_relative 'base'
+
+module Biblionet
+  module Extractors
+    
+    class AuthorExtractor < Base      
+      attr_reader :author
+
+      def initialize(uri=nil)
+        super(uri)        
+        extract_author unless uri.nil? or @page.nil?        
+      end
+
+
+      def load_and_extract_author(uri=nil)
+        load_page(uri)
+        extract_author unless uri.nil? or @page.nil?
+      end   
+
+      # def to_json_pretty
+      #    JSON.pretty_generate(@author) unless @author.nil?
+      # end  
+
+      def extract_author(biblionet_id=@biblionet_id, author_page=@page)
+        puts "Extracting author: #{biblionet_id}"
+        page = AuthorDataExtractor.new(author_page)
+        
+        identity = split_name(page.fullname)
+
+        author_hash = {}
+        if present?(identity[:lastname]) and present?(identity[:firstname]) 
+          author_hash[:name] = identity[:lastname] + ', ' + identity[:firstname] 
+        elsif 
+          author_hash[:name] = identity[:lastname]
+        end      
+        author_hash[:firstname] = identity[:firstname]
+        author_hash[:lastname] = identity[:lastname]
+        author_hash[:lifetime] = identity[:lifetime]
+        author_hash[:image] = page.image
+        author_hash[:bio] = page.bio
+        author_hash[:award] = page.awards
+        author_hash[:b_id] = biblionet_id
+
+        # puts JSON.pretty_generate(author_hash)
+
+        return @author = author_hash
+      end
+
+      def split_name(fullname)
+        #mathes digits-digits or digits- in text like: Tolkien, John Ronald Reuel, 1892-1973
+        years_re = /\d+-\d*/
+
+        parts = fullname.split(',').map(&:strip)  
+
+        identity = {}
+        identity[:lastname] = parts[0]
+        
+        if parts.length == 2
+          if parts[1] =~ years_re
+            identity[:lifetime] = parts[1]
+          else
+            identity[:firstname] = parts[1]
+          end
+        elsif parts.length == 3
+          identity[:firstname] = parts[1]
+          identity[:lifetime] = parts[2]
+        end
+
+        return identity
+      
+      end
+
+    end
+
+    class AuthorDataExtractor
+      attr_reader :nodeset
+
+      def initialize(document)
+        # No need to operate on whole page. Just on part containing the content.
+        content_re = /<!-- CONTENT START -->.*<!-- CONTENT END -->/m
+        if (content_re.match(document)).nil?
+          puts document
+        end
+        content = content_re.match(document)[0]
+
+        @nodeset = Nokogiri::HTML(content)        
+      end  
+
+      def fullname
+        @nodeset.css('h1.page_title').text
+      end
+
+      def bio
+        @nodeset.css('//p[align="justify"]').text
+      end
+
+      def image
+        img_node  = @nodeset.xpath("//img[@src[contains(.,'/persons/')]][1]")                                                   
+        img       = (img_node.nil? or img_node.empty?) ? nil : BASE_URL+(img_node.first)['src']                             
+        return img         
+      end
+
+      def awards
+        awards = []        
+        @nodeset.xpath("//a[@class='booklink' and @href[contains(.,'page=showaward') ]]").each do |item|
+          award = {name: item.text, year: item.next_sibling.text.strip.gsub(/[^\d]/, '')}          
+          awards << award
+        end
+
+        return awards
+      end
+
+    end  
+
+  end
+end

data/lib/bookshark/extractors/base.rb

@@ -0,0 +1,187 @@
+require 'rubygems'
+require 'open-uri'
+require 'fileutils'
+require 'nokogiri'
+require 'json'
+require 'logger'
+require 'pp'
+require 'htmlentities'
+
+require File.expand_path(File.join(File.dirname(__FILE__), '../storage', 'file_manager'))
+
+module Biblionet
+  module Extractors
+    
+    BASE_URL = "http://www.biblionet.gr"
+
+    class Base
+      include FileManager
+
+      attr_reader :filepath, :url, :biblionet_id, :page
+
+      # Initializes the Base class. Without arguments nothing happens. Otherwise loads a page by url or file.
+      #
+      # ==== Attributes
+      #
+      # * +uri+ - It can be a url or a path/to/file.ext on local storage.
+      # 
+      def initialize(uri=nil)          
+        load_page(uri)
+      end
+
+      # Loads a page from the web or from local file storage depending on passed argument.
+      #
+      # ==== Attributes
+      #
+      # * +uri+ - It can be a url(starting with http/https) or a path/to/file.ext on local storage.
+      # 
+      def load_page(uri=nil)      
+        if uri.match(/\A#{URI::regexp(['http', 'https'])}\z/)        
+          load_page_from_url(uri)
+        else                
+          load_page_from_file(uri)
+        end unless uri.nil?
+      end
+
+      # Downloads a page from the web.
+      #
+      # ==== Attributes
+      #
+      # * +url+ - The url of webpage to download.
+      # 
+      def load_page_from_url(url)
+        begin
+          @url = url
+          @biblionet_id = url[/\d+(?!.*\d+)/] unless url.nil? # id is expected to be the last number.
+
+          pp "Downloading page: #{url}"
+          open(url, :content_length_proc => lambda do |content_length|
+            raise EmptyPageError.new(url, content_length) unless content_length.nil? or content_length > 1024
+          end) do |f|        
+            # pp f.status == ["200", "OK"] ? "success: #{f.status}" : f.status            
+            # pp  f.meta
+            # pp "Content-Type: " + f.content_type
+            # pp "Content-Size: " + (f.meta)["content-length"]
+            # pp "last modified" + f.last_modified.to_s + is_empty = (f.last_modified.nil?) ? 'Empty' : 'Not Empty' 
+
+            @page = f.read.gsub(/\s+/, " ")
+          end
+        rescue Errno::ENOENT => e
+          pp "Page: #{url} NOT FOUND."
+          pp e
+        rescue EmptyPageError => e
+          pp "Page: #{url} is EMPTY."
+          pp e        
+          @page = nil
+        rescue OpenURI::HTTPError => e
+          pp e
+          pp e.io.status          
+        rescue StandardError => e          
+          pp "Generic error #{e.class}. Will wait for 2 minutes and then try again."
+          pp e        
+          sleep(120)
+          retry        
+        end
+      end
+
+      # Reads a page from the local file system.
+      #
+      # ==== Attributes
+      #
+      # * +filepath+ - The path to target file which will be read.
+      # 
+      def load_page_from_file(filepath)    
+        begin        
+          @filepath = filepath
+          @biblionet_id = filepath[/\d+(?!.*\d+)/] unless filepath.nil?
+          @page = open(filepath).read  
+        rescue StandardError => e
+          puts e
+        end     
+      end
+
+      # Attr writer method. Changes instance variables url, page and loads a new page by calling load_page_from_url
+      #
+      # ==== Attributes
+      #
+      # * +url+ - The new value of url instance var.
+      # 
+      def url=(url)
+        load_page_from_url(url)
+      end 
+
+      # Attr writer method. Changes instance variables filepath, page and loads a new page by calling load_page_from_filename
+      #
+      # ==== Attributes
+      #
+      # * +filepath+ - The path to target file which will be read.
+      # 
+      def filepath=(filepath)
+        load_page_from_file(filepath)
+      end  
+
+      # Saves page to file.
+      #
+      # ==== Attributes
+      #
+      # * +path+ - The path to file(including filename) where content will be saved.
+      #
+      def save_page(path)
+        save_to(path, @page)
+        pp "Saving page: #{path}"
+      end
+
+      # Decodes text with escaped html entities and returns the decoded text.
+      #
+      # ==== Params:
+      #
+      # +encoded_text+:: the text which contains encoded entities
+      #
+      def decode_text(encoded_text)
+        # encoded_text = File.read(encoded_file_path)
+        coder = HTMLEntities.new
+        coder.decode(encoded_text)
+      end
+
+      def present?(value)
+        return (not value.nil? and not value.empty?) ? true : false
+      end        
+
+    end
+
+    # Raised when a page is considered empty.
+    #
+    class EmptyPageError < StandardError
+      attr_reader :url, :content_length
+
+      def initialize(url, content_length)
+        @url = url
+        @content_length = content_length
+
+        msg = "Page: #{url} is only #{content_length} bytes, so it is considered EMPTY."
+        super(msg)
+      end    
+    end
+
+    # Raised when something unexpected or in wrong format is parsed.
+    #
+    class NoIdeaWhatThisIsError < StandardError      
+      attr_reader :biblionet_id, :the_unexpected
+
+      def initialize(biblionet_id, the_unexpected)
+        @biblionet_id = biblionet_id
+        @the_unexpected = the_unexpected
+        
+        msg = "We have no idea what this: #{the_unexpected} is. At book #{biblionet_id}"
+        super(msg)
+      end    
+    end    
+
+  end
+end
+
+# page = Parser::ParserBase.parse("http://www.biblionet.gr/book/300525")
+# pp page.inspect
+# Parser::ParserBase.parse("http://www.biblionet.gr/book/195243421")
+# Parser::ParserBase.parse("dsdfds.com")
+# Parser::ParserBase.save_to("book_195221.html", page)