alexandria-book-collection-manager 0.6.9.pre1

data/lib/alexandria/book_providers/thalia.rb

@@ -0,0 +1,197 @@
+# Copyright (C) 2009 Cathal Mc Ginley
+# Copyright (C) 2014 Matijs van Zuijlen
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+# http://de.wikipedia.org/wiki/Thalia_%28Buchhandel%29
+# Thalia.de bought the Austrian book trade chain Amadeus
+
+# New Tlalia provider, taken from Palatina MetaDataSource and modified
+# for Alexandria. (21 Dec 2009)
+
+require 'net/http'
+require 'cgi'
+require 'alexandria/book_providers/web'
+
+module Alexandria
+  class BookProviders
+    class ThaliaProvider < WebsiteBasedProvider
+      include Alexandria::Logging
+
+      SITE = 'http://www.thalia.de'
+      BASE_SEARCH_URL = "#{SITE}/shop/bde_bu_hg_startseite/suche/?%s=%s" # type,term
+
+      def initialize
+        super('Thalia', 'Thalia (Germany)')
+        # no preferences for the moment
+        prefs.read
+      end
+
+      def url(book)
+        create_search_uri(SEARCH_BY_ISBN, book.isbn)
+      end
+
+      def search(criterion, type)
+        req = create_search_uri(type, criterion)
+        puts req if $DEBUG
+        html_data = transport.get_response(URI.parse(req))
+        if type == SEARCH_BY_ISBN
+          parse_result_data(html_data.body, criterion)
+        else
+          results = parse_search_result_data(html_data.body)
+          raise NoResultsError if results.empty?
+          results.map { |result| get_book_from_search_result(result) }
+        end
+      end
+
+      def create_search_uri(search_type, search_term)
+        search_type_code = {
+          SEARCH_BY_ISBN => 'sq',
+          SEARCH_BY_AUTHORS => 'sa', # Autor
+          SEARCH_BY_TITLE => 'st', # Titel
+          SEARCH_BY_KEYWORD => 'ssw' # Schlagwort
+        }[search_type] or ''
+        search_type_code = CGI.escape(search_type_code)
+        search_term_encoded = search_term
+        if search_type == SEARCH_BY_ISBN
+          # search_term_encoded = search_term.as_isbn_13
+          search_term_encoded = Library.canonicalise_isbn(search_term) # check this!
+        else
+          search_term_encoded = CGI.escape(search_term)
+        end
+        BASE_SEARCH_URL % [search_type_code, search_term_encoded]
+      end
+
+      def parse_search_result_data(html)
+        doc = html_to_doc(html)
+        book_search_results = []
+        results_divs = doc / 'div.articlePresentationSearchCH'
+        results_divs.each do |div|
+         result = {}
+         title_link = div % 'div.articleText/h2/a'
+         result[:title] = title_link.inner_html
+         result[:lookup_url] = title_link['href']
+         book_search_results << result
+       end
+        book_search_results
+      end
+
+      def data_from_label(node, label_text)
+        label_node = node % "strong[text()*='#{label_text}']"
+        if (item_node = label_node.parent)
+          data = ''
+          item_node.children.each do |n|
+            if n.text?
+              data += n.to_html
+            end
+          end
+          data.strip
+        else
+          ''
+        end
+      end
+
+      def get_book_from_search_result(result)
+        log.debug { "Fetching book from #{result[:lookup_url]}" }
+        html_data =  transport.get_response(URI.parse(result[:lookup_url]))
+        parse_result_data(html_data.body, 'noisbn', true)
+      end
+
+      def parse_result_data(html, isbn, recursing = false)
+        doc = html_to_doc(html)
+
+        results_divs = doc / 'div.articlePresentationSearchCH'
+        unless results_divs.empty?
+          if recursing
+            # already recursing, avoid doing so endlessly second time
+            # around *should* lead to a book description, not a result
+            # list
+            return
+          end
+          # ISBN-lookup results in multiple results (trying to be
+          # useful, such as for new editions e.g. 9780974514055
+          # "Programming Ruby" )
+          results = parse_search_result_data(html)
+          isbn10 = Library.canonicalise_isbn(isbn)
+          # e.g. .../dave_thomas/ISBN0-9745140-5-5/ID6017044.html
+          chosen = results.first # fallback!
+          results.each do |rslt|
+            if rslt[:lookup_url] =~ /\/ISBN(\d+[\d-]*)\//
+              if Regexp.last_match[1].gsub('-', '') == isbn10
+                chosen = rslt
+                break
+              end
+            end
+          end
+          html_data = transport.get_response(URI.parse(chosen[:lookup_url]))
+          return parse_result_data(html_data.body, isbn, true)
+        end
+
+        begin
+          if (div = doc % 'div#contentFull')
+            title_img = ((div % :h2) / :img).first
+            title = title_img['alt']
+
+            # note, the following img also has alt="von Author, Author..."
+
+            if (author_h = doc % 'h3[text()*="Mehr von"]') # "More from..." links
+              authors = []
+              author_links = author_h.parent / :a
+              author_links.each do |a|
+                if a['href'] =~ /BUCH\/sa/
+                  # 'sa' means search author, there may also be 'ssw' (search keyword) links
+                  authors << a.inner_text[0..-2].strip
+                  # NOTE stripping the little >> character here...
+                end
+              end
+            end
+
+            item_details = doc % 'ul.itemDataList'
+            isbns = []
+            isbns << data_from_label(item_details, 'EAN')
+            isbns << data_from_label(item_details, 'ISBN')
+
+            year = nil
+            date = data_from_label(item_details, 'Erschienen:')
+            if date =~ /([\d]{4})/
+              year = Regexp.last_match[1].to_i
+            end
+
+            binding = data_from_label(item_details, 'Einband')
+
+            publisher = data_from_label(item_details, 'Erschienen bei:')
+
+            book = Book.new(title, authors, isbns.first,
+                            publisher, year, binding)
+
+            image_url = nil
+            if (image_link = doc % 'a[@id=itemPicStart]')
+              image_url = image_link['href']
+            end
+
+            return [book, image_url]
+
+          end
+        rescue => ex
+          trace = ex.backtrace.join("\n> ")
+          log.warn {'Failed parsing search results for Thalia ' \
+            "#{ex.message} #{trace}" }
+          raise NoResultsError
+        end
+      end
+    end
+  end
+end

data/lib/alexandria/book_providers/web.rb

@@ -0,0 +1,59 @@
+# -*- ruby -*-
+#
+# Copyright (C) 2009 Cathal Mc Ginley
+# Copyright (C) 2014 Matijs van Zuijlen
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'hpricot'
+require 'htmlentities'
+
+module Alexandria
+  class BookProviders
+    class WebsiteBasedProvider < GenericProvider
+      def initialize(name, fullname = nil)
+        super(name, fullname)
+        @htmlentities = HTMLEntities.new
+      end
+
+      def html_to_doc(html, source_data_charset = 'ISO-8859-1')
+        html.force_encoding source_data_charset
+        utf8_html = html.encode('utf-8')
+        normalized_html = @htmlentities.decode(utf8_html)
+        Hpricot(normalized_html)
+      end
+
+      ## from Palatina
+      def text_of(node)
+        if node.nil?
+          nil
+        else
+          if node.text?
+            node.to_html
+          elsif node.elem?
+            if node.children.nil?
+              return nil
+            else
+              node_text = node.children.map { |n| text_of(n) }.join
+              node_text.strip.squeeze(' ')
+            end
+          end
+          # node.inner_html.strip
+        end
+      end
+    end
+  end
+end

data/lib/alexandria/book_providers/webster_it.rb

@@ -0,0 +1,173 @@
+# Copyright (C) 2007 Marco Costantini
+# Copyright (C) 2014 Matijs van Zuijlen
+# based on ibs_it.rb by Claudio Belotti
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'fileutils'
+require 'net/http'
+require 'open-uri'
+# require 'cgi'
+
+module Alexandria
+  class BookProviders
+    class Webster_itProvider < GenericProvider
+      BASE_URI = 'http://www.libreriauniversitaria.it' # also "http://www.webster.it"
+      CACHE_DIR = File.join(Alexandria::Library::DIR, '.webster_it_cache')
+      REFERER = BASE_URI
+      LOCALE = 'BIT' # used only for search by title/author/keyword. possible are: "BIT", "BUS", "BUK", "BDE", "MIT"
+      def initialize
+        super('Webster_it', 'Webster (Italy)')
+        FileUtils.mkdir_p(CACHE_DIR) unless File.exist?(CACHE_DIR)
+        # no preferences for the moment
+        at_exit { clean_cache }
+      end
+
+      def search(criterion, type)
+        criterion = criterion.convert('ISO-8859-15', 'UTF-8')
+        req = BASE_URI + '/'
+        req += case type
+               when SEARCH_BY_ISBN
+                 'isbn/' # "#{LOCALE}/"
+
+               when SEARCH_BY_TITLE
+                 "c_search.php?noinput=1&shelf=#{LOCALE}&title_query="
+
+               when SEARCH_BY_AUTHORS
+                 "c_search.php?noinput=1&shelf=#{LOCALE}&author_query="
+
+               when SEARCH_BY_KEYWORD
+                 "c_search.php?noinput=1&shelf=#{LOCALE}&subject_query="
+
+               else
+                 raise InvalidSearchTypeError
+
+               end
+
+        req += CGI.escape(criterion)
+        p req if $DEBUG
+        data = transport.get(URI.parse(req))
+        if type == SEARCH_BY_ISBN
+          to_book(data) # rescue raise NoResultsError
+        else
+          begin
+            results = []
+            each_book_page(data) do |code, _title|
+              results << to_book(transport.get(URI.parse(BASE_URI + "/#{LOCALE}/" + code)))
+            end
+            return results
+          rescue
+            raise NoResultsError
+          end
+        end
+      end
+
+      def url(book)
+        BASE_URI + '/isbn/' + book.isbn
+      end
+
+      #######
+      private
+      #######
+
+      def to_book(data)
+        raise NoResultsError if /<font color="\#ffffff"><b>Prodotto non esistente<\/b><\/font>/.match(data)
+        data = data.convert('UTF-8', 'ISO-8859-15')
+
+        md = /<li><span class="product_label">Titolo:<\/span><span class="product_text"> ([^<]+)/.match(data)
+        raise unless md
+        title = CGI.unescape(md[1].strip)
+        if (md = /<span class="product_heading_volume">([^<]+)/.match(data))
+          title += ' ' + CGI.unescape(md[1].strip)
+        end
+
+        authors = []
+        if (md = /<li><span class="product_label">Autor[ei]:<\/span> <span class="product_text">(<a href="[^>]+">([^<]+)<\/a>,? ?)+<\/span><li>/.match(data))
+          this = CGI.unescape(md[0].strip)
+          authors = this.scan(/<a href="[^>]+">([^<]+)<\/a>,?/)
+          authors = authors.map { |author| author[0] }
+          # puts this
+          #                 md[1].strip.split(', ').each { |a| authors << CGI.unescape(a.strip) }
+        end
+
+        md = /<li><span class="product_label">ISBN:<\/span> <span class="product_text">([^<]+)/.match(data)
+        raise unless md
+        isbn = Library.canonicalise_ean(md[1].strip)
+
+        # raise unless
+        md = /<li><span class="product_label">Editore:<\/span> <span class="product_text"><a href="[^>]+>([^<]+)/.match(data)
+        publisher = CGI.unescape(md[1].strip) or md
+
+        if (md = /<li><span class="product_label">Pagine:<\/span> <span class="product_text">([^<]+)/.match(data))
+          edition = CGI.unescape(md[1].strip) + ' p.'
+        else
+          edition = nil
+        end
+
+        publish_year = nil
+        if (md = /<li><span class="product_label">Data di Pubblicazione:<\/span> <span class="product_text">([^<]+)/.match(data))
+          publish_year = CGI.unescape(md[1].strip)[-4..-1].to_i
+          publish_year = nil if publish_year == 0
+        end
+
+        if data =~ /javascript:popImage/ and (md = /<img border="0" alt="[^"]+" src="([^"]+)/.match(data))
+          cover_url = BASE_URI + md[1].strip
+          # use "p" instead of "g" for smaller image
+          if cover_url[-5] == 103
+            cover_url[-5] = 112
+          end
+
+          cover_filename = isbn + '.tmp'
+          Dir.chdir(CACHE_DIR) do
+            begin
+              cover_data = open(cover_url, 'Referer' => REFERER).read
+            rescue OpenURI::HTTPError
+              cover_data = nil
+            end
+            if cover_data
+              File.open(cover_filename, 'w') do |file|
+                file.write cover_data
+              end
+            end
+          end
+
+          medium_cover = CACHE_DIR + '/' + cover_filename
+          if File.size(medium_cover) > 0
+            puts medium_cover + ' has non-0 size' if $DEBUG
+            return [Book.new(title, authors, isbn, publisher, publish_year, edition), medium_cover]
+          end
+          puts medium_cover + ' has 0 size, removing ...' if $DEBUG
+          File.delete(medium_cover)
+        end
+        [Book.new(title, authors, isbn, publisher, publish_year, edition)]
+      end
+
+      def each_book_page(data)
+        raise if data.scan(/<tr ><td width="10%" align="center"">&nbsp;<a href="#{LOCALE}\/([^\/]+)/) { |a| yield a }.empty?
+      end
+
+      def clean_cache
+        # FIXME begin ... rescue ... end?
+        Dir.chdir(CACHE_DIR) do
+          Dir.glob('*.tmp') do |file|
+            puts 'removing ' + file if $DEBUG
+            File.delete(file)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/alexandria/book_providers/worldcat.rb

@@ -0,0 +1,251 @@
+# -*- ruby -*-
+#
+# Copyright (C) 2009 Cathal Mc Ginley
+# Copyright (C) 2011, 2014 Matijs van Zuijlen
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+# http://en.wikipedia.org/wiki/WorldCat
+# See http://www.oclc.org/worldcat/policies/terms/
+
+# New WorldCat provider, taken from the Palatina MetaDataSource and
+# modified to fit the structure of Alexandria book providers.
+# (25 Feb 2009)
+#
+# Updated from Palatina, to reflect changes in the worldcat website.
+# (1 Sep 2009)
+
+require 'cgi'
+require 'alexandria/net'
+require 'alexandria/book_providers/web'
+
+module Alexandria
+  class BookProviders
+    class WorldCatProvider < WebsiteBasedProvider
+      include Alexandria::Logging
+
+      SITE = 'http://www.worldcat.org'
+      BASE_SEARCH_URL = "#{SITE}/search?q=%s%s&qt=advanced" # type, term
+
+      def initialize
+        super('WorldCat', 'WorldCat')
+        # prefs.add("enabled", _("Enabled"), true, [true,false])
+        prefs.read
+      end
+
+      def search(criterion, type)
+        # puts create_search_uri(type, criterion)
+        req = create_search_uri(type, criterion)
+        puts req if $DEBUG
+        html_data = transport.get_response(URI.parse(req))
+        # Note: I tried to use Alexandria::WWWAgent,
+        #       but this caused failures here (empty pages...)
+        #       find out how the requests differ
+
+        # puts html_data.class
+        if type == SEARCH_BY_ISBN
+          parse_result_data(html_data.body, criterion)
+        else
+          results = parse_search_result_data(html_data.body)
+          raise NoResultsError if results.empty?
+
+          results.map { |result| get_book_from_search_result(result) }
+        end
+      end
+
+      def url(book)
+        create_search_uri(SEARCH_BY_ISBN, book.isbn)
+      rescue => ex
+        log.warn { "Cannot create url for book #{book}; #{ex.message}" }
+        nil
+      end
+
+      private
+
+      def create_search_uri(search_type, search_term)
+        search_type_code = { SEARCH_BY_ISBN => 'isbn:',
+                             SEARCH_BY_AUTHORS => 'au:',
+                             SEARCH_BY_TITLE => 'ti:',
+                             SEARCH_BY_KEYWORD => ''
+        }[search_type] or ''
+        search_type_code = CGI.escape(search_type_code)
+        search_term_encoded = search_term # TODO, remove attack stuff
+        if search_type == SEARCH_BY_ISBN
+          search_term_encoded = Library.canonicalise_ean(search_term) # isbn-13
+        else
+          search_term_encoded = CGI.escape(search_term)
+        end
+        BASE_SEARCH_URL % [search_type_code, search_term_encoded]
+      end
+
+      def get_book_from_search_result(result)
+        log.debug { "Fetching book from #{result[:url]}" }
+        html_data =  transport.get_response(URI.parse(result[:url]))
+        parse_result_data(html_data.body)
+      end
+
+      def parse_search_result_data(html)
+        doc = html_to_doc(html, 'UTF-8')
+        book_search_results = []
+        begin
+          result_cells = doc / 'td.result/div.name/..'
+          # puts result_cells.length
+          result_cells.each do |td|
+            type_icon = (td % 'div.type/img.icn')
+            next unless type_icon and type_icon['src'] =~ /icon-bks/
+            name_div = td % 'div.name'
+            title = name_div.inner_text
+            anchor = name_div % :a
+            if anchor
+              url = anchor['href']
+            end
+            lookup_url = "#{SITE}#{url}"
+            result = {}
+            result[:title] = title
+            result[:url] = lookup_url
+
+            book_search_results << result
+          end
+        rescue => ex
+          trace = ex.backtrace.join("\n> ")
+          log.warn {'Failed parsing search results for WorldCat ' \
+                    "#{ex.message} #{trace}" }
+        end
+        book_search_results
+      end
+
+      def parse_result_data(html, search_isbn = nil, recursing = false)
+        doc = html_to_doc(html, 'UTF-8')
+
+        begin
+          if doc % 'div#div-results-none'
+            log.debug { 'WorldCat reports no results' }
+            raise NoResultsError
+          end
+
+          if doc % 'table.table-results'
+            if recursing
+              log.warn { 'Infinite loop prevented redirecting through WorldCat' }
+              raise NoResultsError
+            end
+            log.info { 'Found multiple results for lookup: checking each' }
+            search_results = parse_search_result_data(html)
+            book = nil
+            cover_url = nil
+            first_result = nil
+            search_results.each do |rslt|
+              # rslt = search_results.rslt
+              log.debug { "checking #{rslt[:url]}" }
+              rslt2 = transport.get_response(URI.parse(rslt[:url]))
+              html2 = rslt2.body
+
+              book, cover_url = parse_result_data(html2, search_isbn, true)
+              if first_result.nil?
+                first_result = [book, cover_url]
+              end
+
+              log.debug { "got book #{book}" }
+
+              if search_isbn
+                search_isbn_canon = Library.canonicalise_ean(search_isbn)
+                rslt_isbn_canon = Library.canonicalise_ean(book.isbn)
+                if search_isbn_canon == rslt_isbn_canon
+                  log.info { "book #{book} is a match" }
+                  return [book, cover_url]
+                end
+                log.debug { 'not a match, checking next' }
+              else
+                # no constraint to match isbn, just return first result
+                return [book, cover_url]
+              end
+            end
+
+            # gone through all and no ISBN match, so just return first result
+            log.info { 'no more results to check. Returning first result, just an approximation' }
+            return first_result
+
+          end
+
+          title_header = doc % 'h1.title'
+          title = title_header.inner_text if title_header
+          unless title
+            log.warn { 'Unexpected lack of title from WorldCat lookup' }
+            raise NoResultsError
+          end
+          log.info { "Found book #{title} at WorldCat" }
+
+          authors = []
+          authors_tr = doc % 'tr#details-allauthors'
+          if authors_tr
+            (authors_tr / :a).each do |a|
+              authors << a.inner_text
+            end
+          end
+
+          # can we do better? get the City name?? or multiple publishers?
+          bibdata = doc % 'div#bibdata'
+          bibdata_table = bibdata % :table
+          publisher_row = bibdata_table % 'th[text()*=Publisher]/..'
+
+          if publisher_row
+            publication_info = (publisher_row / 'td').last.inner_text
+
+            if publication_info.index(';')
+              publication_info =~ /;[\s]*([^\d]+)[\s]*[\d]*/
+            elsif publication_info.index(':')
+              publication_info =~ /:[\s]*([^;:,]+)/
+            else
+              publication_info =~ /([^;,]+)/
+            end
+
+            publisher = Regexp.last_match[1]
+            publication_info =~ /([12][0-9]{3})/
+            year = Regexp.last_match[1].to_i if Regexp.last_match[1]
+          else
+            publisher = nil
+            year = nil
+          end
+
+          isbn = search_isbn
+          unless isbn
+            isbn_row = doc % 'tr#details-standardno' # #bibdata_table % 'th[text()*=ISBN]/..'
+            if isbn_row
+              isbns = (isbn_row / 'td').last.inner_text.split
+              isbn = Library.canonicalise_isbn(isbns.first)
+            else
+              log.warn { 'No ISBN found on page' }
+            end
+          end
+
+          binding = '' # not given on WorldCat website (as far as I can tell)
+
+          book = Book.new(title, authors, isbn, publisher, year, binding)
+
+          image_url = nil # hm, it's on the website, but uses JavaScript...
+
+          return [book, image_url]
+
+        rescue => ex
+          raise ex if ex.instance_of? NoResultsError
+          trace = ex.backtrace.join("\n> ")
+          log.warn {'Failed parsing search results for WorldCat ' \
+                    "#{ex.message} #{trace}" }
+          raise NoResultsError
+        end
+      end
+    end # class WorldCatProvider
+  end # class BookProviders
+end # module Alexandria