alexandria-book-collection-manager 0.6.9.pre1

data/lib/alexandria/book_providers/ibs_it.rb

@@ -0,0 +1,149 @@
+# Copyright (C) 2005-2006 Claudio Belotti
+# Copyright (C) 2014 Matijs van Zuijlen
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'fileutils'
+require 'net/http'
+require 'open-uri'
+# require 'cgi'
+
+module Alexandria
+  class BookProviders
+    class IBS_itProvider < GenericProvider
+      BASE_URI = 'http://www.internetbookshop.it'
+      CACHE_DIR = File.join(Alexandria::Library::DIR, '.ibs_it_cache')
+      REFERER = BASE_URI
+      def initialize
+        super('IBS_it', 'Internet Bookshop (Italy)')
+        FileUtils.mkdir_p(CACHE_DIR) unless File.exist?(CACHE_DIR)
+        # no preferences for the moment
+        at_exit { clean_cache }
+      end
+
+      def search(criterion, type)
+        criterion = criterion.convert('ISO-8859-1', 'UTF-8')
+        req = BASE_URI + '/ser/'
+        req += case type
+               when SEARCH_BY_ISBN
+                 'serdsp.asp?isbn='
+
+               when SEARCH_BY_TITLE
+                 'serpge.asp?Type=keyword&T='
+
+               when SEARCH_BY_AUTHORS
+                 'serpge.asp?Type=keyword&A='
+
+               when SEARCH_BY_KEYWORD
+                 'serpge.asp?Type=keyword&S='
+
+               else
+                 raise InvalidSearchTypeError
+
+               end
+
+        req += CGI.escape(criterion)
+        p req if $DEBUG
+        data = transport.get(URI.parse(req))
+        if type == SEARCH_BY_ISBN
+          to_book(data) # rescue raise NoResultsError
+        else
+          begin
+            results = []
+            each_book_page(data) do |code, _title|
+              results << to_book(transport.get(URI.parse('http://www.internetbookshop.it/ser/serdsp.asp?cc=' + code)))
+            end
+            return results
+          rescue
+            raise NoResultsError
+          end
+        end
+      end
+
+      def url(book)
+        'http://www.internetbookshop.it/ser/serdsp.asp?isbn=' + book.isbn
+      end
+
+      #######
+      private
+      #######
+
+      def to_book(data)
+        raise NoResultsError if /<b>Il libro che hai cercato non &egrave; presente nel nostro catalogo<\/b><br>/.match(data)
+        data = data.convert('UTF-8', 'ISO-8859-1')
+
+        md = />Titolo<\/td><td valign="top" class="lbarrasup">([^<]+)/.match(data)
+        raise 'No title' unless md
+        title = CGI.unescape(md[1].strip)
+
+        authors = []
+        if (md = /<b>Autore<\/b><\/td>.+<b>([^<]+)/.match(data))
+          md[0].strip.gsub(/<.*?>|Autore/m, ' ').split('; ').each { |a| authors << CGI.unescape(a.strip) }
+        end
+
+        md = /<input type=\"hidden\" name=\"isbn\" value=\"([^"]+)\">/i.match(data)
+        raise 'No ISBN' unless md
+        isbn = md[1].strip
+
+        # raise "No publisher" unless
+        md = /<b>Editore<\/b><\/td>.+<b>([^<]+)/.match(data)
+        publisher = CGI.unescape(md[1].strip) or md
+
+        # raise "No edition" unless
+        md = /Dati<\/b><\/td><td valign="top">([^<]+)/.match(data)
+        edition = CGI.unescape(md[1].strip) or md
+
+        publish_year = nil
+        if (md = /Anno<\/b><\/td><td valign="top">([^<]+)/.match(data))
+          publish_year = CGI.unescape(md[1].strip).to_i
+          publish_year = nil if publish_year == 0
+        end
+
+        md = /src="http:\/\/giotto.ibs.it\/cop\/copt13.asp\?f=(\d+)">/.match(data)
+        cover_url = 'http://giotto.ibs.it/cop/copt13.asp?f=' + md[1] # use copa13.asp, copt13.asp, copj13.asp, for small, medium, big image
+        cover_filename = isbn + '.tmp'
+        Dir.chdir(CACHE_DIR) do
+          File.open(cover_filename, 'w') do |file|
+            file.write open(cover_url, 'Referer' => REFERER).read
+          end
+        end
+
+        medium_cover = CACHE_DIR + '/' + cover_filename
+        if File.size(medium_cover) > 0 and File.size(medium_cover) != 1822 # 1822 is the size of the fake image "copertina non disponibile"
+          puts medium_cover + ' has non-0 size' if $DEBUG
+          return [Book.new(title, authors, isbn, publisher, publish_year, edition), medium_cover]
+        end
+        puts medium_cover + ' has 0 size, removing ...' if $DEBUG
+        File.delete(medium_cover)
+        [Book.new(title, authors, isbn, publisher, publish_year, edition)]
+      end
+
+      def each_book_page(data)
+        raise if data.scan(/<a href="http:\/\/www.internetbookshop.it\/ser\/serdsp.asp\?shop=1&amp;cc=([\w\d]+)"><b>([^<]+)/) { |a| yield a }.empty?
+      end
+
+      def clean_cache
+        # FIXME begin ... rescue ... end?
+        Dir.chdir(CACHE_DIR) do
+          Dir.glob('*.tmp') do |file|
+            puts 'removing ' + file if $DEBUG
+            File.delete(file)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/alexandria/book_providers/mcu.rb

@@ -0,0 +1,177 @@
+# Copyright (C) 2004 Javier Fernandez-Sanguino
+# Copyright (C) 2007 Javier Fernandez-Sanguino and Marco Costantini
+# Modifications Copyright (C) 2011 Matijs van Zuijlen
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'cgi'
+require 'net/http'
+
+# http://www.mcu.es/libro/CE/AgenciaISBN/BBDDLibros/Sobre.html
+# http://www.mcu.es/comun/bases/isbn/ISBN.html
+
+module Alexandria
+  class BookProviders
+    class MCUProvider < GenericProvider
+      include Logging
+      include GetText
+      GetText.bindtextdomain(Alexandria::TEXTDOMAIN, charset: 'UTF-8')
+
+      LANGUAGES = {
+        'es' => '1'
+      }
+
+      #        BASE_URI = "http://www.mcu.es/cgi-bin/BRSCGI3701?"
+      BASE_URI = 'http://www.mcu.es/cgi-brs/BasesHTML/isbn/BRSCGI?'
+      def initialize
+        super('MCU', _('Spanish Culture Ministry'))
+        # No preferences
+        prefs.read
+      end
+
+      def search(criterion, type)
+        prefs.read
+        begin
+          criterion = criterion.convert('ISO-8859-1', 'UTF-8') # still needed??
+        rescue GLib::ConvertError
+          log.info { "Cannot search for non-ISO-8859-1 terms at MCU : #{criterion}" }
+          raise NoResultsError
+        end
+        print "Doing search with MCU #{criterion}, type: #{type}\n" if $DEBUG # for DEBUGing
+        req = BASE_URI + 'CMD=VERLST&BASE=ISBN&DOCS=1-15&CONF=AEISPA.cnf&OPDEF=AND&DOCS=1-1000&SEPARADOR=&'
+        req += case type
+               when SEARCH_BY_ISBN
+                 "WGEN-C=&WISB-C=#{CGI.escape(criterion)}&WAUT-C=&WTIT-C=&WMAT-C=&WEDI-C=&WFEP-C=&%40T353-GE=&%40T353-LE=&WSER-C=&WLUG-C=&WDIS-C=%28DISPONIBLE+or+AGOTADO%29&WLEN-C=&WCLA-C=&WSOP-C="
+
+               when SEARCH_BY_TITLE
+                 "WGEN-C=&WISB-C=&WAUT-C=&WTIT-C=#{CGI.escape(criterion)}&WMAT-C=&WEDI-C=&WFEP-C=&%40T353-GE=&%40T353-LE=&WSER-C=&WLUG-C=&WDIS-C=%28DISPONIBLE+or+AGOTADO%29&WLEN-C=&WCLA-C=&WSOP-C="
+
+               when SEARCH_BY_AUTHORS
+                 "WGEN-C=&WISB-C=&WAUT-C=#{CGI.escape(criterion)}&WTIT-C=&WMAT-C=&WEDI-C=&WFEP-C=&%40T353-GE=&%40T353-LE=&WSER-C=&WLUG-C=&WDIS-C=%28DISPONIBLE+or+AGOTADO%29&WLEN-C=&WCLA-C=&WSOP-C="
+
+               when SEARCH_BY_KEYWORD
+                 "WGEN-C=#{CGI.escape(criterion)}&WISB-C=&WAUT-C=&WTIT-C=&WMAT-C=&WEDI-C=&WFEP-C=&%40T353-GE=&%40T353-LE=&WSER-C=&WLUG-C=&WDIS-C=%28DISPONIBLE+or+AGOTADO%29&WLEN-C=&WCLA-C=&WSOP-C="
+
+               else
+                 raise InvalidSearchTypeError
+               end
+        products = {}
+        print "Request page is #{req}\n" if $DEBUG # for DEBUGing
+        transport.get(URI.parse(req)).each do |line|
+          # line = line.convert("ISO-8859-1", "UTF-8")
+          print "Reading line: #{line}" if $DEBUG # for DEBUGing
+          if (line =~ /CMD=VERDOC.*&DOCN=([^&]*)&NDOC=([^&]*)/) and (!products[Regexp.last_match[1]]) and (book = parseBook(Regexp.last_match[1], Regexp.last_match[2])) then
+            products[Regexp.last_match[1]] = book
+            puts Regexp.last_match[1] if $DEBUG # for DEBUGing
+          end
+        end
+
+        raise NoResultsError if products.values.empty?
+        type == SEARCH_BY_ISBN ? products.values.first : products.values
+      end
+
+      def url(book)
+        isbn = Library.canonicalise_isbn(book.isbn)
+        "http://www.mcu.es/cgi-brs/BasesHTML/isbn/BRSCGI?CMD=VERLST&BASE=ISBN&DOCS=1-15&CONF=AEISPA.cnf&OPDEF=AND&DOCS=1&SEPARADOR=&WGEN-C=&WISB-C=#{isbn}&WAUT-C=&WTIT-C=&WMAT-C=&WEDI-C=&WFEP-C=&%40T353-GE=&%40T353-LE=&WSER-C=&WLUG-C=&WDIS-C=%28DISPONIBLE+or+AGOTADO%29&WLEN-C=&WCLA-C=&WSOP-C="
+      rescue => ex
+        log.warn { "Cannot create url for book #{book}; #{ex.message}" }
+        nil
+      end
+
+      #######
+      private
+      #######
+
+      def parseBook(docn, ndoc)
+        detailspage = 'http://www.mcu.es/cgi-brs/BasesHTML/isbn/BRSCGI?CMD=VERDOC&CONF=AEISPA.cnf&BASE=ISBN&DOCN=' + docn + '&NDOC=' + ndoc
+        print "Looking at detailspage: #{detailspage}\n" if $DEBUG # for DEBUGing
+        product = {}
+        product['authors'] = []
+        robotstate = 0
+        transport.get(URI.parse(detailspage)).each do |line|
+          # This is a very crude robot interpreter
+          # Note that the server provides more information
+          # we don't store:
+          # - Language  - Description
+          # - Binding   - Price
+          # - Colection - Theme
+          # - CDU      - Last update
+
+          # There seems to be an issue with accented chars..
+          line = line.convert('UTF-8', 'ISO-8859-1')
+          print "Reading line (robotstate #{robotstate}): #{line}" if $DEBUG # for DEBUGing
+          if line =~ /^<\/td>$/ or line =~ /^<\/tr>$/
+            robotstate = 0
+          elsif robotstate == 1 and line =~ /^([^<]+)</
+            author = Regexp.last_match[1].gsub('&nbsp;', ' ').sub(/ +$/, '')
+            if author.length > 3 then
+              # Only add authors of appropiate length
+              product['authors'] << author
+              print "Authors are #{product['authors']}\n" if $DEBUG # for DEBUGing
+              robotstate = 0
+            end
+          elsif robotstate == 2 and line =~ /^(.*)$/ # The title es the next line to title declaration and has not tags on web src code
+            product['name'] = Regexp.last_match[1].strip
+            print "Name is #{product['name']}\n" if $DEBUG # for DEBUGing
+            robotstate = 0
+          elsif robotstate == 3 and line =~ /^([0-9]+-[0-9]+-[0-9]+-[0-9]+-[0-9]).*/
+            product['isbn'] = Regexp.last_match[1]
+            print "ISBN is #{product['isbn']}\n" if $DEBUG # for DEBUGing
+            robotstate = 0
+          elsif robotstate == 4 and line =~ /^([^<]+)</
+            product['manufacturer'] = Regexp.last_match[1].strip
+            print "Manufacturer is #{product['manufacturer']}\n" if $DEBUG # for DEBUGing
+            robotstate = 0
+            #                elsif robotstate == 5 and line =~ /^([^<]+)</
+          elsif robotstate == 5 and line =~ /<span>([^<]+)</
+            product['media'] = Regexp.last_match[1].strip
+            print "Media is #{product['media']}\n" if $DEBUG # for DEBUGing
+            robotstate = 0
+          elsif line =~ /^.*>Autor:\s*</
+            robotstate = 1
+          elsif line =~ /^.*>T(.|&iacute;)tulo:\s*</
+            robotstate = 2
+          elsif line =~ /^.*>ISBN \(13\):\s*</
+            robotstate = 3
+          elsif line =~ /^.*>Publicaci(.|&oacute;)n:\s*</
+            robotstate = 4
+          elsif line =~ /^.*>Encuadernaci(.|&oacute;)n:\s*</
+            robotstate = 5
+          end
+        end
+
+        # TODO: This provider does not include picture for books
+        #            %w{name isbn media manufacturer}.each do |field|
+        #               print "Checking #{field} for nil\n" if $DEBUG # for DEBUGing
+        #                product[field]="" if product[field].nil?
+        #            end
+
+        print "Creating new book\n" if $DEBUG # for DEBUGing
+        book = Book.new(product['name'],
+                        product['authors'],
+                        product['isbn'].delete('-'),
+                        product['manufacturer'],
+                        nil, # TODO: furnish publish year
+                        product['media'])
+        if book.title.nil?
+          log.warn { "No title was returned for #{book.isbn}" }
+          book.title = ''
+        end
+        [book]
+      end
+    end
+  end
+end

data/lib/alexandria/book_providers/proxis.rb

@@ -0,0 +1,205 @@
+# Copyright (C) 2009 Cathal Mc Ginley
+# Copyright (C) 2014 Matijs van Zuijlen
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+# New Proxis provider, taken from Palatina MetaDataSource and modified
+# for Alexandria. (20 Dec 2009)
+
+require 'cgi'
+require 'alexandria/book_providers/web'
+
+module Alexandria
+  class BookProviders
+    class ProxisProvider < WebsiteBasedProvider
+      # include GetText
+      include Alexandria::Logging
+      # GetText.bindtextdomain(Alexandria::TEXTDOMAIN, :charset => "UTF-8")
+
+      # Proxis essentially has three book databases, NL, FR and EN.
+      # Currently, this provider only searches the NL database, since
+      # it adds most to Alexandria (Amazon already has French and
+      # English titles).
+
+      SITE = 'http://www.proxis.nl'
+      BASE_SEARCH_URL = "#{SITE}/NLNL/Search/IndexGSA.aspx?search=%s" \
+        '&shop=100001NL&SelRubricLevel1Id=100001NL'
+      ISBN_REDIRECT_BASE_URL = "#{SITE}/NLNL/Search/Index.aspx?search=%s" \
+        '&shop=100001NL&SelRubricLevel1Id=100001NL'
+
+      def initialize
+        super('Proxis', 'Proxis (Belgium)')
+        # prefs.add("lang", _("Language"), "fr",
+        #          LANGUAGES.keys)
+        prefs.read
+      end
+
+      ## criterion = criterion.convert("windows-1252", "UTF-8")
+      ## is the above still needed??
+      ## current pages are returned in UTF-8, so I think probably not!
+
+      def search(criterion, type)
+        req = create_search_uri(type, criterion)
+        puts req if $DEBUG
+        html_data = transport.get_response(URI.parse(req))
+
+        results = parse_search_result_data(html_data.body)
+        raise NoResultsError if results.empty?
+        if type == SEARCH_BY_ISBN
+          get_book_from_search_result(results.first)
+        else
+          results.map { |result| get_book_from_search_result(result) }
+        end
+      end
+
+      def create_search_uri(search_type, search_term)
+        if search_type == SEARCH_BY_ISBN
+          BASE_SEARCH_URL % Library.canonicalise_ean(search_term)
+        else
+          BASE_SEARCH_URL % CGI.escape(search_term)
+        end
+      end
+
+      def get_book_from_search_result(result)
+        log.debug { "Fetching book from #{result[:lookup_url]}" }
+        html_data =  transport.get_response(URI.parse(result[:lookup_url]))
+        parse_result_data(html_data.body)
+      end
+
+      def url(book)
+        if book.isbn.nil? or book.isbn.empty?
+          ISBN_REDIRECT_BASE_URL % Library.canonicalise_ean(book.isbn)
+        else
+          nil
+        end
+      end
+
+      ## from Palatina
+      def text_of(node)
+        if node.nil?
+          nil
+        else
+          if node.text?
+            node.to_html
+          elsif node.elem?
+            if node.children.nil?
+              return nil
+            else
+              node_text = node.children.map { |n| text_of(n) }.join
+              node_text.strip.squeeze(' ')
+            end
+          end
+          # node.inner_html.strip
+        end
+      end
+
+      def parse_search_result_data(html)
+        doc = html_to_doc(html)
+        book_search_results = []
+        items = (doc.search('table.searchResult tr'))
+        items.each do |item|
+          result = {}
+          title_link = item % 'h5 a'
+          if title_link
+            result[:title] = text_of(title_link)
+            result[:lookup_url] = title_link['href']
+            unless result[:lookup_url] =~ /^http/
+              result[:lookup_url] = "#{SITE}#{result[:lookup_url]}"
+            end
+          end
+          book_search_results << result
+        end
+        # require 'pp'
+        # pp book_search_results
+        # raise :Ruckus
+        book_search_results
+      end
+
+      def data_for_header(th)
+        tr = th.parent
+        td = tr.at('td')
+        if td
+          text_of(td)
+        else
+          nil
+        end
+      end
+
+      def parse_result_data(html)
+        doc = html_to_doc(html)
+        book_data = {}
+        book_data[:authors] = []
+        # TITLE
+        if (title_header = doc.search('div.detailBlock h3'))
+          header_spans = title_header.first.search('span')
+          title = text_of(header_spans.first)
+          if title =~ /(.+)-$/
+            title = Regexp.last_match[1].strip
+          end
+          book_data[:title] = title
+        end
+
+        info_headers = doc.search('table.productInfoTable th')
+
+        isbns = []
+        unless info_headers.empty?
+          info_headers.each do |th|
+            if th.inner_text =~ /(ISBN|EAN)/
+              isbns << data_for_header(th)
+            end
+          end
+          book_data[:isbn] = Library.canonicalise_ean(isbns.first)
+        end
+
+        # book = Book.new(title, ISBN.get(isbns.first))
+
+        unless info_headers.empty?
+          info_headers.each do |th|
+            header_text = th.inner_text
+            if header_text =~ /Type/
+              book_data[:binding] = data_for_header(th)
+            elsif header_text =~ /Verschijningsdatum/
+              date = data_for_header(th)
+              date =~ /\/([\d]{4})/
+              book_data[:publish_year] = Regexp.last_match[1].to_i
+            elsif header_text =~ /Auteur/
+              book_data[:authors] << data_for_header(th)
+            elsif header_text =~ /Uitgever/
+              book_data[:publisher] = data_for_header(th)
+            end
+          end
+        end
+
+        image_url = nil
+        if (cover_img = doc.at("img[@id$='imgProduct']"))
+          if cover_img['src'] =~ /^http/
+            image_url = cover_img['src']
+          else
+            image_url = "#{SITE}/#{cover_img['src']}" # TODO use html <base>
+          end
+          if image_url =~ /ProductNoCover/
+            image_url = nil
+          end
+        end
+
+        book = Book.new(book_data[:title], book_data[:authors],
+                        book_data[:isbn], book_data[:publisher],
+                        book_data[:publish_year], book_data[:binding])
+        [book, image_url]
+      end
+    end
+  end
+end