alexandria-book-collection-manager 0.6.9.pre1

data/lib/alexandria/book_providers/bol_it.rb

@@ -0,0 +1,162 @@
+# Copyright (C) 2007 Marco Costantini
+# Copyright (C) 2014 Matijs van Zuijlen
+# based on ibs_it.rb by Claudio Belotti
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'fileutils'
+require 'net/http'
+require 'open-uri'
+# require 'cgi'
+
+module Alexandria
+  class BookProviders
+    class BOL_itProvider < GenericProvider
+      BASE_URI = 'http://www.bol.it'
+      CACHE_DIR = File.join(Alexandria::Library::DIR, '.bol_it_cache')
+      REFERER = BASE_URI
+      LOCALE = 'libri' # possible locales are: "libri", "inglesi", "video", "musica", "choco"
+      def initialize
+        super('BOL_it', 'BOL (Italy)')
+        FileUtils.mkdir_p(CACHE_DIR) unless File.exist?(CACHE_DIR)
+        # no preferences for the moment
+        at_exit { clean_cache }
+      end
+
+      def search(criterion, type)
+        criterion = criterion.convert('ISO-8859-1', 'UTF-8')
+        req = BASE_URI + '/' + LOCALE + '/'
+        req += case type
+               when SEARCH_BY_ISBN
+                 'scheda/'
+
+               when SEARCH_BY_TITLE
+                 'risultatoricerca?action=bolrisultatoricerca&skin=bol&filtro_ricerca=BOL&quick_type=Titolo&titolo='
+
+               when SEARCH_BY_AUTHORS
+                 'risultatoricerca?action=bolrisultatoricerca&skin=bol&filtro_ricerca=BOL&quick_type=Autore&titolo='
+
+               when SEARCH_BY_KEYWORD
+                 'risultatoricerca?action=bolrisultatoricerca&skin=bol&filtro_ricerca=BOL&quick_type=Parola%20chiave&titolo='
+
+               else
+                 raise InvalidSearchTypeError
+
+               end
+
+        ## warning: this provider uses pages like http://www.bol.it/libri/scheda/ea978888584104 with 12 numbers, without the checksum
+        criterion = 'ea' + Library.canonicalise_ean(criterion)[0..-2] + '.html' if type == SEARCH_BY_ISBN
+        req += CGI.escape(criterion)
+        p req if $DEBUG
+        data = transport.get(URI.parse(req))
+        if type == SEARCH_BY_ISBN
+          to_book(data) # rescue raise NoResultsError
+        else
+          begin
+            results = []
+            each_book_page(data) do |code, _title|
+              results << to_book(transport.get(URI.parse(BASE_URI + "/#{LOCALE}/scheda/ea" + code)))
+            end
+            return results
+          rescue
+            raise NoResultsError
+          end
+        end
+      end
+
+      def url(book)
+        BASE_URI + "/#{LOCALE}/scheda/ea" + Library.canonicalise_ean(book.isbn)[0..-2] + '.html'
+      end
+
+      #######
+      private
+      #######
+
+      def to_book(data)
+        raise NoResultsError if /Scheda libro non completa  \(TP null\)/.match(data)
+        data = data.convert('UTF-8', 'ISO-8859-1')
+
+        md = /<INPUT type =hidden name ="mailTitolo" value="([^"]+)/.match(data)
+        raise 'No title' unless md
+        title = CGI.unescape(md[1].strip)
+
+        authors = []
+        if (md = /<INPUT type =HIDDEN name ="mailAutore" value="([^"]+)/.match(data))
+          md[1].strip.split(', ').each { |a| authors << CGI.unescape(a.strip) }
+        end
+
+        md = /<INPUT type =HIDDEN name ="mailEAN" value="([^"]+)/.match(data)
+        raise 'No ISBN' unless md
+        isbn = md[1].strip
+        isbn += String(Library.ean_checksum(Library.extract_numbers(isbn)))
+
+        # raise unless
+        md = /<INPUT type =HIDDEN name ="mailEditore" value="([^"]+)/.match(data)
+        publisher = CGI.unescape(md[1].strip) or md
+
+        # raise unless
+        md = /<INPUT type =HIDDEN name ="mailFormato" value="([^"]+)/.match(data)
+        edition = CGI.unescape(md[1].strip) or md
+
+        if (md = /#{edition}\&nbsp\;\|\&nbsp\;(\d+)\&nbsp\;\|\&nbsp\;/.match(data))
+          nr_pages = CGI.unescape(md[1].strip)
+        elsif (md = / (\d+) pagine \| /.match(data))
+          nr_pages = CGI.unescape(md[1].strip)
+        end
+        if nr_pages != '0' and !nr_pages.nil?
+          edition = nr_pages + ' p., ' + edition
+        end
+
+        publish_year = nil
+        if (md = /<INPUT type =HIDDEN name ="mailAnnoPubbl" value="([^"]+)/.match(data))
+          publish_year = CGI.unescape(md[1].strip).to_i
+          publish_year = nil if publish_year == 0
+        end
+
+        cover_url = BASE_URI + '/bol/includes/tornaImmagine.jsp?cdSoc=BL&ean=' + isbn[0..11] + '&tipoOggetto=PIB&cdSito=BL' # use "FRB" instead of "PIB" for smaller images
+        cover_filename = isbn + '.tmp'
+        Dir.chdir(CACHE_DIR) do
+          File.open(cover_filename, 'w') do |file|
+            file.write open(cover_url, 'Referer' => REFERER).read
+          end
+        end
+
+        medium_cover = CACHE_DIR + '/' + cover_filename
+        if File.size(medium_cover) > 43 and File.size(medium_cover) != 2382 # 2382 is the size of the fake image "copertina non disponibile"
+          puts medium_cover + ' has non-0 size' if $DEBUG
+          return [Book.new(title, authors, isbn, publisher, publish_year, edition), medium_cover]
+        end
+        puts medium_cover + ' has 0 size, removing ...' if $DEBUG
+        File.delete(medium_cover)
+        [Book.new(title, authors, isbn, publisher, publish_year, edition)]
+      end
+
+      def each_book_page(data)
+        raise if data.scan(/<a href="\/#{LOCALE}\/scheda\/ea(\d+)\.html;jsessionid=[^"]+">\s*Scheda completa\s*<\/a>/) { |a| yield a }.empty?
+      end
+
+      def clean_cache
+        # FIXME begin ... rescue ... end?
+        Dir.chdir(CACHE_DIR) do
+          Dir.glob('*.tmp') do |file|
+            puts 'removing ' + file if $DEBUG
+            File.delete(file)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/alexandria/book_providers/deastore.rb

@@ -0,0 +1,277 @@
+# -*- ruby -*-
+#
+# Copyright (C) 2009 Cathal Mc Ginley
+# Copyright (C) 2011, 2014 Matijs van Zuijlen
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+# New DeaStore provider, taken from the Palatina MetaDataSource and
+# modified to fit the structure of Alexandria book providers.
+# (24 Feb 2009)
+
+require 'cgi'
+require 'alexandria/net'
+require 'alexandria/book_providers/web'
+
+module Alexandria
+  class BookProviders
+    class DeaStoreProvider < WebsiteBasedProvider
+      include Alexandria::Logging
+
+      SITE = 'http://www.deastore.com'
+      BASE_SEARCH_URL = "#{SITE}/search/italian_books/0/%s/%s" # type/term
+
+      def initialize
+        super('DeaStore', 'DeaStore (Italy)')
+        prefs.read
+        @agent = nil
+      end
+
+      def agent
+        unless @agent
+          @agent = Alexandria::WWWAgent.new
+          @agent.language = :it
+        end
+        @agent
+      end
+
+      def get_book_from_search_result(result)
+        log.debug { "Fetching book from #{result[:url]}" }
+        html_data = agent.get(result[:url])
+        # File.open("rsltflarn#{Time.now().usec()}.html", 'wb') do |f|
+        #  f.write(html_data.body)
+        # end
+        parse_result_data(html_data.body)
+      end
+
+      def search(criterion, type)
+        begin
+          criterion = criterion.convert('ISO-8859-1', 'UTF-8') # still needed??
+        rescue GLib::ConvertError
+          log.info { "Cannot search for non-ISO-8859-1 terms at DeaStore : #{criterion}" }
+          raise NoResultsError
+        end
+        html_data = agent.get(create_search_uri(type, criterion))
+        # File.open("flarn#{Time.now().usec()}.html", 'wb') do |f|
+        #  f.write(html_data.body)
+        # end
+        results = parse_search_result_data(html_data.body)
+        raise NoResultsError if results.empty?
+
+        if type == SEARCH_BY_ISBN
+          get_book_from_search_result(results.first)
+        else
+          results.map { |result| get_book_from_search_result(result) }
+        end
+      end
+
+      # it isn't possible to create a URL for a book given only the ISBN...
+      def url(_book)
+        nil
+      end
+
+      private
+
+      def create_search_uri(search_type, search_term)
+        # bah! very, very similar to the siciliano code! refactor out this duplication
+        search_type_code = { SEARCH_BY_ISBN => 'isbn',
+                             SEARCH_BY_TITLE => 'title',
+                             SEARCH_BY_AUTHORS => 'author',
+                             SEARCH_BY_KEYWORD => 'keywords'
+        }[search_type] or 'keywords'
+
+        search_term_encoded = search_term
+        if search_type == SEARCH_BY_ISBN
+          search_term_encoded = Library.canonicalise_isbn(search_term) # isbn-10
+        else
+          search_term_encoded = CGI.escape(search_term)
+        end
+
+        uri = BASE_SEARCH_URL % [search_type_code, search_term_encoded]
+        log.debug { uri }
+        uri
+      end
+
+      def parse_search_result_data(html)
+        doc = html_to_doc(html)
+        book_search_results = []
+
+        result_divs = doc.search('div.scheda_prodotto')
+        result_divs.each do |div|
+          begin
+            # The layout...
+            # a > img
+            # div.scheda_content
+            #  a[link->productpage] title ##  a.titolo_link
+            #  p (genre I think) ##  !ignore
+            #  a[link->author] author ## a.info
+            #  p.editore (publisher? editor?)
+            #  p Data di pubblicazione: \n     2009
+            #  p.prezzo (price)
+
+            #             cover_url = ''
+            #             cover_images = div/'a/img'
+            #             unless cover_images.empty?
+            #               img = cover_images.first
+            #               image_url = img['src']
+            #               if image_url =~ /^http/
+            #                 cover_url = '' # image_url
+            #               elsif image_url[0..0] != '/'
+            #                 cover_url = "#{SITE}/#{image_url}"
+            #               else
+            #                 cover_url = "#{SITE}#{image_url}"
+            #               end
+            #               log.debug { "Search Cover Image URL #{cover_url}" }
+
+            #             end
+
+            content = div / 'div.scheda_content'
+            title_link = (content / :a).first
+            title = normalize(title_link.inner_text)
+            link_to_description = title_link['href']
+            lookup_url =  "#{SITE}#{link_to_description}"
+
+            authors = []
+            (content / 'a.info').each do |link|
+              authors << normalize(link.inner_text)
+            end
+
+            result = {}
+            result[:author] = authors.first # HACK, what about multiple authors
+            result[:title] = title
+            result[:url] = lookup_url
+
+            publishers = (content / 'p.editore')
+            unless publishers.empty?
+              result[:publisher] = normalize(publishers.first.inner_text)
+            end
+
+            book_search_results << result
+          rescue => ex
+            trace = ex.backtrace.join("\n> ")
+            log.error { "Failed parsing DeaStore search page #{ex.message}\n#{trace}" }
+          end
+        end
+        book_search_results
+      end
+
+      def parse_result_data(html)
+        doc = html_to_doc(html)
+        data = doc % 'div#dati_scheda'
+        # sotto_data_hdr = doc % 'div.sotto_schede/h1.titolo_sotto[text()*="Informazioni generali"]/..'
+        # title
+        title_span = data % 'h1.titolo_scheda'
+        title = normalize(title_span.inner_text)
+        # cover
+        cover_link = nil
+        cover_img = data / 'a/img'
+        unless cover_img.empty?
+          cover_link = cover_img.first['src']
+        end
+        # author(s)
+        authors = []
+        author_span = data % 'span.int_scheda[text()*=Autore]'
+        unless author_span
+          author_span = data % 'span.int_scheda[text()*=cura]' # editor
+        end
+        if author_span
+          author_links = author_span / 'a.info'
+          authors = []
+          author_links.each do |link|
+            authors << normalize(link.inner_html)
+          end
+        end
+        # if author_span
+        #  author_links = author_span/'a.info'
+        #  author_links.each do |link|
+        #    authors << normalize(link.inner_text)
+        #  end
+        # end
+        # publisher
+        publisher_par = data % 'span.int_scheda[text()*=Editore]/..'
+        publisher_link = publisher_par % 'a.info'
+        publisher = normalize(publisher_link.inner_text)
+        # skip 'Collana', (ummm, possibly genre information, Babelfish
+        # says "Necklace")
+        # format
+        format_par = data % 'span.int_scheda[text()*=Formato]/..'
+        format_par.inner_text =~ /:[\s]*(.+)[\s]*$/
+        binding = normalize(Regexp.last_match[1])
+        # year
+        date_par = data % 'span.int_scheda[text()*=Data di pubblicazione]/..'
+        date_par.inner_text =~ /:[\s]*([12][0-9]{3})[\s]*$/
+        publish_year = nil
+        if Regexp.last_match[1]
+          publish_year = Regexp.last_match[1].to_i
+        end
+        isbn_spans = data / 'div.sotto/span.isbn'
+        isbns = []
+        isbn_spans.each do |span|
+          span.inner_text =~ /:[\s]*(.+)[\s]*$/
+          isbns << Regexp.last_match[1]
+        end
+        isbn = nil
+        unless isbns.empty?
+          isbn = Library.canonicalise_isbn(isbns.first)
+        end
+        # Editore & Imprint : as publisher info above...
+        # pages
+        # page_par = data % 'span.int_scheda[text()*=Pagine]/..'
+        # if page_par
+        #  page_par.inner_text =~ /:[\s]*([0-9]+)[\s]*$/
+        #  pages = $1.to_i
+        # end
+        # synopsis_div = doc % 'div.sotto_schede' # exclude the first span though
+        # book = Book.new(title, isbns.first, authors)
+        # if publisher
+        #  book.publisher = Publisher.new(publisher)
+        # end
+        # if format
+        #  book.binding = CoverBinding.new(format, binding_type(format))
+        # end
+        # cover
+        image_url = nil
+        if cover_link
+          if cover_link =~ /^http/
+            # e.g. http://images.btol.com/ContentCafe/Jacket.aspx?\
+            # Return=1&amp;Type=M&amp;Value=9788873641803&amp;password=\
+            # CC70580&amp;userID=DEA40305
+            # seems not to work, or to be blank anyway, so set to nil
+            image_url = nil
+          elsif cover_link[0..0] != '/'
+            image_url = "#{SITE}/#{cover_link}"
+          else
+            image_url = "#{SITE}#{cover_link}"
+          end
+          log.debug { "Cover Image URL:: #{image_url}" }
+        end
+        book = Book.new(title, authors, isbn, publisher, publish_year, binding)
+        return [book, image_url]
+      rescue => ex
+        trace = ex.backtrace.join("\n> ")
+        log.error { "Failed parsing DeaStore product page #{ex.message}\n#{trace}" }
+        return nil
+      end
+
+      def normalize(str)
+        unless str.nil?
+          str = str.squeeze(' ').strip
+        end
+        str
+      end
+    end
+  end
+end

data/lib/alexandria/book_providers/douban.rb

@@ -0,0 +1,135 @@
+# -*- ruby -*-
+#
+# Copyright (C) 2009 Cathal Mc Ginley
+# Copyright (C) 2010 Sun Ning
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+# http://book.douban.com/
+# Douban.com book repository, provides many Chinese books.
+
+# Author: Sun Ning <classicning@gmail.com>, http://sunng.info/
+
+require 'cgi'
+require 'alexandria/net'
+require 'yaml'
+
+module Alexandria
+  class BookProviders
+    class DoubanProvider < GenericProvider
+      include Alexandria::Logging
+
+      SITE = 'http://www.douban.com'
+      BASE_URL = 'http://api.douban.com/book/subjects?q=%s&max-results=5&alt=json'
+
+      def initialize
+        super('Douban', 'Douban (China)')
+        prefs.read
+      end
+
+      def url(_book)
+        nil
+      end
+
+      def search(criterion, type)
+        keyword = criterion
+        request_url = BASE_URL % CGI.escape(keyword)
+
+        search_response = transport.get_response(URI.parse(request_url))
+
+        results = parse_search_result(search_response.body)
+        if results.length == 0
+          raise NoResultsError
+        end
+
+        if type == SEARCH_BY_ISBN
+          return results.first
+        else
+          return results
+        end
+      end
+
+      private
+
+      # The YAML parser in Ruby 1.8.6 chokes on the extremely
+      # compressed inline-style of JSON returned by Douban. (Also, it
+      # doesn't un-escape forward slashes).
+      #
+      # This is a quick-and-dirty method to pre-process the JSON into
+      # YAML-parseable format, so that we don't have to drag in a new
+      # dependency.
+      def json2yaml(json)
+        # insert spaces after : and , except within strings
+        # i.e. when followed by numeral, quote, { or [
+        yaml = json.gsub(/(\:|\,)([0-9'"{\[])/) do |_match|
+          "#{Regexp.last_match[1]} #{Regexp.last_match[2]}"
+        end
+        yaml.gsub!(/\\\//, '/') # unescape forward slashes
+        yaml
+      end
+
+      public
+
+      def parse_search_result(response)
+        book_search_results = []
+        begin
+          # dbresult = JSON.parse(response)
+          dbresult = YAML.load(json2yaml(response))
+          # File.open(",douban.yaml", "wb") {|f| f.write(json2yaml(response)) }
+          if dbresult['opensearch:totalResults']['$t'].to_i > 0
+            for item in dbresult['entry']
+              name = item['title']['$t']
+              isbn = nil
+              publisher = nil
+              pubdate = nil
+              binding = nil
+              for av in item['db:attribute']
+                if av['@name'] == 'isbn13'
+                  isbn = av['$t']
+                end
+                if av['@name'] == 'publisher'
+                  publisher = av['$t']
+                end
+                if av['@name'] == 'pubdate'
+                  pubdate = av['$t']
+                end
+                if av['@name'] == 'binding'
+                  binding = av['$t']
+                end
+              end
+              if item['author']
+                authors = item['author'].map { |a| a['name']['$t'] }
+              else
+                authors = []
+              end
+              image_url = nil
+              for av in item['link']
+                if av['@rel'] == 'image'
+                  image_url = av['@href']
+                end
+              end
+              book = Book.new(name, authors, isbn, publisher, pubdate, binding)
+              book_search_results << [book, image_url]
+            end
+          end
+        rescue => ex
+          log.warn(ex.backtrace.join('\n'))
+        end
+        book_search_results
+      end
+    end
+  end
+end