alexandria-book-collection-manager 0.6.9.pre1

data/lib/alexandria/book_providers/pseudomarc.rb

@@ -0,0 +1,185 @@
+# Copyright (C) 2009 Cathal Mc Ginley
+# Copyright (C) 2010 Martin Sucha
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+module Alexandria
+  # A really simple regex-based parser to grab data out of marc text records.
+  class PseudoMarcParser
+    BNF_FR_MAPPINGS = {
+      title: ['200', 'a'],
+      authors: ['700', 'a'],
+      isbn: ['010', 'a'],
+      publisher: ['210', 'g'],
+      year: ['210', 'd'],
+      binding: ['225', 'a'],
+      notes: ['520', 'a']
+    }
+
+    USMARC_MAPPINGS = {
+      title: ['245', 'a', 'b'],
+      authors: ['100', 'a'],
+      isbn: ['020', 'a'],
+      publisher: ['490', 'a'],
+      year: ['260', 'c'],
+      binding: ['020', 'a'], # listed with isbn here
+      notes: ['520', 'a']
+    }
+
+    def self.get_fields(data, type, stripping, m = USMARC_MAPPINGS)
+      field = ''
+      m[type][1..m[type].length - 1].each do |part|
+        if data.first[part]
+          part_data = data.first[part].strip
+          if part_data =~ stripping
+            part_data = Regexp.last_match[1]
+            part_data = part_data.strip
+          end
+          if field != ''
+            field += ': '
+          end
+          field += part_data
+        end
+      end
+      if field == ''
+        field = nil
+      end
+      field
+    end
+
+    def self.marc_text_to_book(marc, m = USMARC_MAPPINGS)
+      details = marc_text_to_details(marc)
+      unless details.empty?
+        title = nil
+        title_data = details[m[:title][0]]
+        if title_data
+          title_data_all = get_fields(title_data, :title, /(.*)[\/:]$/, m)
+          if title_data_all
+            title = title_data_all
+          end
+        end
+
+        authors = []
+        author_data = details[m[:authors][0]]
+        if author_data
+          author_data.each do |ad|
+            author = ad[m[:authors][1]]
+            if author
+              author = author.strip
+              if author =~ /(.*),$/
+                author = Regexp.last_match[1]
+              end
+              authors << author
+            end
+          end
+        end
+
+        isbn = nil
+        binding = nil
+        isbn_data = details[m[:isbn][0]]
+        if isbn_data
+          if isbn_data.first[m[:isbn][1]] =~ /([-0-9xX]+)/
+            isbn = Regexp.last_match[1]
+          end
+        end
+
+        binding_data = details[m[:binding][0]]
+        if binding_data
+          if binding_data.first[m[:binding][1]] =~ /([a-zA-Z][a-z\s]+[a-z])/
+            binding = Regexp.last_match[1]
+          end
+        end
+
+        publisher = nil
+        publisher_data = details[m[:publisher][0]]
+        if publisher_data
+          publisher = publisher_data.first[m[:publisher][1]]
+        end
+
+        year = nil
+        publication_data = details[m[:year][0]]
+        if publication_data
+          year = publication_data.first[m[:year][1]]
+          if year =~ /(\d+)/
+            year = Regexp.last_match[1].to_i
+          end
+        end
+
+        notes = ''
+        notes_data = details[m[:notes][0]]
+        if notes_data
+          notes_data.each do |note|
+            txt = note[m[:notes][1]]
+            if txt
+              notes += txt
+            end
+          end
+        end
+
+        if title.nil? and isbn.nil?
+          # probably didn't undertand the MARC dialect
+          return nil
+        end
+
+        book = Alexandria::Book.new(title, authors, isbn,
+                                    publisher, year, binding)
+        book.notes = notes unless notes.empty?
+        book
+      end
+    end
+
+    def self.marc_text_to_details(marc)
+      details = {}
+      marc.each_line do |line|
+        if line =~ /(\d+)\s*(.+)/
+          code = Regexp.last_match[1]
+          data = Regexp.last_match[2]
+
+          this_line_data = {}
+
+          # puts code
+          # puts data
+          d_idx = 0
+          while d_idx < data.size
+            d_str = data[d_idx..-1]
+            # puts d_str
+            if (idx = d_str =~ /\$([a-z]) ([^\$]+)/)
+              # puts idx
+              sub_code = Regexp.last_match[1]
+              sub_data = Regexp.last_match[2]
+              this_line_data[sub_code] = sub_data
+              # puts "  " + $1
+              # puts "    " + $2
+              # puts idx
+              d_idx += idx + 2 # (2 extra to push beyond this '$a' etc.)
+            else
+              break
+            end
+          end
+
+          unless this_line_data.empty?
+            unless details.key?(code)
+              details[code] = []
+            end
+            details[code] << this_line_data
+          end
+
+        end
+      end
+      details
+    end
+  end
+end

data/lib/alexandria/book_providers/renaud.rb

@@ -0,0 +1,142 @@
+# Copyright (C) 2005-2006-2006 Mathieu Leduc-Hamel
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+# http://en.wikipedia.org/wiki/Renaud-Bray
+
+require 'net/http'
+require 'cgi'
+
+module Alexandria
+  class BookProviders
+    class RENAUDProvider < GenericProvider
+      include GetText
+      # GetText.bindtextdomain(Alexandria::TEXTDOMAIN, :charset => "UTF-8")
+      BASE_URI = 'http://www.renaud-bray.com/'
+      ACCENTUATED_CHARS = 'áàâäçéèêëíìîïóòôöúùûü'
+
+      def initialize
+        super('RENAUD', 'Renaud-Bray (Canada)')
+      end
+
+      def search(criterion, type)
+        criterion = criterion.convert('ISO-8859-1', 'UTF-8')
+        req = BASE_URI + 'francais/menu/gabarit.asp?Rubrique=&Recherche=&Entete=Livre&Page=Recherche_wsc.asp&OnlyAvailable=false&Tri='
+        #        req = BASE_URI + "francais/menu/gabarit.asp?Rubrique=&Recherche=&Entete=Livre&Page=Recherche_section_wsc.asp&OnlyAvailable=false&Tri="
+        req += case type
+               when SEARCH_BY_ISBN
+                 'ISBN'
+               when SEARCH_BY_TITLE
+                 'Titre'
+               when SEARCH_BY_AUTHORS
+                 'Auteur'
+               when SEARCH_BY_KEYWORD
+                 ''
+               else
+                 raise InvalidSearchTypeError
+               end
+        req += '&Phrase='
+
+        req += CGI.escape(criterion)
+        p req if $DEBUG
+        data = transport.get(URI.parse(req))
+        begin
+          if type == SEARCH_BY_ISBN
+            return to_books(data).pop
+          else
+            results = []
+            to_books(data).each {|book|
+              results << book
+            }
+            while /Suivant/.match(data)
+              md = /Enteterouge\">([\d]*)<\/b>/.match(data)
+              num = md[1].to_i + 1
+              data = transport.get(URI.parse(req + '&PageActuelle=' + num.to_s))
+              to_books(data).each {|book|
+                results << book
+              }
+            end
+            return results
+          end
+        rescue
+          raise NoResultsError
+        end
+      end
+
+      def url(book)
+        #        "http://www.renaud-bray.com/francais/menu/gabarit.asp?Rubrique=&Recherche=&Entete=Livre&Page=Recherche_section_wsc.asp&OnlyAvailable=false&Tri=ISBN&Phrase=" + book.isbn
+        'http://www.renaud-bray.com/francais/menu/gabarit.asp?Rubrique=&Recherche=&Entete=Livre&Page=Recherche_wsc.asp&OnlyAvailable=false&Tri=ISBN&Phrase=' + book.isbn
+      end
+
+      #######
+      private
+      #######
+
+      def to_books(data)
+        data = CGI.unescapeHTML(data)
+        data = data.convert('UTF-8', 'ISO-8859-1')
+        raise NoResultsError if /<strong class="Promotion">Aucun article trouv. selon les crit.res demand.s<\/strong>/.match(data)
+
+        titles = []
+        data.scan(/"(Jeune|Lire)Hyperlien" href.*><strong>([-,'\(\)&\#;\w\s#{ACCENTUATED_CHARS}]*)<\/strong><\/a><br>/).each {|md|
+          titles << md[1].strip
+        }
+        raise if titles.empty?
+        authors = []
+        data.scan(/Nom_Auteur.*><i>([,'.&\#;\w\s#{ACCENTUATED_CHARS}]*)<\/i>/).each {|md|
+          authors2 = []
+          for author in md[0].split('  ')
+            authors2 << author.strip
+          end
+          authors << authors2
+        }
+        raise if authors.empty?
+        isbns = []
+        data.scan(/ISBN : ?<\/td><td>(\d+)/).each {|md|
+          isbns << md[0].strip
+        }
+        raise if isbns.empty?
+        editions = []
+        publish_years = []
+        data.scan(/Parution : <br>(\d{4,}-\d{2,}-\d{2,})/).each {|md|
+          editions << md[0].strip
+          publish_years << md[0].strip.split(/-/)[0].to_i
+        }
+        raise if editions.empty? or publish_years.empty?
+        publishers = []
+        data.scan(/diteur : ([,'.&\#;\w\s#{ACCENTUATED_CHARS}]*)<\/span><br>/).each {|md|
+          publishers << md[0].strip
+        }
+        raise if publishers.empty?
+        book_covers = []
+        data.scan(/(\/ImagesEditeurs\/[\d]*\/([\dX]*-f.(jpg|gif))|\/francais\/suggestion\/images\/livre\/livre.gif)/).each {|md|
+          book_covers << BASE_URI + md[0].strip
+        }
+        raise if book_covers.empty?
+
+        books = []
+        titles.each_with_index {|title, i|
+          books << [Book.new(title, authors[i], isbns[i], publishers[i], publish_years[i], editions[i]),
+                    book_covers[i]]
+          # print books
+        }
+        raise if books.empty?
+
+        books
+      end
+    end
+  end
+end

data/lib/alexandria/book_providers/siciliano.rb

@@ -0,0 +1,271 @@
+# Copyright (C) 2004 Laurent Sansonetti
+# Copyright (C) 2007 Laurent Sansonetti and Marco Costantini
+# Copyright (C) 2009 Cathal Mc Ginley
+# Copyright (C) 2011, 2014 Matijs van Zuijlen
+#
+# Alexandria is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Alexandria is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with Alexandria; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Adapted code from 'bn.rb' (I hope that it works!)
+
+# Almost completely rewritten by Cathal Mc Ginley (21 Feb 2009)
+# based on the new code for Palatina
+
+require 'net/http'
+require 'cgi'
+require 'alexandria/book_providers/web'
+
+module Alexandria
+  class BookProviders
+    class SicilianoProvider < WebsiteBasedProvider
+      include Logging
+
+      SITE = 'http://www.siciliano.com.br'
+
+      # The string interpolations in this URL are the search term and search
+      # type, respectively.
+      BASE_SEARCH_URL = "#{SITE}/pesquisaweb/pesquisaweb.dll/pesquisa?" \
+        '&FIL_ID=102' \
+        '&PALAVRASN1=%s' \
+        '&FILTRON1=%s' \
+        '&ESTRUTN1=0301&ORDEMN2=E'
+
+      def initialize
+        super('Siciliano', 'Livraria Siciliano (Brasil)')
+        # no preferences for the moment
+        prefs.read
+      end
+
+      def get_book_from_search_result(result)
+        log.info { "Fetching book from #{result[:url]}" }
+        html_data = transport.get(URI.parse(result[:url]))
+        parse_result_data(html_data, result)
+      end
+
+      def search(criterion, type)
+        begin
+          criterion = criterion.convert('ISO-8859-1', 'UTF-8') # still needed??
+        rescue GLib::ConvertError
+          log.info { "Cannot search for non-ISO-8859-1 terms at Siciliano : #{criterion}" }
+          raise NoResultsError
+        end
+        trying_again = false
+        begin
+          req = create_search_uri(type, criterion, trying_again)
+          log.debug { "#{name} #{trying_again ? 'retrying ' : ''}request = #{req}" }
+          data = transport.get(URI.parse(req))
+          results = parse_search_result_data(data)
+          raise NoResultsError if results.empty?
+
+          if type == SEARCH_BY_ISBN
+            get_book_from_search_result(results.first)
+          else
+            results.map { |result| get_book_from_search_result(result) }
+          end
+
+        rescue NoResultsError => err
+          if (type == SEARCH_BY_ISBN) and (trying_again == false)
+            trying_again = true
+            retry
+          else
+            raise err
+          end
+        end
+      end
+
+      # the new Siciliano website no longer has direct links to books by their ISBN
+      # (the permalink now seems to be based on the product id)
+      def url(_book)
+        nil
+      end
+
+      private
+
+      def create_search_uri(search_type, search_term, trying_again = false)
+        search_type_code = { SEARCH_BY_ISBN => 'G',
+                             SEARCH_BY_TITLE => 'A',
+                             SEARCH_BY_AUTHORS => 'B',
+                             SEARCH_BY_KEYWORD => 'X'
+        }[search_type] or 'X'
+        search_term_encoded = search_term
+        if search_type == SEARCH_BY_ISBN
+          if trying_again
+            # on second attempt, try ISBN-10...
+            search_term_encoded = Library.canonicalise_isbn(search_term) # isbn-10
+          else
+            # search by ISBN-13 first
+            search_term_encoded = Library.canonicalise_ean(search_term) # isbn-13
+          end
+        else
+          search_term_encoded = CGI.escape(search_term)
+        end
+
+        BASE_SEARCH_URL % [search_term_encoded, search_type_code]
+      end
+
+      def parse_search_result_data(html)
+        # The layout...
+        # td[@class="normal"]
+        #   span[@class="vitrine_nome_produto"]
+        #      a (title and link to 'product page')
+        #   br
+        #   TEXT --> author / publisher
+        #   br
+        #   div[@class="vitrine_preco_por"] (price info)
+
+        doc = html_to_doc(html)
+        book_search_results = []
+        # each result will be a dict with keys :title, :author, :publisher, :url
+
+        list_items = doc.search('div.pesquisa-item-lista-conteudo')
+        list_items.each do |item|
+          begin
+            result = {}
+
+            # author & publisher
+            author_publisher = ''
+            item.children.each do |node|
+              author_publisher += node.to_s if node.text?
+              author_publisher.strip!
+              break unless author_publisher.empty?
+            end
+            author, publisher = author_publisher.split('/')
+            result[:author] = author.strip if author
+            result[:publisher] = publisher.strip if publisher
+
+            # title & url
+            link = item % 'a'
+            result[:title] = link.inner_text.strip
+            link_to_description = link['href']
+            slash = ''
+            unless link_to_description =~ /^\//
+              slash = '/'
+            end
+            result[:url] =  "#{SITE}#{slash}#{link_to_description}"
+
+            book_search_results << result
+          rescue => ex
+            trace = ex.backtrace.join("\n> ")
+            log.error { "Failed parsing Siciliano search page #{ex.message}\n#{trace}" }
+          end
+        end
+
+        book_search_results
+      end
+
+      def parse_result_data(html, search_result)
+        # checked against Siciliano website 21 Feb 2009
+        doc = html_to_doc(html)
+        # title
+        title_div = doc % 'div#conteudo//div.titulo'
+        raise NoResultsError unless title_div
+        title_h = title_div % 'h2'
+        title = title_h.inner_text if title_h
+        # title = first_non_empty_text_node(title_div)
+        # author_spans = doc/'span.rotulo'
+        author_hs = title_div / 'h3.autor'
+        authors = []
+        author_hs.each do |h|
+          authors << h.inner_text.strip
+        end
+        ## synopsis_div = doc % 'div#sinopse'
+        details_div = doc % 'div#tab-caracteristica'
+        details = string_array_to_map(lines_of_text_as_array(details_div))
+        # ISBN
+        isbn =  details['ISBN']
+        ## ean = details["CdBarras"]
+        translator = details['Tradutor']
+        if translator
+          authors << translator
+        end
+        binding = details['Acabamento']
+        publisher = search_result[:publisher]
+        # publish year
+        publish_year = nil
+        edition = details['Edio']
+        if edition
+          if edition =~ /([12][0-9]{3})/ # publication date
+            publish_year = Regexp.last_match[1].to_i
+          end
+        end
+        # cover
+        # ImgSrc[1]="/imagem/imagem.dll?pro_id=1386929&PIM_Id=658849";
+        image_urls = []
+        (doc / 'script').each do |script|
+          next if script.children.nil?
+          script.children.each do |ch|
+            ch_text = ch.to_s
+            if ch_text =~ /ImgSrc\[[\d]\]="(.+)";/
+              img_link = Regexp.last_match[1]
+              image_urls << img_link
+            end
+          end
+        end
+        book = Book.new(title, authors, isbn, publisher, publish_year, binding)
+        result =  [book, image_urls.first]
+        return result
+      rescue => ex
+        trace = ex.backtrace.join("\n> ")
+        log.error { "Failed parsing Siciliano product page #{ex.message}\n#{trace}" }
+        return nil
+      end
+
+      def first_non_empty_text_node(elem)
+        text = ''
+        elem.children.each do |node|
+          next unless node.text?
+          text = node.to_s.strip
+          break unless text.empty?
+        end
+        text
+      end
+
+      def lines_of_text_as_array(elem)
+        lines = []
+        current_text = ''
+        elem.children.each do |e|
+          if e.text?
+            current_text += e.to_s
+          elsif e.name == 'br'
+            lines << current_text.strip
+            current_text = ''
+          else
+            current_text += e.inner_text
+          end
+        end
+        lines << current_text.strip
+        lines.delete('')
+        lines
+      end
+
+      def string_array_to_map(arr)
+        map = {}
+        arr.each do |str|
+          key, val = str.split(':')
+          # a real hack for not handling encoding properly :^)
+          if val
+            map[key.gsub(/[^a-zA-Z]/, '')] = val.strip
+          end
+        end
+        map
+      end
+
+      # def binding_type(binding) # portuguese string
+      #  {"brochura" => :paperback,
+      #    "encadernado" => :hardback}[binding.downcase] or :unknown
+      # end
+    end
+  end
+end