math_metadata_lookup 0.1

data/lib/math_metadata_lookup/sites/mr.rb

@@ -0,0 +1,67 @@
+# -*-: coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+
+module MathMetadata
+
+  # Mathematical Reviews
+  # http://www.ams.org/mr-database
+  class MR < Site
+    ID = :mr
+    NAME = "MathSciNet"
+    URL = "http://www.ams.org/mr-database"
+
+
+    # AUTHOR_URL % "Author, Name"
+    AUTHOR_URL = %~http://www.ams.org/mathscinet/search/authors.html?authorName=%s&Submit=Search~
+
+    AUTHORS_RE = %r{<h1 class="profileHead">(.*)<\/h1>.*?<li>\s*MR Author ID:\s*<b>\s*(\d+)\s*</b></li>(?:.*?<ul class="variations">(.*?)<\/ul>)?}mi
+    AUTHOR_RE = %r{<li>(.*?)<\/li>}mi
+
+
+    ARTICLE_ID_URL = "http://www.ams.org/msnmain?preferred_language=en&pg3=MR&s3=%s&l=20&reference_lists=show&simple_headlines=full&contributed_items=show&redirect=Providence%%2C+RI+USA&Submit=Start+Search&fn=130&form=basicsearch"
+#    ARTICLE_URL = "http://www.ams.org/mathscinet/search/publications.html?pg4=TI&s4=%s&co4=AND&%s&Submit=Search&dr=all&yrop=eq&arg3=%s&dr=pubyear&yearRangeFirst=&yearRangeSecond=&pg8=ET&s8=All&review_format=html"
+    #ARTICLE_URL = "http://www.ams.org/mathscinet/search/publdoc.html?co4=AND&dr=pubyear&pg4=TI&pg8=ET&r=1&review_format=html&s4=%s&%s&All&vfpref=html&yearRangeFirst=&yearRangeSecond=&yrop=eq&arg3=%s"
+    ARTICLE_URL = "http://www.ams.org/mathscinet/search/publications.html?co4=AND&dr=pubyear&pg4=TI&pg8=ET&r=1&review_format=html&s4=%s&%s&All&vfpref=html&yearRangeFirst=&yearRangeSecond=&yrop=eq&arg3=%s"
+
+    LIST_OF_ARTICLES_RE = %r{<strong>Matches:</strong>\s*\d*}mi
+    ARTICLE_ENTRY_RE = %r{<div class="headlineText">\s*<a href="/mathscinet/search/publdoc.html[^"]+">\s*<strong>\s*([^< ]+)\s*</strong>\s*<strong>}mi
+
+    ARTICLE_ID_RE = %r{<strong>(.*?)</strong>}mi
+    ARTICLE_TITLE_RE = %r{<span class="title">(?:<span class="searchHighlight">)?(.*?)</span>\s*(?:<span class="sumlang">\(?(.*?)\)?</span>)?}mi
+    ARTICLE_AUTHORS_RE = %r{<br />(<a href="/mathscinet/search/publications.html[^"]*">.*?</a>)<br />}mi
+    ARTICLE_AUTHOR_RE = %r{<a href="/mathscinet/search/publications.html[^"]*">(.*?)</a>}mi
+    ARTICLE_MSCS_RE = %r{<a href="/mathscinet/search/mscdoc.html\?code=[^"]*">(.*?)</a>}mi
+    ARTICLE_MSC_RE = %r{([^, ]+)}mi
+    ARTICLE_PUBLICATION_RE = %r{<a href="/mathscinet/search/journaldoc\.html\?cn=[^"]*">\s*<em>(.*?)</em>\s*</a>}mi
+    ARTICLE_RANGE_RE = %r{(\d+–\d+)}mi
+    ARTICLE_YEAR_RE = %r{<a href="/mathscinet/search/publications\.html[^"]*">\s*\(?(\d{4})\)?, </a>}mi
+    ARTICLE_ISSNS_RE = %r{(ISSN.*?)<br>}mi
+    ARTICLE_ISSN_RE = %r{ISSN\s*(.........)}mi
+    ARTICLE_KEYWORDS_RE = %r{<p><i>Keywords:</i>\s*(.*?)\s*</p>}mi
+    ARTICLE_KEYWORD_RE = %r{([^;]) ?}mi
+    #ARTICLE_REFERENCES_RE = %r{<center>\s*<strong>\s*References\s*</strong>\s*</center>\s*<ol>\s*(.*?)\s*</ol>}mi
+    ARTICLE_REFERENCES_RE = %r{<center>\s*<strong>\s*References\s*</strong>\s*</center>\s*<ol>\s*(.*?)\s*</ol>}mi
+    ARTICLE_REFERENCE_RE = %r{<li>\s*([^:]+:.*?)\s*</li>}
+
+    #ARTICLE_REFERENCE_RE = %r{([^:]+):(.*?)\s*<span class="bf">\s*(.*?)\s*<\/span>\s*\((\d+)\)\s*(?:,\s*([^ ]+?)\s*<a href="[^"]+"\s*>\s*([^ ]+)\s*.*?)?}mi
+    #
+    def join_article_authors( authors )
+      i = 4
+      authors.collect { |author| 
+        i += 1
+        "pg#{i}=AUCN&s#{i}=#{URI.escape MathMetadata.normalize_name(author)}&co#{i}=AND"
+      }.join("&")
+    end
+
+    def get_article_references( page )
+      refs = super page
+      refs.each do |r|
+        r.source =~ /(MR[^\s]+)/
+        r.article[:id] = $1
+      end
+      refs 
+    end
+
+  end # MRev
+
+end

data/lib/math_metadata_lookup/sites/zbl.rb

@@ -0,0 +1,97 @@
+# -*-: coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+
+require 'rexml/document'
+
+module MathMetadata
+
+  # Zentralblatt
+  # http://www.zentralblatt-math.org/zmath/
+  class ZBL < Site
+    ID = :zbl
+    NAME = "Zentralblatt"
+    URL = "http://www.zentralblatt-math.org/zmath/"
+
+    
+    AUTHOR_URL ="http://www.zentralblatt-math.org/zbmath/authors/?q=%s"
+
+    AUTHORS_RE = %r{<div class="name">\s*<strong>(.*?)</strong>.*?Author-Id:\s*</div>\s*([^ <]+)\s*</div>.*?<div class="table">\s*<div class="title">Spellings:</div>\s*(.*?)\s*</div>}mi
+    AUTHOR_RE = %r{(.*?)\s*\[\d+\](?:;\s*)?}
+    
+    
+    ARTICLE_ID_URL = "http://www.zentralblatt-math.org/zmath/en/search?q=an:%s&type=xml&format=complete"
+    ARTICLE_URL = "http://www.zentralblatt-math.org/zmath/en/search?q=ti:%s%%26%s%%26py:%s&type=xml&format=complete"
+    
+    LIST_OF_ARTICLES_RE = %r{<strong class="middle">Result:</strong>}mi
+    ARTICLE_ENTRY_RE = %r{<span[^>]*?>\s*<a href="\?q=an:([^\&]+)\&format=complete">[^<]+</a>\s*<b>}mi
+
+    ARTICLE_ID_RE = %r{<a href="\?q=an:.*?complete">(.*?)</a>}mi
+    ARTICLE_TITLE_RE = %r{</a><br>(.*?)\.</b>\s*\((.*?)\)<br>}mi
+    ARTICLE_AUTHORS_RE = %r{<br><b>(<a href="\?q=[^"]*">.*?</a>)<br>}mi
+    ARTICLE_AUTHOR_RE = %r{<a href="\?q=[^"]*">(.*?)</a>}mi
+    ARTICLE_MSCS_RE = %r{<dd>(.*?)</dd>}mi
+    ARTICLE_MSC_RE = %r{<a href=".*?">(.*?)</a>}mi
+    ARTICLE_PUBLICATION_RE = %r{<a href="[^"j]*?journals[^"]*">(.*?)</a>}mi
+    ARTICLE_RANGE_RE = %r{</a> \d+(?:-\d+)?,\s*(\d+-\d+).*?ISSN}
+    ARTICLE_YEAR_RE = %r{</a>\s*\d+-\d+, \d+-\d+ \((\d+)\)\.}mi
+    ARTICLE_ISSNS_RE = %r{(ISSN.*?)<br>}mi
+    ARTICLE_ISSN_RE = %r{ISSN\s*(.........)}mi
+    ARTICLE_KEYWORDS_RE = %r{<p><i>Keywords:</i>\s*(.*?)\s*</p>}mi
+    ARTICLE_KEYWORD_RE = %r{([^;]+) ?}mi
+    ARTICLE_REFERENCES_RE = %r{<p><i>Citations:</i>\s*(.*?)\s*</p>}
+    # 1=authors, 2=journal, 3=volume/issue, 4=year, 5=range, 6=ref
+    ARTICLE_REFERENCE_RE = %r{xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx}
+#<p><i>Citations:</i> <a href="?an=0962.76001">Zbl 0962.76001</a>; <a href="?an=0784.46029">Zbl 0784.46029</a>; <a href="?an=0974.46040">Zbl 0974.46040</a></p>
+
+    def article( args={} )
+      opts = {:id => nil, :title => "", :year => "", :authors => [], :references => true, :entities => false}.merge(args)
+      page = fetch_article(opts)
+      xml = ::REXML::Document.new page
+
+      articles = []
+      xml.elements.each("//zbml/answers/rec") do |element|
+        article = Article.new({
+          :id => get_element(element, "an"),
+          :title => get_element(element, "ti"),
+          :authors => get_element(element, "au").split(/;\s*/),
+          :language => get_element(element, "la"),
+          :msc => normalize_mscs(get_element(element,"cc").split(" ")),
+          :year => get_element(element, "py"),
+          :keywords => get_element(element, "ut").split(/;\s*/),
+          :issn => get_element(element, "is").split(/;\s*/),
+          :publication => get_element(element, "so")
+        })
+        article.references = get_article_references(get_element(element, "ci")) if opts[:references]
+        article.publication =~ /,\s*(\d+-\d+)\s*\(\d{4}\)/
+        article.range = $1
+        articles << article
+      end
+
+      articles
+    end
+
+    protected
+
+    def get_element(xml, path)
+      element = xml.elements.each(path){|e|}.first
+      return "" unless element
+      CGI::unescapeHTML(element.children.first.to_s.strip)
+    end
+
+    def join_article_authors( authors )
+      authors.collect { |author| "au:#{URI.escape author}" }.join("%26")
+    end
+
+    def get_article_references( str )
+      ids = str.to_s.split(/;\s*/)
+      references = []
+      ids.each_with_index do |id, idx|
+        references << Reference.new(article(:id => id, :references => false).first, idx+1)
+      end
+      references
+    end
+
+
+  end # ZBL
+
+end

data/lib/math_metadata_lookup/tools.rb

@@ -0,0 +1,110 @@
+# -*-: coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+
+require 'unicode'
+
+module MathMetadata
+
+  class << self
+
+    def levenshtein_distance( s1, s2 )
+      return 1.0 if s1 == s2
+
+      s1u, s2u = s1.split(//u), s2.split(//u)
+      tab = Array.new(s1u.size+1){ Array.new(s2u.size+1){0} }
+
+      (0..s1u.size).each do |i|
+        tab[i][0] = i
+      end
+      (0..s2u.size).each do |j|
+        tab[0][j] = j
+      end
+
+      (1..s2u.size).each do |j|
+        (1..s1u.size).each do |i|
+          if s2u[j-1] == s1u[i-1]
+            tab[i][j] = tab[i-1][j-1]
+          else
+            tab[i][j] = [
+              tab[i-1][j] + 1,
+              tab[i][j-1] + 1,
+              tab[i-1][j-1] + 1
+            ].sort.first
+          end
+        end
+      end
+      1 - (tab.last.last.to_f / ([s1u.size, s2u.size].sort.last))
+    end # levenshtein_distance
+
+
+    def normalize_range( range )
+      range.to_s.gsub(/–|--/,'-')
+    end
+
+
+    def normalize_mscs( mscs )
+      mscs.map{|m| m.split(/,|;/) }.flatten.map{|m| m =~ /\s*\(?([^\s\)\(]+)\)?\s*/; $1}
+    end
+
+
+    def normalize_name( name )
+      # only latin chars
+      trans = latex_to_utf8(name.to_s)
+      trans = I18n.transliterate(trans)
+
+      # remove Jr. 
+      trans.sub! %r{\bjr\.(\b|$)}i, ' '
+
+      # remove abbr.: Rakosnik, J. => Rakosnik, 
+      trans.sub! %r{(\W|^)\w\.}i, ' '
+  
+      # transform: Surname, N.M. => Surname, N. M.
+      trans.gsub( /([^\s,])?\.([^\s,])/, '\1. \2' )
+
+      #MathMetadata.remove_punctuation(trans)
+      trans
+    end
+
+
+    def remove_punctuation( s )
+      str = s.gsub %r{(\w)[.,]+( |$)}i, '\1 '
+      str.gsub! %r{(\s)[.,]+( |$)}i, '\1 '
+      str.strip
+    end
+
+
+    def normalize_text( s )
+      str = latex_to_utf8(s)
+      str = I18n.transliterate(str).downcase
+      str = remove_punctuation(str)
+      str.gsub!(%r{\W+}, ' ')
+      str.gsub!(%r{(?:the|a|of|)\s+}i, ' ')
+      str.strip
+    end
+
+    ACCENT_REPL = {
+      "`" => "\u0300", # grave accent
+      "'" => "\u0301", # acute accent
+      "^" => "\u0302", # circumflex
+      '"' => "\u0308", # umlaut or dieresis
+      "~" => "\u0303", # tilde
+      "H" => "\u030b", # long Hungarian umlaut (double acute)
+      "c" => "\u0327", # cedilla
+      "=" => "\u0304", # macron accent
+      "." => "\u0307", # dot over the letter
+      "r" => "\u030a", # ring over the letter
+      "u" => "\u0306", # breve over the letter
+      "v" => "\u030c"  # caron/hacek ("v") over the letter
+    }
+    
+    def latex_to_utf8( s )
+      str = s.gsub( /\\(.)(?:([a-zA-Z])|\{([a-zA-Z])\}|\{\\([a-zA-Z])\})/ ) do |match|
+        accent = ACCENT_REPL[$1]
+        char = $2 || $3 || $4
+        accent ? Unicode.normalize_KC( char + accent ) : match
+      end
+    end
+ 
+  end # <<self
+
+end # module

data/math_metadata_lookup.gemspec

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+# 
+# @author: Petr Kovar <pejuko@gmail.com>
+
+require 'rubygems'
+require 'find'
+
+spec = Gem::Specification.new do |s|
+  s.platform = Gem::Platform::RUBY
+  s.summary = "Search mathematical reviews sites and fetches metadata about articles."
+  s.homepage = "http://github.com/pejuko/math_metadata_lookup"
+  s.email = "pejuko@gmail.com"
+  s.authors = ["Petr Kovar"]
+  s.name = 'math_metadata_lookup'
+  s.version = '0.1'
+  s.date = Time.now.strftime("%Y-%m-%d")
+  s.add_dependency('i18n', '>= 0.5.0')
+  s.add_dependency('unicode')
+  s.require_path = 'lib'
+  s.files = ["bin/math_metadata_lookup", "README.md", "math_metadata_lookup.gemspec", "TODO", "Rakefile"]
+  s.files += Dir["lib/**/*.rb"]
+  s.executables = ["math_metadata_lookup"]
+  s.description = <<EOF
+This utility/library search mathematical reviews sites and fetches metadata about articles.
+It can return results as one of text, xml, html, yaml or ruby formats.
+EOF
+end
+

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification 
+name: math_metadata_lookup
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  version: "0.1"
+platform: ruby
+authors: 
+- Petr Kovar
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-25 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: i18n
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 5
+        - 0
+        version: 0.5.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: unicode
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+description: |
+  This utility/library search mathematical reviews sites and fetches metadata about articles.
+  It can return results as one of text, xml, html, yaml or ruby formats.
+
+email: pejuko@gmail.com
+executables: 
+- math_metadata_lookup
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/math_metadata_lookup
+- README.md
+- math_metadata_lookup.gemspec
+- TODO
+- Rakefile
+- lib/math_metadata_lookup.rb
+- lib/math_metadata_lookup/site.rb
+- lib/math_metadata_lookup/article.rb
+- lib/math_metadata_lookup/tools.rb
+- lib/math_metadata_lookup/lookup.rb
+- lib/math_metadata_lookup/result.rb
+- lib/math_metadata_lookup/sites/mr.rb
+- lib/math_metadata_lookup/sites/zbl.rb
+- lib/math_metadata_lookup/reference.rb
+- lib/math_metadata_lookup/entity.rb
+- lib/math_metadata_lookup/author.rb
+has_rdoc: true
+homepage: http://github.com/pejuko/math_metadata_lookup
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Search mathematical reviews sites and fetches metadata about articles.
+test_files: []
+