math_metadata_lookup 0.1

data/lib/math_metadata_lookup/entity.rb

@@ -0,0 +1,39 @@
+module MathMetadata
+
+  class Entity
+
+    def initialize( meta={} )
+      @metadata = meta.dup
+    end
+
+    def method_missing( meth, *args )
+      case meth.to_s
+      when /(.*?)=/
+        self[$1] = args.first
+      else
+        self[meth]
+      end
+    end
+
+    def [](key)
+      @metadata[key.to_sym]
+    end
+
+    def []=(key, value)
+      @metadata[key.to_sym] = value
+    end
+
+    def format( f=:ruby )
+      result = self
+
+      case f.to_sym
+      when :text, :html, :xml
+        result = self.send("to_#{f}")
+      end
+
+      result
+    end
+
+  end
+
+end

data/lib/math_metadata_lookup/lookup.rb

@@ -0,0 +1,85 @@
+# -*-: coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+
+module MathMetadata
+
+  # Main class for searching through all sites
+  class Lookup
+    attr_accessor :options
+
+    # :sites can be :all or array of allowed sites ([:mrev, :zbl])
+    def initialize( opts={} )
+      @options = { :sites => :all, :verbose => true }.merge(opts)
+      @sites = []
+    end
+
+    # calls method for each site
+    def method_missing(meth, *args)
+      result = []
+
+      sites = SITES.dup
+      if (@options[:sites] != :all) or @options[:sites].kind_of?(Array)
+        allowed = [@options[:sites]].flatten
+        sites.delete_if{|s| not allowed.include?(s::ID) }
+      end
+
+      sites.each do |klass|
+        site = klass.new(:verbose => @options[:verbose], :nwords => args[0][:nwords])
+
+        entry = {:site => klass::ID, :name => klass::NAME, :url => klass::URL}
+        entry[:result] = site.send(meth, *args)
+
+        result << entry
+      end
+
+      Result.new(result)
+    end
+
+
+    # try to decide what is best result for query and combine results from all sites to one article response
+    def heuristic( args={} )
+      opts = {:threshold => 0.6}.merge(args)
+      result = Result.new
+      
+      # use only authors surnames
+      args_dup = args.dup
+      args_dup[:authors].map!{|a| a =~ /([^,]+)/; $1 ? $1 : a}
+      args_dup[:authors].map!{|a| a =~ /([^ ]+) \S+/; $1 ? $1 : a}
+      args_dup[:nwords] = 2
+      sites = article(args_dup)
+
+      # query article has to contain full names
+      query_article = Article.new( {:title => args[:title].to_s, :authors => args[:authors], :year => args[:year]} )
+      sites.each do |site|
+        site[:result].to_a.each do |article|
+          next if article[:title].to_s.empty?
+          article[:similarity] = query_article.similarity(article)
+        end
+        site[:result].to_a.delete_if{|a| a[:similarity].to_f < opts[:threshold].to_f}
+        if site[:result].to_a.size > 0
+          site[:result].sort!{|a,b| a[:similarity]<=>b[:similarity]}
+          site[:result].reverse!
+          site[:result] = [site[:result].to_a.first]
+        end
+      end
+
+      sites
+    end
+
+
+    # parse reference string and execute heuristic to query for article in databases
+    def reference( args={} )
+      ref = Reference.new args[:reference]
+      pp ref if args[:verbose]
+
+      opts = {:threshold => 0.6}.merge(args)
+      opts[:title] = ref.article[:title]
+      opts[:authors] = ref.article[:authors]
+      opts[:year] = ref.article[:year]
+
+      heuristic opts
+    end
+
+  end # Lookup
+
+end # module

data/lib/math_metadata_lookup/reference.rb

@@ -0,0 +1,122 @@
+# -*-: coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+
+module MathMetadata
+
+  class Reference
+
+    # 1=authors, 2=title, 3=publication, 4=year, 5=range
+    ARTICLE_REFERENCE_1_RE = %r{([^:]+):\s*(.*?),\s*([^,]+),\s*\((\d{4})\)\s*,\s*([^ ]+)\s*.*?}mi
+    # 1=authors, 2=title, 3=publication, 4=range, 5=publisher, 6=place, 7=year
+    ARTICLE_REFERENCE_2_RE = %r{([^:]+):\s*(.*?),\s*(.*?,\s*[^,]+,\s*[^,]+,\s*[^,]+),\s*pp\.\s*([^,]+?),\s*([^,]+),\s*(.*?),\s*(\d{4})\s*.*?}mi
+    # 1=authors, 2=title, 3=range, 4=publication, 5=place, 6=year
+    ARTICLE_REFERENCE_3_RE = %r{([^:]+):\s*(.*?),\s*pp\.\s*([^,]+?),\s*([^,]+),\s*(.*?),\s*(\d{4})}mi
+    # 1=authors, 2=title, 3=publication, 4=publisher, 5=place, 6=year
+    ARTICLE_REFERENCE_4_RE = %r{([^:]+):\s*(.*?),\s*(.*?),\s*([^,]+),\s*([^,]+),\s*(\d{4})\s*.*?}mi
+    # 1=authors, 2=title, 4=publisher, 5=place, 6=year
+    ARTICLE_REFERENCE_5_RE = %r{([^:]+):\s*(.*?),\s*(.*?),\s*([^,]+),\s*(\d{4})\s*.*?}mi
+    # 1=authors, 2=title, 3=publisher, 4=place, 5=year
+    ARTICLE_REFERENCE_6_RE = %r{([^:]+):\s*(.*?),\s*([^,]+),\s*([^,]+),\s*(\d{4})\s*}mi
+    # 1=authors, 2=title, 3=publication, 4=year, 5=range
+    ARTICLE_REFERENCE_7_RE = %r{([^:]+):\s*(.*),\s*(.*?,\s*\d+)\s*\((\d{4})\),\s*([^ ]+)\s*.*?}mi
+    # 1=authors, 2=title, 3=publication, 4=year, 5=range
+    ARTICLE_REFERENCE_8_RE = %r{([^:]+):\s*(.*),\s*(.*?)\s*\((\d{4})\),\s*([^ ]+)\s*.*?}mi
+    # 1=authors, 2=title, 3=publisher, 4=place
+    ARTICLE_REFERENCE_9_RE = %r{([^:]+):\s*(.*?),\s*([^,]+),\s*(.*)}mi
+    # 1=authors, 2=title, 3=publication
+    ARTICLE_REFERENCE_10_RE = %r{([^:]+):\s*(.*?),\s*(.*?)\s*.*?}mi
+    # 1=authors, 2=title, 3=place, 4=year
+    ARTICLE_REFERENCE_11_RE = %r{([^:]+):\s*(.*),(.*?)\s+(\d{4})}mi
+
+
+    attr_accessor :source, :article, :suffix, :number, :reg
+
+    def initialize( str=nil, i=1 )
+      @number = i
+      if str.kind_of?(Article)
+        @source = @suffix = nil
+        @article = str
+      else
+        @source = str
+        @article, @suffix = Reference.parse(str) unless str.to_s.empty?
+      end
+    end
+
+
+    def self.parse( str )
+      article = Article.new
+      rnumber = 0
+      suffix = nil
+      found = []
+      (1..11).each do |j|
+        # 1=authors, 2=title, 3=publication, 4=year, 5=range, 6=id, 7=place, 8=publisher
+        re = eval("Reference::ARTICLE_REFERENCE_#{j}_RE")
+        if str =~ re
+          case j
+          when 1
+            # 1=authors, 2=title, 3=publication, 4=year, 5=range, 6=id
+            found = [$1, $2, $3, $4, MathMetadata.normalize_range($5), nil]
+          when 2
+            # 1=authors, 2=title, 3=publication, 4=range, 5=publisher, 6=place, 7=year, 8=id
+            found = [$1, $2, $3, $7, MathMetadata.normalize_range($4), nil, $6, $5]
+          when 3
+            # 1=authors, 2=title, 3=range, 4=publication, 5=place, 6=year
+            found = [$1, $2, $4, $6, MathMetadata.normalize_range($3), nil, $5]
+          when 4
+            # 1=authors, 2=title, 3=publication, 4=publisher, 5=place, 6=year, 7=id
+            found = [$1, $2, $3, $6, nil, nil, $5, $4]
+          when 5
+            # 1=authors, 2=title, 3=publisher, 4=place, 5=year, 6=id
+            found = [$1, $2, nil, $5, nil, nil, $4, $3]
+          when 6
+            # 1=authors, 2=title, 3=publisher, 4=place, 5=year, 6=id
+            found = [$1, $2, nil, $5, nil, nil, $4, $3]
+          when 7
+            # 1=authors, 2=title, 3=publication, 4=year, 5=range, 6=id
+            found = [$1, $2, $3, $4, MathMetadata.normalize_range($5), nil]
+          when 8
+            # 1=authors, 2=title, 3=publication, 4=year, 5=range, 6=id
+            found = [$1, $2, $3, $4, MathMetadata.normalize_range($5), nil]
+          when 9
+            # 1=authors, 2=title, 3=publisher, 4=place
+            found = [$1, $2, nil, nil, nil, nil, $4, $3]
+          when 10
+            # 1=authors, 2=title, 3=publication, 4=id
+            found = [$1, $2, $3, nil, nil, nil, nil, nil]
+          when 11
+            # 1=authors, 2=title, 3=place, 4=year
+            found = [$1, $2, nil, $4, nil, nil, $3]
+          end
+          rnumber = j
+          break
+        end
+      end
+
+      [:authors, :title, :publication, :year, :range, :id, :place, :publisher].each_with_index do |key, idx|
+        article[key] = found[idx]
+      end
+      article.authors = Reference.split_authors article.authors
+
+      [article, suffix, rnumber]
+    end
+
+
+    def self.split_authors( str )
+      res = [
+        /;\s*/,
+        /,?\s*(?:and|und|et)\s+/,
+        /(\S+,\s*[^,]+),?\s*/
+      ]
+
+      authors = [str]
+      res.each do |re|
+        authors = authors.map{|a| a.to_s.split(re)}.flatten
+      end
+      authors.delete_if{|a| a.strip.empty?}
+
+      authors
+    end
+
+  end
+
+end

data/lib/math_metadata_lookup/result.rb

@@ -0,0 +1,97 @@
+# -*-: coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+
+module MathMetadata
+
+  class Result
+    include Enumerable
+
+    FORMATS = [:ruby, :yaml, :xml, :html, :text]
+
+    def initialize( meta=[] )
+      @metadata = meta
+    end
+
+
+    def <<(val)
+      @metadata << val
+    end
+
+
+    def each
+      @metadata.each do |site|
+        yield site
+      end
+    end
+
+
+    def format( f=:ruby )
+      self.send "to_#{f}"
+    end
+
+
+    def to_html
+      result = ""
+      @metadata.each do |site|
+        result << %~
+<div class="site">
+    <h3>Site: #{site[:name]}</h3>~
+        site[:result].each do |entity|
+          result << entity.to_html
+        end
+        result << %~</div>~
+      end
+      result
+    end
+
+
+    def to_xml
+      result = ""
+
+      result << %~<?xml version="1.0" encoding="utf-8"?>
+<mml>~
+      @metadata.each do |site|
+        result << %~
+    <site name="#{site[:name]}">~
+        site[:result].each do |entity|
+          result << entity.to_xml
+        end
+        result << %~
+    </site>
+~
+      end
+      result << %~</mml>~
+
+      result
+    end
+
+
+    def to_yaml
+      @metadata.to_yaml
+    end
+
+
+    def to_array
+      @metadata
+    end
+
+
+    def to_text
+      result = ""
+      @metadata.each do |site|
+        next unless site[:result]
+        result << "Site: #{site[:name]}\n"
+        result << "URL: #{site[:url]}\n"
+        result << "\n"
+        site[:result].each do |entity|
+          result << entity.to_text
+        end
+        result << "\n"
+      end
+      result
+    end
+    alias :to_str :to_text
+
+  end
+
+end # MathMetadata

data/lib/math_metadata_lookup/site.rb

@@ -0,0 +1,221 @@
+# -*-: coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+
+require 'htmlentities'
+require 'open-uri'
+require 'i18n'
+require 'cgi'
+
+
+module MathMetadata
+
+  SITES = []
+
+  # Abstract class. Inherit in your sites definition.
+  class Site 
+
+    def initialize( opts={} )
+      @options = { :verbose => true }.merge(opts)
+    end
+
+    # register new site class
+    def self.inherited( site )
+      SITES << site
+    end
+
+
+    # search for authors
+    def author( args={} )
+      opts = {:name => nil}.merge(args)
+      anf = author_name_forms opts[:name]
+
+      authors = []
+      anf.each do |af|
+        entry = Author.new({:id => af[1], :preferred => af[0], :forms => af[2]})
+        authors << entry unless entry[:id].to_s.strip.empty?
+      end
+
+      authors
+    end
+
+
+    # search for articles
+    def article( args={} )
+      opts = {:id => nil, :title => "", :year => "", :authors => [], :references => true}.merge(args)
+
+      page = fetch_article(opts)
+      articles = []
+  
+      return metadata unless page
+
+      if list_of_articles?(page)
+        articles = get_article_list(page)
+      else
+        a = get_article(page, opts)
+        articles << a unless a[:title].to_s.strip.empty?
+      end
+  
+      return nil if articles.size == 0
+      articles
+    end
+
+
+  protected
+
+
+    def method_missing(meth, *args)
+      page = args.first
+
+      case meth.to_s
+      when /^list_of_(.*)\?$/
+        re = eval("self.class::LIST_OF_#{$1.upcase}_RE")
+        return page =~ re
+      when /^get_(.*)_m$/
+        re = eval("self.class::#{$1.upcase}_RE")
+        re_s = eval("self.class::#{$1.upcase}S_RE")
+        m, n = args[1,2]
+        m ||= 1
+        n ||= 1
+        res = []
+        page.scan(re_s) do |match|
+          entry = []
+          m.times {|i| entry << match[i].to_s.strip}
+          entry << []
+          match[m].scan(re) do |form|
+            n.times {|i| entry[m] << form[i]}
+          end if match[m]
+          res << entry
+        end 
+        return res
+
+      when /^get_(.*)_s$/
+        res = []
+        what = $1
+        re = eval("self.class::#{what.upcase}_RE")
+        re_s = eval("self.class::#{what.upcase}S_RE")
+        page =~ re_s
+        entries = $1
+        entries.to_s.strip.scan(re) do |match|
+          res << match[0].to_s.strip
+        end
+        return res
+
+      when /^get_(.*)$/
+        match = eval("self.class::#{$1.upcase}_RE").match(page).to_a.map{|x| x.to_s.strip}
+        match.shift
+        return match.first if args[1].to_i <= 1
+        return match
+      end
+    end
+  
+
+    # search for author name forms
+    def author_name_forms( name )
+      forms = []
+
+      page = fetch_author name
+      forms = get_author_m page, 2, 1
+
+      forms
+    end
+
+
+    def get_article_references( page )
+      references = []
+
+      refs = get_article_reference_s page
+
+      i = 0;
+      refs.each do |r|
+        i+=1
+        ref = Reference.new r.gsub(/<.*?>/,'').gsub(/  +/,' ').strip, i
+        references << ref
+      end
+      
+      references
+    end
+
+
+    def get_article_msc( page )
+      mscs = get_article_msc_s page
+      mscs = MathMetadata.normalize_mscs(mscs)
+      mscs
+    end
+
+
+    def get_article( page, opts={} )
+      a = Article.new( {
+        :id => get_article_id(page),
+        :authors => get_article_author_s(page),
+        :msc => get_article_msc(page),
+        :publication => get_article_publication(page),
+        :range => MathMetadata.normalize_range(get_article_range(page)),
+        :year => get_article_year(page),
+        :keywords => get_article_keyword_s(page),
+        :issn => get_article_issn_s(page)
+      } )
+
+      a.title, a.language = get_article_title(page, 2)
+      a.title = a.title.to_s.gsub(/<\/span>/,'')
+      a.references = get_article_references(page) if opts[:references]
+
+      a
+    end
+
+
+    def get_article_list( page )
+      articles = []
+      page.scan(self.class::ARTICLE_ENTRY_RE).each do |match|
+        a = article(:id => match[0]).first
+        articles << a unless a[:title].to_s.strip.empty?
+      end
+      articles
+    end
+
+
+    def nwords(s)
+      s.split(" ")[0...@options[:nwords].to_i].join(" ")
+    end
+
+
+    def fetch_page( url, args={} )
+      opts = {:entities => true}.merge(args)
+  
+      puts "fetching #{url}" if @options[:verbose]
+      page = URI.parse(url).read
+      page = HTMLEntities.decode_entities(page) if page and opts[:entities]
+    
+      page
+    end
+  
+  
+    def fetch_author( name )
+      nn = MathMetadata.normalize_name(name)
+      url = self.class::AUTHOR_URL % URI.escape(nn)
+
+      fetch_page(url)
+    end
+  
+
+    def join_article_authors( authors )
+      authors.collect { |author| URI.escape MathMetadata.normalize_name(author) }.join('; ') || ''
+    end
+  
+    def fetch_article( args={} )
+      opts = {:id => nil, :title => "", :year => "", :authors => []}.merge(args)
+      url = self.class::ARTICLE_ID_URL % URI.escape(opts[:id].to_s.strip)
+      if opts[:id].to_s.strip.empty?
+        author = join_article_authors opts[:authors]
+        title = opts[:title]
+        title = '' if not title.kind_of?(String)
+        title = MathMetadata.normalize_text(title)
+        title = nwords(title) if @options[:nwords]
+        url = self.class::ARTICLE_URL % [URI.escape(title), author, opts[:year].to_s]
+      end
+  
+      fetch_page(url, opts)
+    end
+
+  end # Site
+
+end # Module