mesh-medical-subject-headings 1.2.2 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ebe0318253b106903a11e4084cd1e457e136766e
-  data.tar.gz: d60d3d541858a7dfffa11f4fa96e8e373665e631
+  metadata.gz: 3e0859a3e5c4757eb7babc4072ac97177ae0feee
+  data.tar.gz: 2488bf617fa6761cf5b91c47c7b6a0b0641afca4
 SHA512:
-  metadata.gz: b6ed7232863986263b34bbed703da3f3dba58bda02b68a514af772f847f425b43ea25d757264bc6ee0363edcc33f061262f406defb3b4cd98e4b986da4fdf067
-  data.tar.gz: 7f99f960a64fd84595f1c46076b588c590fb33689464449e8a176888db4ca91a309c356f98f2f443850d94e8e7812cbc07fd2feee1853fd3e83056f071b291b8
+  metadata.gz: 2770ad376f99544f8825eaab301ab457ad29d559beeca42cc0e66fee92bc68f70d6eecdf23775c34de57734a31505e2e44e79461fa052ab73e8eba27738c2fad
+  data.tar.gz: 9be16f17ace608d05d8dca68c407f76e7c450ffd1b79907a00fe20d6879aaa21acbfc1c2b0e5820f9ff04a0b52067f9f5e9464b19639d3b2135b0d6eae08d56d

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+#1.3.0 / 2014-04-10
+* [FEATURE] Released classifier to group and score matched headings
+
 #1.2.2 / 2014-03-05
 * [FEATURE] Optimised match_in_text to find headings much, much faster
 

data/Gemfile.lock

@@ -1,18 +1,18 @@
 PATH
   remote: .
   specs:
-    mesh-medical-subject-headings (1.2.2)
+    mesh-medical-subject-headings (1.3.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    metaclass (0.0.2)
+    metaclass (0.0.4)
     minitest (5.0.8)
     mocha (1.0.0)
       metaclass (~> 0.0.1)
-    rake (10.1.1)
+    rake (10.2.2)
     ruby-prof (0.14.2)
-    yard (0.8.7.3)
+    yard (0.8.7.4)
 
 PLATFORMS
   ruby

data/MESH.gemspec

@@ -4,24 +4,24 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'MESH/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "mesh-medical-subject-headings"
+  spec.name          = 'mesh-medical-subject-headings'
   spec.version       = Mesh::VERSION
-  spec.authors       = ["mmmmmrob"]
-  spec.email         = ["rob@dynamicorange.com"]
-  spec.description   = %q{A ruby gem containing MeSH subject headings (https://www.nlm.nih.gov/mesh/) for use in classifying and entity recognition.}
-  spec.summary       = %q{A ruby gem containing MeSH subject headings (https://www.nlm.nih.gov/mesh/) for use in classifying and entity recognition.}
-  spec.homepage      = ""
-  spec.license       = "AGPL3"
+  spec.authors       = ['Rob Styles']
+  spec.email         = ['rob@dynamicorange.com']
+  spec.description   = 'A ruby gem containing MeSH subject headings (https://www.nlm.nih.gov/mesh/) for use in classifying and entity recognition.'
+  spec.summary       = spec.description
+  spec.homepage      = ''
+  spec.license       = 'AGPL3'
 
   spec.files         = `git ls-files`.split($/)
   spec.executables   = nil
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "mocha"
-  spec.add_development_dependency "yard"
-  spec.add_development_dependency "minitest", "~> 5.0.8"
-  spec.add_development_dependency "ruby-prof"
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'mocha'
+  spec.add_development_dependency 'yard'
+  spec.add_development_dependency 'minitest', '~> 5.0.8'
+  spec.add_development_dependency 'ruby-prof'
 end

data/lib/MESH.rb

@@ -1,2 +1,4 @@
 require 'MESH/version'
+require 'MESH/translator'
 require 'MESH/mesh'
+require 'MESH/classifier'

data/lib/MESH/classifier.rb

@@ -0,0 +1,103 @@
+module MESH
+  class Classifier
+    def classify(weighted_matches)
+
+      weighted_headings = []
+      weighted_matches.each do |wm|
+        wm[:matches].each do |match|
+          weighted_headings << [wm[:weight], match[:heading]]
+        end
+      end
+
+      root_groups = {}
+      weighted_headings.each do |weight, heading|
+        heading.roots.each do |root|
+          root_groups[root] ||= []
+          root_groups[root] << [weight, heading]
+        end
+      end
+
+      chosen = {}
+
+      root_groups.each do |root, weighted_headings|
+        scored = {}
+        weighted_headings.each do |weight, heading|
+          calculate_scores(scored, root, heading, weight)
+          #scored[heading] ||= 0
+          #scored[heading] += weight
+          #heading.parents.each do |p|
+          #  if p.roots.include? root
+          #    scored[p] ||= 0
+          #    scored[p] += weight / 3.0
+          #  end
+          #end
+        end
+        scored.each { |h,s| scored[h] = s.round(3) }
+        scored.delete_if { |h,s| s == 0 }
+        best_score, best_connected = scored.reduce({}) { |h, (k, v)| (h[v] ||= []) << k; h }.max
+        most_specific = best_connected.max_by { |h| h.deepest_position(root) }
+        #chosen[root] = [most_specific, best_score]
+        chosen[root] = [best_score, scored]
+      end
+
+      chosen
+
+    end
+
+    def calculate_scores(scored, root, heading, weight)
+      scored[heading] ||= 0
+      scored[heading] += weight
+      heading.parents.each do |p|
+        if p.roots.include? root
+          calculate_scores(scored, root, p, weight / 3.0)
+        end
+      end
+    end
+
+
+
+
+    #text = "#{document[:title]}\n#{document[:abstract]}\n#{document[:content]}"
+    #matches = MESH::Mesh.match_in_text(text)
+    #headings = matches.map { |m| m[:heading] }
+    #root_groups = headings.reduce({}) do |rg, heading|
+    #  heading.roots.each { |root| (rg[root] ||= []) << heading }
+    #  rg
+    #end
+    #root_groups.reduce({}) do |chosen, (root, candidates)|
+    #  connections = calculate_connections(root, candidates)
+    #  best_score, best_connected = connections.reduce({}) { |h, (k, v)| (h[v] ||= []) << k; h }.max
+    #  most_specific = best_connected.max_by { |h| h.deepest_position }
+    #  chosen[root] = most_specific
+    #  chosen
+    #end
+    #end
+
+    private
+
+    def calculate_connections(root, headings, weight)
+      connections = {}
+      headings.each do |h|
+        add_connection(connections, root, h, weight)
+      end
+      connections
+    end
+
+    def add_connection(connections, root, heading, weight)
+      return unless heading.roots.include? root
+      connections[heading] ||= 0
+      connections[heading] += weight
+      heading.parents.each do |p|
+        connections[p] ||= 0
+        connections[p] += weight
+        #add_connection(connections, root, p, weight)
+      end
+      #heading.siblings.each do |p|
+      #  add_connection(connections, p)
+      #end
+    end
+
+  end
+
+end
+

data/lib/MESH/mesh.rb

@@ -1,73 +1,79 @@
-require_relative 'translator'
-
 module MESH
   class Mesh
 
-    attr_accessor :unique_id, :original_heading, :tree_numbers, :parents, :children, :natural_language_name, :summary, :entries, :useful
+    include Comparable
+    attr_accessor :unique_id, :tree_numbers, :roots, :parents, :children, :useful, :descriptor_class
+
+    def <=> other
+      self.unique_id <=> other.unique_id
+    end
 
-    def original_heading(locale = nil)
-      return @original_heading if locale.nil?
-      @@translator.translate(@original_heading)
+    def original_heading(locale = @@default_locale)
+      return @original_heading[locale]
     end
 
-    def natural_language_name(locale = nil)
-      return @natural_language_name if locale.nil?
-      @@translator.translate(@natural_language_name)
+    def natural_language_name(locale = @@default_locale)
+      return @natural_language_name[locale]
     end
 
-    def summary(locale = nil)
-      return @summary if locale.nil?
-      @@translator.translate(@summary)
+    def summary(locale = @@default_locale)
+      return @summary[locale]
     end
 
-    def entries(locale = nil)
-      return @entries if locale.nil?
-      @entries.map { |entry| @@translator.translate(entry) }.sort
+    def entries(locale = @@default_locale)
+      @entries[locale] ||= []
+      return @entries[locale]
     end
 
     def self.configure(args)
       return if @@configured
       raise ArgumentError.new('MeshHeadingGraph requires a filename in order to configure itself') unless not args[:filename].nil?
+
       gzipped_file = File.open(args[:filename])
       file = Zlib::GzipReader.new(gzipped_file)
+
       current_heading = Mesh.new
       file.each_line do |line|
-        if line.match(/^\*NEWRECORD$/) #Then store the previous record before continuing
-          unless current_heading.unique_id.nil?
-            current_heading.entries.sort!
-            @@headings << current_heading
-            @@by_unique_id[current_heading.unique_id] = current_heading
-            @@by_original_heading[current_heading.original_heading] = current_heading
-            current_heading.tree_numbers.each do |tree_number|
-              @@by_tree_number[tree_number] = current_heading
+
+        case
+
+          when matches = line.match(/^\*NEWRECORD$/)
+            unless current_heading.unique_id.nil?
+              current_heading.entries.sort!
+              @@headings << current_heading
+              @@by_unique_id[current_heading.unique_id] = current_heading
+              @@by_original_heading[current_heading.original_heading] = current_heading
+              current_heading.tree_numbers.each do |tree_number|
+                @@by_tree_number[tree_number] = current_heading
+              end
             end
-          end
-          current_heading = Mesh.new
-        end
+            current_heading = Mesh.new
 
-        matches = line.match(/^UI = (.*)/)
-        current_heading.unique_id = matches[1] unless matches.nil?
+          when matches = line.match(/^UI = (.*)/)
+            current_heading.unique_id = matches[1]
 
-        matches = line.match(/^MN = (.*)/)
-        current_heading.tree_numbers << matches[1] unless matches.nil?
+          when matches = line.match(/^MN = (.*)/)
+            current_heading.tree_numbers << matches[1]
+            current_heading.roots << matches[1][0] unless current_heading.roots.include?(matches[1][0])
 
-        matches = line.match(/^MS = (.*)/)
-        current_heading.summary = matches[1] unless matches.nil?
+          when matches = line.match(/^MS = (.*)/)
+            current_heading.set_summary(matches[1])
 
-        matches = line.match(/^MH = (.*)/)
-        unless matches.nil?
-          mh = matches[1]
-          current_heading.original_heading = mh
-          current_heading.natural_language_name = mh
-          current_heading.entries << mh
-          librarian_parts = mh.match(/(.*), (.*)/)
-          current_heading.natural_language_name = "#{librarian_parts[2]} #{librarian_parts[1]}" unless librarian_parts.nil?
-        end
+          when matches = line.match(/^DC = (.*)/)
+            current_heading.descriptor_class = @@descriptor_classes[matches[1].to_i]
+
+          when matches = line.match(/^MH = (.*)/)
+            mh = matches[1]
+            current_heading.set_original_heading(mh)
+            current_heading.entries << mh
+            librarian_parts = mh.match(/(.*), (.*)/)
+            nln = librarian_parts.nil? ? mh : "#{librarian_parts[2]} #{librarian_parts[1]}"
+            current_heading.set_natural_language_name(nln)
+
+          when matches = line.match(/^(?:PRINT )?ENTRY = ([^|]+)/)
+            entry = matches[1].chomp
+            current_heading.entries << entry
 
-        matches = line.match(/^(?:PRINT )?ENTRY = ([^|]+)/)
-        unless matches.nil?
-          mh = matches[1].chomp
-          current_heading.entries << mh
         end
 
       end
@@ -88,6 +94,19 @@ module MESH
       @@configured = true
     end
 
+    def self.translate(locale, tr)
+      return if @@locales.include? locale
+      @@headings.each_with_index do |h, i|
+        h.set_original_heading(tr.translate(h.original_heading), locale)
+        h.set_natural_language_name(tr.translate(h.natural_language_name), locale)
+        h.set_summary(tr.translate(h.summary), locale)
+        h.entries.each { |entry| h.entries(locale) << tr.translate(entry) }
+        h.entries(locale).sort!
+      end
+
+      @@locales << locale
+    end
+
     def self.find(unique_id)
       raise 'MeshHeadingGraph.configure must be called before use' unless @@configured
       return @@by_unique_id[unique_id]
@@ -118,20 +137,72 @@ module MESH
     end
 
     def self.match_in_text(text)
-      text = text.downcase
+      return [] if text.nil?
+      downcased = text.downcase
       matches = []
       @@headings.each do |heading|
         next unless heading.useful
-        heading.entries.each do |entry|
-          if text.include? entry.downcase #This is a looser check than the regex but much, much faster
-            regex = /(^|\W)#{Regexp.quote(entry)}(\W|$)/i
-            if regex =~ text
-              matches << {heading: heading, matched: entry}
+        @@locales.each do |locale|
+          heading.entries(locale).each do |entry|
+            if downcased.include? entry.downcase #This is a looser check than the regex but much, much faster
+              if /^[A-Z0-9]+$/ =~ entry
+                regex = /(^|\W)#{Regexp.quote(entry)}(\W|$)/
+              else
+                regex = /(^|\W)#{Regexp.quote(entry)}(\W|$)/i
+              end
+              text.to_enum(:scan, regex).map do |m,|
+                matches << {heading: heading, matched: entry, index: $`.size}
+              end
             end
           end
         end
       end
-      matches
+      confirmed_matches = []
+      matches.combination(2) do |l, r|
+        if (r[:index] >= l[:index]) && (r[:index] + r[:matched].length <= l[:index] + l[:matched].length)
+          #r is within l
+          r[:delete] = true
+        elsif (l[:index] >= r[:index]) && (l[:index] + l[:matched].length <= r[:index] + r[:matched].length)
+          #l is within r
+          l[:delete] = true
+        end
+      end
+      matches.delete_if { |match| match[:delete] }
+    end
+
+    def has_ancestor(heading)
+      return false if parents.empty?
+      return true if parents.include? heading
+      in_grandparents = parents.map { |p| p.has_ancestor(heading) }
+      return in_grandparents.include? true
+    end
+
+    def has_descendant(heading)
+      return false if children.empty?
+      return true if children.include? heading
+      in_grandchildren = children.map { |p| p.has_descendant(heading) }
+      return in_grandchildren.include? true
+    end
+
+    def sibling?(heading)
+      common_parents = parents & heading.parents
+      !common_parents.empty?
+    end
+
+    def deepest_position(root = '')
+      return nil if tree_numbers.empty?
+      deepest_tree_number = tree_numbers.max_by { |tn| tn.start_with?(root) ? tn.length : 0 }
+      deepest_tree_number.split('.').length
+    end
+
+    def shallowest_position
+      return nil if tree_numbers.empty?
+      shallowest_tree_number = tree_numbers.min_by { |tn| tn.length }
+      shallowest_tree_number.split('.').length
+    end
+
+    def self.cluster(headings)
+      return headings
     end
 
     def matches(conditions)
@@ -141,6 +212,8 @@ module MESH
           return false unless field_content.find { |fc| pattern =~ fc }
         elsif field_content.is_a?(TrueClass) || field_content.is_a?(FalseClass)
           return false unless field_content == pattern
+        elsif field_content.is_a? Symbol
+          return field_content == pattern
         else
           return false unless pattern =~ field_content
         end
@@ -149,7 +222,19 @@ module MESH
     end
 
     def inspect
-      return "#{@unique_id}, #{@original_heading}"
+      return "#{unique_id}, #{original_heading}, [#{tree_numbers.join(',')}]"
+    end
+
+    def set_original_heading(heading, locale = @@default_locale)
+      @original_heading[locale] = heading
+    end
+
+    def set_natural_language_name(name, locale = @@default_locale)
+      @natural_language_name[locale] = name
+    end
+
+    def set_summary(summary, locale = @@default_locale)
+      @summary[locale] = summary
     end
 
     private
@@ -160,16 +245,24 @@ module MESH
     @@by_tree_number = {}
     @@by_original_heading = {}
     @@default_locale = 'en-US'
-    @@translator = Translator.new
+    @@locales = [@@default_locale]
+    @@us_to_gb = Translator.new(Translator.enus_to_engb)
+    @@descriptor_classes = [:make_array_start_at_1, :topical_descriptor, :publication_type, :check_tag, :geographic_descriptor]
 
     def initialize
       @useful = true
       @tree_numbers = []
+      @roots = []
       @parents = []
       @children = []
-      @entries = []
+      @entries = {}
+      @entries[@@default_locale] = []
+      @original_heading = {}
+      @natural_language_name = {}
+      @summary = {}
     end
 
+
   end
 end