mesh-medical-subject-headings 2.2.1 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd1eb2f1f911abc7f2421de83fb5e7d6c83dc3cb
-  data.tar.gz: 70b0081953e3cdfb68335d8cd33d4841b6424205
+  metadata.gz: 05dc9df68ff77e4f8a3b89e41f4b67a3da727332
+  data.tar.gz: 8f57de7c848aa7e112ae9b3ecf0d34db871eca28
 SHA512:
-  metadata.gz: 8b4d63674735db2a1675056d65446fadec07be1b3f81d0d157008c5073f8c1733b165a2c785f220e9578c8e3e53173657c621abe4ba97da9821fcbc79ede1280
-  data.tar.gz: cd889c0a8a95d462d4df42b26980ce3506d9e2e52a4ad973aac0f45dd0e1a63af3c5a241e812cdf134e63d577074a69766991f937d005e812863f8890cba3709
+  metadata.gz: 345a7a0f904a25a0518a53f7babf30f13a1e4cb05e0862fe79eb1ae1a054b4224803bb8ba4d55dbe1f49575f0fa2f6ed6c5d32ec9163c7f195b025e59fa3ee6d
+  data.tar.gz: 6090811709b5570512708496c2bf468f6ae62ae4b551f95df54e3101b94dc93ac763133d8b5e6b91bf6e0ac5b1978206b32162ab7638b47a57b8481b4ebba77b

data/.gitignore

@@ -12,7 +12,7 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
-
+data/mesh_data_2014/*.bin
 # YARD artifacts
 .yardoc
 _yardoc

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+#2.3.0 / 2014-09-30
+* [FEATURE] Significant performance improvements to entity recognition in text
+
 #2.2.1 / 2014-07-18
 * [FEATURE] Headings now have forward references
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mesh-medical-subject-headings (2.2.1)
+    mesh-medical-subject-headings (2.3.0)
 
 GEM
   remote: https://rubygems.org/

data/bin/benchmark_match_in_text

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+
+require 'MESH'
+
+class Numeric
+  def duration
+    secs, millisecs = self.divmod 1
+    # secs = self.to_int
+    mins = secs / 60
+    hours = mins / 60
+    days = hours / 24
+
+    if days > 0
+      "#{days} days and #{hours % 24} hours"
+    elsif hours > 0
+      "#{hours} hours and #{mins % 60} minutes"
+    elsif mins > 0
+      "#{mins} minutes and #{secs % 60} seconds"
+    elsif secs >= 0
+      "#{(millisecs + secs).round(3)} seconds"
+    end
+  end
+end
+
+def time_this(name, &block)
+  print "#{name}"
+  STDOUT.flush
+  start = Time.now.to_f
+  result = yield
+  finish = Time.now.to_f
+  puts "\t#{(finish - start).duration}"
+  result
+end
+
+mesh_tree = time_this('Loading MeSH Tree') { MESH::Tree.new }
+time_this('Loading en_gb translation') { mesh_tree.load_translation(:en_gb) }
+time_this('Loading wikipedia') { mesh_tree.load_wikipedia }
+
+json_str = File.new('./example.json').read
+extracted = JSON.parse(json_str)
+
+title_headings = time_this('Matching in title') { mesh_tree.match_in_text(extracted['title']) }
+description_headings = time_this('Matching in description') { mesh_tree.match_in_text(extracted['description']) }
+content_headings = time_this('Matching in content') { mesh_tree.match_in_text(extracted['content']) }
+
+classifier = MESH::Classifier.new()
+classification = time_this('Classifying from matches') { classifier.classify([
+                                                                                 {weight: 10.0, matches: title_headings},
+                                                                                 {weight: 5.0, matches: description_headings},
+                                                                                 {weight: 1.0, matches: content_headings}
+                                                                             ]) }

data/bin/example.json

@@ -0,0 +1,2 @@
+
+{ "title": "S4 Heart Sound - Auscultation Review and Summary | Learntheheart.com", "description": "The S4 heart sound is reviewed including auscultation, causes and detailed discussion about the atrial gallop.", "content": "Skip to Main Content Home About Contact ECG Review ECG Basics Intro to ECG Approach to ECG Interpretation Determining HR Determining Rhythm Determining Axis P Wave PR Interval PR Segment Q Wave R Wave S Wave QRS Complex T Wave QT Interval ST Segment TP Segment ECG Reviews / Criteria ECG Quiz ECG Cases ECG Archive Cardiology Review Cardiology Topic Reviews All Cardiology Topics Atrial Fibrillation CAD - STEMI CAD - Stable Angina CAD - Unstable Angina/NSTEMI Heart Sounds Heart Murmurs Heart Failure Aortic Stenosis Aortic Regurgitation Mitral Stenosis Mitral Regurgitation USMLE Review Clinical Trials Acute Coronary Syndrome Coronary Artery Disease Coronary Intervention Congestive Heart Failure Atrial Fibrillation Arrhythmia Hypertension Lipid Disorders Cardiac Surgery Cardiology Pearls Cardiology Mnemonics Cardiology Guidelines Quizzes General Cardiology Expert Cardiology ECG Quiz Multiple Choice Questions Heart Sounds Heart Murmurs Coronary Artery Disease - Stable Angina Coronary Artery Disease - Unstable Angina/NSTEMI Coronary Artery Disease - STEMI Atrial Fibrillation Aortic Stenosis Aortic Regurgitation Mitral Stenosis Mitral Regurgitation Case Questions Coronary Artery Disease - Stable Angina Coronary Artery Disease - Unstable Angina/NSTEMI Coronary Artery Disease - STEMI Atrial Fibrillation Aortic Stenosis Aortic Regurgitation Mitral Stenosis Mitral Regurgitation Jeopardy Games ECG Jeopardy Acute Coronary Syndromes Aortic Stenosis Atrial Fibrillation Blog   Home|Cardiology Review|Cardiology Topic Reviews|Heart Sounds|S4 Heart Sound Print Email /* */ var googletag = googletag || {}; googletag.cmd = googletag.cmd || []; (function() { var gads = document.createElement('script'); gads.async = true; gads.type = 'text/javascript'; var useSSL = 'https:' == document.location.protocol; gads.src = (useSSL ? 'https:' : 'http:') + '//www.googletagservices.com/tag/js/gpt.js'; var node = document.getElementsByTagName('script')[0]; node.parentNode.insertBefore(gads, node); })(); googletag.cmd.push(function() { googletag.defineSlot('/45868137/Heart_Sounds_Review', [160, 600], 'div-gpt-ad-1373795768965-0').addService(googletag.pubads()); googletag.pubads().enableSingleRequest(); googletag.enableServices(); }); googletag.cmd.push(function() { googletag.display('div-gpt-ad-1373795768965-0'); }); S4 Heart Sound The fourth heart sound (S4), also known as the \"atrial gallop\", occurs just before S1 when the atria contract to force blood into the LV. If the LV is non-compliant and atrial contraction forces blood through the AV valves, an S4 is produced by the blood striking the LV. CLINICAL PEARL: A S4 heart sound occurs during active LV filling when atrial contraction forces blood into a non-compliant LV. Therefore any condition that creates a non-compliant LV will produce a S4, while any condition that creates an overly compliant LV will produce a S3 (as described above). A S4 heart sound can be an important sign of diastolic heart failure or active ischemia and is rarely a normal finding. Diastolic heart failure frequently results from severe left ventricular hypertrophy (LVH) resulting in impaired relaxation (compliance) of the LV. In this setting, a S4 is often heard. Also, if a person is actively having myocardial ischemia, adequate ATP can't be synthesized to allow for the release of myosin from actin, thus the myocardium is not able to relax and a S4 will be present. CLINICAL PEARL: A S4 heart sound is often a sign of diastolic heart failure and it is rarely a normal finding (unlike a S3).   Normal LV                      Dilated LV - S3 Present CLINICAL PEARL: If the patient is in atrial fibrillation, the atria are not contracting and it is impossible to have a S4 heart sound. Like S3, the S4 sound is low pitched and best heard at the apex with the patient in the left lateral decubitus position. Comparing the 3rd and 4th heart sounds S3 - \"ventricular gallop\" S4 - \"atrial gallop\" Occurs in early diastole Occurs during passive LV filling May be normal at times Requires a very compliant LV Can be a sign of systolic CHF Occurs in late diastole Occurs during active LV filling Almost always abnormal Requires a non-compliant LV Can be a sign of diastolic CHF   PROCEED TO THE EXTRA HEART SOUND REVIEW Related Links: Introduction to Heart Sounds S1 Heart Sound S2 Heart Sound S3 Heart Sound S4 Heart Sound Extra Heart Sounds Heart Sounds Multiple Choice Questions googletag.cmd.push(function() { googletag.display('div-gpt-ad-1375011221937-1'); }); googletag.cmd.push(function() { googletag.display('div-gpt-ad-1375179175725-0'); }); Site Map Advertise Disclaimer Contact Us !function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs'); © 2014 LearntheHeart.com. All rights reserved. Cardiology and ECG Blog Sign-Up E-mail * close (X) /* */", "url": "http://www.learntheheart.com/cardiology-review/s4-heart-sound/" }

data/bin/profile_in_text

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require 'MESH'
+require 'ruby-prof'
+
+class Numeric
+  def duration
+    secs, millisecs = self.divmod 1
+    # secs = self.to_int
+    mins = secs / 60
+    hours = mins / 60
+    days = hours / 24
+
+    if days > 0
+      "#{days} days and #{hours % 24} hours"
+    elsif hours > 0
+      "#{hours} hours and #{mins % 60} minutes"
+    elsif mins > 0
+      "#{mins} minutes and #{secs % 60} seconds"
+    elsif secs >= 0
+      "#{(millisecs + secs).round(3)} seconds"
+    end
+  end
+end
+
+def time_this(name, &block)
+  # print "#{name}"
+  # STDOUT.flush
+  start = Time.now.to_f
+  result = yield
+  finish = Time.now.to_f
+  # puts "\t#{(finish - start).duration}"
+  result
+end
+
+mesh_tree = time_this('Loading MeSH Tree') { MESH::Tree.new }
+time_this('Loading en_gb translation') { mesh_tree.load_translation(:en_gb) }
+time_this('Loading wikipedia') { mesh_tree.load_wikipedia }
+
+json_str = File.new('./example.json').read
+extracted = JSON.parse(json_str)
+
+result = RubyProf.profile do
+
+  title_headings = time_this('Matching in title') { mesh_tree.match_in_text(extracted['title']) }
+  # description_headings = time_this('Matching in description') { mesh_tree.match_in_text(extracted['description']) }
+  # content_headings = time_this('Matching in content') { mesh_tree.match_in_text(extracted['content']) }
+
+  # classifier = MESH::Classifier.new()
+  # classification = time_this('Classifying from matches') { classifier.classify([
+  #                                                                                  {weight: 10.0, matches: title_headings},
+  #                                                                                  {weight: 5.0, matches: description_headings},
+  #                                                                                  {weight: 1.0, matches: content_headings}
+  #                                                                              ]) }
+end
+
+printer = RubyProf::GraphHtmlPrinter.new(result)
+printer.print(STDOUT)

data/lib/MESH.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'set'
 require 'zlib'
 require 'MESH/version'
 require 'MESH/tree'

data/lib/MESH/tree.rb

@@ -11,6 +11,7 @@ module MESH
       @by_tree_number = {}
       @by_original_heading = {}
       @by_entry = {}
+      @by_entry_word = Hash.new { |h, k| h[k] = Set.new }
       @locales = [@@default_locale]
 
       filename = File.expand_path('../../../data/mesh_data_2014/d2014.bin.gz', __FILE__)
@@ -42,6 +43,7 @@ module MESH
       @headings << mh
       @by_unique_id[mh.unique_id] = mh
       @by_original_heading[mh.original_heading] = mh
+      add_heading_by_entry_word(mh, mh.original_heading)
       mh.tree_numbers.each do |tree_number|
         raise if @by_tree_number[tree_number]
         @by_tree_number[tree_number] = mh
@@ -50,6 +52,14 @@ module MESH
       match_headings.each do |entry|
         raise if @by_entry[entry]
         @by_entry[entry] = mh
+        add_heading_by_entry_word(mh, entry)
+      end
+    end
+
+    def add_heading_by_entry_word(mh, entry)
+      entry.split.each do |word|
+        word.downcase!
+        @by_entry_word[word] << mh
       end
     end
 
@@ -80,7 +90,10 @@ module MESH
                 heading.set_original_heading(original_heading, locale) unless original_heading.nil?
                 heading.set_natural_language_name(natural_language_name, locale) unless natural_language_name.nil?
                 heading.set_summary(summary, locale) unless summary.nil?
-                entries.each { |entry| heading.entries(locale) << entry }
+                entries.each do |entry|
+                  heading.entries(locale) << entry
+                  add_heading_by_entry_word(heading, entry)
+                end
               end
 
               entries = []
@@ -187,6 +200,10 @@ module MESH
       return @by_entry[entry_match_key(entry)]
     end
 
+    def find_by_entry_word(word)
+      return @by_entry_word[word]
+    end
+
     def where(conditions)
       matches = []
       @headings.each do |heading|
@@ -204,8 +221,12 @@ module MESH
     def match_in_text(text)
       return [] if text.nil?
       downcased = text.downcase
+      candidate_headings = Set.new
+      downcased.split(/\W+/).uniq.each do |word|
+        candidate_headings.merge(find_by_entry_word(word))
+      end
       matches = []
-      @headings.each do |heading|
+      candidate_headings.each do |heading|
         next unless heading.useful
         @locales.each do |locale|
           heading.entries(locale).each do |entry|
@@ -216,7 +237,8 @@ module MESH
                 regex = /(^|\W)#{Regexp.quote(entry)}(\W|$)/i
               end
               text.to_enum(:scan, regex).map do |m,|
-                matches << {heading: heading, matched: entry, index: $`.size}
+                match = Regexp.last_match
+                matches << {heading: heading, matched: entry, index: match.offset(0)}
               end
             end
           end
@@ -224,10 +246,10 @@ module MESH
       end
       confirmed_matches = []
       matches.combination(2) do |l, r|
-        if (r[:index] >= l[:index]) && (r[:index] + r[:matched].length <= l[:index] + l[:matched].length)
+        if (r[:index][0] >= l[:index][0]) && (r[:index][1] <= l[:index][1])
           #r is within l
           r[:delete] = true
-        elsif (l[:index] >= r[:index]) && (l[:index] + l[:matched].length <= r[:index] + r[:matched].length)
+        elsif (l[:index][0] >= r[:index][0]) && (l[:index][1] <= r[:index][1])
           #l is within r
           l[:delete] = true
         end

data/lib/MESH/version.rb

@@ -1,3 +1,3 @@
 module Mesh
-  VERSION = '2.2.1'
+  VERSION = '2.3.0'
 end

data/test/tree_test.rb

@@ -43,19 +43,19 @@ module MESH
     def test_find_by_entry
 
       expected_entries = [
-        'Adult Reye Syndrome',
-        'Adult Reye\'s Syndrome',
-        'Fatty Liver with Encephalopathy',
-        'Reye Johnson Syndrome',
-        'Reye Like Syndrome',
-        'Reye Syndrome',
-        'Reye Syndrome, Adult',
-        'Reye\'s Like Syndrome',
-        'Reye\'s Syndrome',
-        'Reye\'s Syndrome, Adult',
-        'Reye\'s-Like Syndrome',
-        'Reye-Johnson Syndrome',
-        'Reye-Like Syndrome'
+          'Adult Reye Syndrome',
+          'Adult Reye\'s Syndrome',
+          'Fatty Liver with Encephalopathy',
+          'Reye Johnson Syndrome',
+          'Reye Like Syndrome',
+          'Reye Syndrome',
+          'Reye Syndrome, Adult',
+          'Reye\'s Like Syndrome',
+          'Reye\'s Syndrome',
+          'Reye\'s Syndrome, Adult',
+          'Reye\'s-Like Syndrome',
+          'Reye-Johnson Syndrome',
+          'Reye-Like Syndrome'
       ]
 
       entries_to_test = expected_entries.flat_map do |e|
@@ -73,6 +73,28 @@ module MESH
       assert_nil @mesh_tree.find_by_entry('foo')
     end
 
+    def test_find_by_entry_word
+      expected_ids = %w(D000003)
+      actual = @mesh_tree.find_by_entry_word('abattoir')
+      actual_ids = actual.map { |mh| mh.unique_id }
+      assert_equal expected_ids, actual_ids, 'Should return all headings with this word in any entry'
+    end
+
+    def test_find_by_entry_word_case_insensitive
+      skip 'find by word does not support case insensitive searches'
+      # expected_ids = %w(D000003)
+      # actual = @mesh_tree.find_by_entry_word('AbaTToir')
+      # actual_ids = actual.map { |mh| mh.unique_id }
+      # assert_equal expected_ids, actual_ids, 'Should return all headings with this word in any entry'
+    end
+
+    def test_find_by_anglicised_entry_word
+      expected_ids = %w(D001471 D004938 D004947 D015154)
+      actual = @mesh_tree.find_by_entry_word('oesophagus')
+      actual_ids = actual.map { |mh| mh.unique_id }
+      assert_equal expected_ids, actual_ids, 'Should return all headings with this word in any entry'
+    end
+
     def test_linkifies_all_summaries
       mesh = MESH::Tree.new
       mesh.linkify_summaries do |text, heading|
@@ -107,7 +129,7 @@ module MESH
         expected = expected_ids.map { |id| @mesh_tree.find(id) }
         matches = @mesh_tree.match_in_text(@example_text)
         actual = matches.map { |match| match[:heading] }.uniq
-        assert_equal expected, actual
+        assert_equal expected.sort, actual.sort
       ensure
         not_useful_ids.each { |id| @mesh_tree.find(id).useful = true }
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mesh-medical-subject-headings
 version: !ruby/object:Gem::Version
-  version: 2.2.1
+  version: 2.3.0
 platform: ruby
 authors:
 - Rob Styles
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-18 00:00:00.000000000 Z
+date: 2014-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -125,8 +125,11 @@ files:
 - MESH.gemspec
 - README.md
 - Rakefile
+- bin/benchmark_match_in_text
+- bin/example.json
 - bin/extract_wikipedia_abstracts
 - bin/match_wikipedia
+- bin/profile_in_text
 - bin/translate
 - data/mesh_data_2014/c2014.bin.gz
 - data/mesh_data_2014/d2014.bin.gz