mesh-medical-subject-headings 2.3.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 05dc9df68ff77e4f8a3b89e41f4b67a3da727332
-  data.tar.gz: 8f57de7c848aa7e112ae9b3ecf0d34db871eca28
+  metadata.gz: 9c3b985c4316ac7e79bfcb96be4f6c1e96bf880a
+  data.tar.gz: 1929dd7755fa64eefdc9d806d4169cb8eb0a3537
 SHA512:
-  metadata.gz: 345a7a0f904a25a0518a53f7babf30f13a1e4cb05e0862fe79eb1ae1a054b4224803bb8ba4d55dbe1f49575f0fa2f6ed6c5d32ec9163c7f195b025e59fa3ee6d
-  data.tar.gz: 6090811709b5570512708496c2bf468f6ae62ae4b551f95df54e3101b94dc93ac763133d8b5e6b91bf6e0ac5b1978206b32162ab7638b47a57b8481b4ebba77b
+  metadata.gz: 0a2b332c6768577055fb64a0b9db08a3ed13c0d16d87b9f542d556ed97ad33716edc43d41e2de2c6c4571301de4561433f899cb32c406e37bc6b85a3c53ddebd
+  data.tar.gz: 0b1b8ad7dc99b21b4e911760e16c15c44612cabd7f6850959f313d2cecf552f9c2a8fe5bfb8fa410a26b5b9a3952dd1d85d54d5df92f41074339de6bcd2eb73c

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+#3.0.0 / 2014-10-06
+* [FEATURE] Finder methods on MESH::Tree renamed for clarity
+* [FEATURE] Significant performance improvements to Tree.match_in_text
+* [FEATURE] Using google_hash to improve GC performance (best in 2.1.3+)
+* [FEATURE] Heading.entries replaced by heading.structured_entries (first introduced in 2.2.0)
+* [FEATURE] Matching in text now returns structured entry used for match, as well as heading
+* [FEATURE] Entries are now able to find themselves within given text (Entry.match_in_text)
+
 #2.3.0 / 2014-09-30
 * [FEATURE] Significant performance improvements to entity recognition in text
 

data/Gemfile.lock

@@ -1,11 +1,16 @@
 PATH
   remote: .
   specs:
-    mesh-medical-subject-headings (2.3.0)
+    mesh-medical-subject-headings (3.0.0)
+      clarifier
+      google_hash
 
 GEM
   remote: https://rubygems.org/
   specs:
+    clarifier (0.0.3)
+    google_hash (0.8.4)
+      sane (~> 0)
     metaclass (0.0.4)
     mini_portile (0.6.0)
     minitest (5.0.8)
@@ -13,8 +18,11 @@ GEM
       metaclass (~> 0.0.1)
     nokogiri (1.6.2.1)
       mini_portile (= 0.6.0)
+    os (0.9.6)
     rake (10.2.2)
     ruby-prof (0.14.2)
+    sane (0.25.8)
+      os (~> 0)
     yard (0.8.7.4)
 
 PLATFORMS

data/MESH.gemspec

@@ -17,6 +17,8 @@ Gem::Specification.new do |spec|
   spec.executables   = nil
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
+  spec.add_dependency 'google_hash'
+  spec.add_dependency 'clarifier'
 
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'rake'

data/bin/benchmark_match_in_text

@@ -23,29 +23,52 @@ class Numeric
 end
 
 def time_this(name, &block)
-  print "#{name}"
-  STDOUT.flush
+  # print "#{name}"
+  # STDOUT.flush
   start = Time.now.to_f
   result = yield
   finish = Time.now.to_f
-  puts "\t#{(finish - start).duration}"
-  result
+  # puts "\t#{(finish - start).duration}"
+  finish - start
 end
 
-mesh_tree = time_this('Loading MeSH Tree') { MESH::Tree.new }
-time_this('Loading en_gb translation') { mesh_tree.load_translation(:en_gb) }
-time_this('Loading wikipedia') { mesh_tree.load_wikipedia }
+mesh_tree = MESH::Tree.new
+mesh_tree.load_translation(:en_gb)
+mesh_tree.load_wikipedia
 
 json_str = File.new('./example.json').read
 extracted = JSON.parse(json_str)
 
-title_headings = time_this('Matching in title') { mesh_tree.match_in_text(extracted['title']) }
-description_headings = time_this('Matching in description') { mesh_tree.match_in_text(extracted['description']) }
-content_headings = time_this('Matching in content') { mesh_tree.match_in_text(extracted['content']) }
+timings = Hash.new { |h, k| h[k] = [] }
 
-classifier = MESH::Classifier.new()
-classification = time_this('Classifying from matches') { classifier.classify([
-                                                                                 {weight: 10.0, matches: title_headings},
-                                                                                 {weight: 5.0, matches: description_headings},
-                                                                                 {weight: 1.0, matches: content_headings}
-                                                                             ]) }
+(0..10).each do |i|
+  time = time_this('Matching in title') { mesh_tree.match_in_text(extracted['title']) }
+  timings[:title_headings] << time
+  time = time_this('Matching in description') { mesh_tree.match_in_text(extracted['description']) }
+  timings[:description_headings] << time
+  time = time_this('Matching in content') { mesh_tree.match_in_text(extracted['content']) }
+  timings[:content_headings] << time
+end
+
+# (0..10).each do |i|
+#   time = time_this('Matching in title') { mesh_tree.match_in_text_2(extracted['title']) }
+#   timings[:title_entries_2] << time
+#   time = time_this('Matching in description') { mesh_tree.match_in_text_2(extracted['description']) }
+#   timings[:description_entries_2] << time
+#   time = time_this('Matching in content') { mesh_tree.match_in_text_2(extracted['content']) }
+#   timings[:content_entries_2] << time
+# end
+
+timings.each do |k, v|
+  t = v.map { |e| e.round(3) }.join("\t")
+  puts "#{k}\t#{t}"
+  avg = v.inject { |sum, el| sum + el }.to_f / v.size
+  puts "#{k}\t#{avg.round(3)}"
+end
+
+# classifier = MESH::Classifier.new()
+# classification = time_this('Classifying from matches') { classifier.classify([
+#                                                                                  {weight: 10.0, matches: title_headings},
+#                                                                                  {weight: 5.0, matches: description_headings},
+#                                                                                  {weight: 1.0, matches: content_headings}
+#                                                                              ]) }

data/bin/profile_in_text

@@ -42,9 +42,9 @@ extracted = JSON.parse(json_str)
 
 result = RubyProf.profile do
 
-  title_headings = time_this('Matching in title') { mesh_tree.match_in_text(extracted['title']) }
+  # title_headings = time_this('Matching in title') { mesh_tree.match_in_text(extracted['title']) }
   # description_headings = time_this('Matching in description') { mesh_tree.match_in_text(extracted['description']) }
-  # content_headings = time_this('Matching in content') { mesh_tree.match_in_text(extracted['content']) }
+  content_headings = time_this('Matching in content') { mesh_tree.match_in_text(extracted['content']) }
 
   # classifier = MESH::Classifier.new()
   # classification = time_this('Classifying from matches') { classifier.classify([

data/bin/translate

@@ -1,15 +1,25 @@
 #!/usr/bin/env ruby
 
 require_relative '../lib/MESH'
-
-mesh_tree = MESH::Tree.new
 tr = MESH::Translator.new(MESH::Translator.enus_to_engb)
 
-mesh_tree.each do |h|
-  puts "*NEWRECORD"
-  puts "MH = #{tr.translate(h.original_heading)}"
-  puts "MS = #{tr.translate(h.summary)}"
-  h.entries.each { |e| puts "ENTRY = #{tr.translate(e)}" }
-  puts "UI = #{h.unique_id}"
-  puts ''
+filename = File.expand_path('../../data/mesh_data_2014/d2014.bin.gz', __FILE__)
+gzipped_file = File.open(filename)
+file = Zlib::GzipReader.new(gzipped_file)
+
+file.each_line do |line|
+  case
+    when line.start_with?('*NEWRECORD')
+      puts "\n*NEWRECORD"
+    when line.start_with?('UI = ')
+      puts line
+    when line.start_with?('MH = ')
+      puts tr.translate(line)
+    when line.start_with?('MS = ')
+      puts tr.translate(line)
+    when line.start_with?('PRINT ENTRY = ')
+      puts tr.translate(line)
+    when line.start_with?('ENTRY = ')
+      puts tr.translate(line)
+  end
 end

data/lib/MESH.rb

@@ -1,3 +1,5 @@
+require 'clarifier'
+require 'google_hash'
 require 'json'
 require 'set'
 require 'zlib'

data/lib/MESH/entry.rb

@@ -2,10 +2,20 @@ module MESH
 
   class Entry
 
-    attr_accessor :heading, :term, :semantic_types, :semantic_relationship, :lexical_type
+    include Comparable
+    attr_accessor :heading, :term, :semantic_types, :semantic_relationship, :lexical_type, :regex, :case_sensitive,
+                  :downcased, :locales, :loose_match_term
 
-    def initialize(heading, entry_text)
+    @@wordy_characters = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
+
+        def <=> other
+      self.term <=> other.term
+    end
+
+    def initialize(heading, entry_text, locale)
       @heading = heading
+      @locales = Set.new
+      @locales << locale
       @semantic_types = []
       parts = entry_text.split('|')
       if entry_text.include? '|'
@@ -29,6 +39,37 @@ module MESH
       else
         @term = entry_text
       end
+      if /^[A-Z0-9]+$/ =~ @term
+        @regex = /(^|\W)#{Regexp.quote(@term)}(\W|$)/
+        @case_sensitive = true
+      else
+        @regex = /(^|\W)#{Regexp.quote(@term)}(\W|$)/i
+        @case_sensitive = false
+      end
+
+      @downcased = @term.downcase
+      @loose_match_term = Entry.loose_match(@term)
+
+    end
+
+    def self.loose_match(term)
+      term.gsub(/\W+/, ' ').upcase
+    end
+
+    def match_in_text(text, downcased)
+      matches = []
+      return matches if text.nil? || text.empty?
+
+      loose_match = @case_sensitive ? (text.include? @term) : (downcased.include? @downcased)
+      if loose_match
+        text.to_enum(:scan, @regex).map do |m,|
+          match = Regexp.last_match
+          matches << {heading: @heading, matched: self, index: match.offset(0)}
+        end
+      end
+
+      matches
+
     end
 
   end

data/lib/MESH/heading.rb

@@ -4,7 +4,8 @@ module MESH
     @@descriptor_classes = [:make_array_start_at_1, :topical_descriptor, :publication_type, :check_tag, :geographic_descriptor]
 
     include Comparable
-    attr_accessor :unique_id, :tree_numbers, :roots, :parents, :children, :useful, :descriptor_class, :default_locale, :semantic_types, :wikipedia_links, :structured_entries, :forward_references
+    attr_accessor :unique_id, :tree_numbers, :roots, :parents, :children, :useful, :descriptor_class, :default_locale,
+                  :semantic_types, :wikipedia_links, :structured_entries, :forward_references
     attr_reader :linkified_summary
 
     def <=> other
@@ -23,18 +24,18 @@ module MESH
       @summary[locale]
     end
 
+    def entries
+      @structured_entries
+    end
+
     def linkify_summary
       return if summary.nil?
       @linkified_summary = summary.gsub(/[A-Z]+[A-Z,\s-]+[A-Z]+/).each do |text|
-        heading = @tree.find_by_entry(text)
-        heading ? yield(text, heading) : text
+        entry = @tree.find_entry_by_loose_match(text)
+        entry ? yield(text, entry) : text
       end
     end
 
-    def entries(locale = default_locale)
-      @entries[locale] ||= []
-    end
-
     def has_ancestor(heading)
       return false if parents.empty?
       return true if parents.include? heading
@@ -68,6 +69,11 @@ module MESH
 
     def matches(conditions)
       conditions.each do |field, pattern|
+        if field == :entries
+          entries = @structured_entries.select { |entry| pattern =~ entry.term }
+          return !entries.nil? && !entries.empty?
+        end
+
         field_content = self.send(field)
         if field_content.kind_of?(Array)
           return false unless field_content.find { |fc| pattern =~ fc }
@@ -110,7 +116,7 @@ module MESH
           if parts.size > 1
             parts.pop
             parent_tree_number = parts.join '.'
-            parent = @tree.find_by_tree_number(parent_tree_number)
+            parent = @tree.find_heading_by_tree_number(parent_tree_number)
             @parents << parent unless parent.nil? || @parents.include?(parent)
             parent.children << self unless parent.nil? || parent.children.include?(self)
           end
@@ -122,13 +128,15 @@ module MESH
     def connect_to_forward_references
       if !@connected_to_forward_references
         @forward_references = @forward_reference_terms.map do |term|
-          @tree.find_by_original_heading(term)
+          @tree.find_heading_by_main_heading(term)
         end
         @connected_to_forward_references = true
       end
     end
 
-    private
+    def entries_by_term
+      Hash[@structured_entries.map { |entry| [entry.term, entry] }]
+    end
 
     def initialize(tree, default_locale, lines)
       @tree = tree
@@ -141,7 +149,6 @@ module MESH
       @children = []
       @forward_references = []
       @forward_reference_terms = []
-      @entries = {@default_locale => []}
       @structured_entries = []
       @original_heading = {}
       @natural_language_name = {}
@@ -170,16 +177,14 @@ module MESH
           when matches = line.match(/^MH = (.*)/)
             mh = matches[1]
             set_original_heading(mh)
-            @entries[@default_locale] << mh unless @entries.include? mh
+            @structured_entries << MESH::Entry.new(self, mh, default_locale)
             librarian_parts = mh.match(/(.*), (.*)/)
             nln = librarian_parts.nil? ? mh : "#{librarian_parts[2]} #{librarian_parts[1]}"
             set_natural_language_name(nln)
 
           when matches = line.match(/^(?:PRINT )?ENTRY = (.*)/)
             entry = matches[1]
-            term = entry.match(/([^|]+)/)
-            @entries[@default_locale] << term[1] unless @entries.include? term[1]
-            @structured_entries << MESH::Entry.new(self, entry)
+            @structured_entries << MESH::Entry.new(self, entry, default_locale)
 
           when matches = line.match(/^FX = (.*)/)
             @forward_reference_terms << matches[1]
@@ -187,9 +192,46 @@ module MESH
         end
 
       end
-      @entries[@default_locale].sort!
 
     end
+
+
+    def load_translation(lines, locale)
+      new_entries = []
+      lines.each do |line|
+        case
+
+          when matches = line.match(/^MS = (.*)/)
+            set_summary(matches[1], locale)
+
+          when matches = line.match(/^MH = (.*)/)
+            set_original_heading(matches[1], locale)
+            librarian_parts = matches[1].match(/(.*), (.*)/)
+            natural_language_name = librarian_parts.nil? ? matches[1] : "#{librarian_parts[2]} #{librarian_parts[1]}"
+            set_natural_language_name(natural_language_name, locale)
+            entry = new_or_existing_entry(matches[1], locale)
+            new_entries << entry
+
+          when matches = line.match(/^(?:PRINT )?ENTRY = (.*)/)
+            entry = new_or_existing_entry(matches[1], locale)
+            new_entries << entry
+
+        end
+      end
+      new_entries
+    end
+
+    def new_or_existing_entry(term, locale)
+      existing_entries = @structured_entries.select { |entry| entry.term == term }
+      if existing_entries.empty?
+        new_entry = MESH::Entry.new(self, term, locale)
+        @structured_entries << new_entry
+      else
+        new_entry = existing_entries[0]
+        new_entry.locales << locale
+      end
+      new_entry
+    end
   end
 end
 

data/lib/MESH/tree.rb

@@ -3,15 +3,20 @@ module MESH
   class Tree
 
     @@default_locale = :en_us
+    @@sw = Clarifier::StopWords.new()
 
     def initialize
 
-      @headings = []
-      @by_unique_id = {}
-      @by_tree_number = {}
-      @by_original_heading = {}
-      @by_entry = {}
-      @by_entry_word = Hash.new { |h, k| h[k] = Set.new }
+      @headings_last_position = -1
+      @headings = GoogleHashDenseLongToRuby.new
+      @headings_by_unique_id = GoogleHashDenseLongToRuby.new
+      @headings_by_tree_number = GoogleHashDenseLongToRuby.new
+      @headings_by_original_heading = GoogleHashDenseLongToRuby.new
+      @entries_by_term = GoogleHashDenseLongToRuby.new
+      @entries_by_loose_match_term = GoogleHashDenseLongToRuby.new #case insensitive, no punctuation, normalised whitespace
+      # @entries_by_word = Hash.new { |h, k| h[k] = Set.new }
+      @entries_by_first_word = GoogleHashDenseLongToRuby.new
+      # @entries_by_first_word = Hash.new { |h, k| h[k] = Set.new }
       @locales = [@@default_locale]
 
       filename = File.expand_path('../../../data/mesh_data_2014/d2014.bin.gz', __FILE__)
@@ -21,10 +26,26 @@ module MESH
       lines = []
       file.each_line do |line|
         case
-          when line.match(/^\*NEWRECORD$/)
+          when line.start_with?('*NEWRECORD')
             unless lines.empty?
               mh = MESH::Heading.new(self, @@default_locale, lines)
-              add_heading_to_hashes(mh)
+              @headings_last_position += 1
+              @headings[@headings_last_position] = mh
+              @headings_by_unique_id[mh.unique_id.hash] = mh
+              @headings_by_original_heading[mh.original_heading.hash] = mh
+              mh.tree_numbers.each do |tree_number|
+                hash = tree_number.hash
+                raise if @headings_by_tree_number[hash]
+                @headings_by_tree_number[hash] = mh
+              end
+              mh.structured_entries.each do |entry|
+                @entries_by_term[entry.term.hash] = entry
+                @entries_by_loose_match_term[entry.loose_match_term.hash] = entry
+                entry_words = entry.term.downcase.split(/\W+/)
+                hash = entry_words[0].hash
+                @entries_by_first_word[hash] ||= Set.new
+                @entries_by_first_word[hash] << entry
+              end
               lines = [line]
             end
           else
@@ -32,39 +53,12 @@ module MESH
         end
       end
 
-      @headings.each do |heading|
-        heading.connect_to_parents
-        heading.connect_to_forward_references
-      end
-
-    end
-
-    def add_heading_to_hashes(mh)
-      @headings << mh
-      @by_unique_id[mh.unique_id] = mh
-      @by_original_heading[mh.original_heading] = mh
-      add_heading_by_entry_word(mh, mh.original_heading)
-      mh.tree_numbers.each do |tree_number|
-        raise if @by_tree_number[tree_number]
-        @by_tree_number[tree_number] = mh
-      end
-      match_headings = mh.entries.map { |e| entry_match_key(e) }.uniq
-      match_headings.each do |entry|
-        raise if @by_entry[entry]
-        @by_entry[entry] = mh
-        add_heading_by_entry_word(mh, entry)
+      (0..@headings_last_position).each do |i|
+        # @headings.each do |heading|
+        @headings[i].connect_to_parents
+        @headings[i].connect_to_forward_references
       end
-    end
 
-    def add_heading_by_entry_word(mh, entry)
-      entry.split.each do |word|
-        word.downcase!
-        @by_entry_word[word] << mh
-      end
-    end
-
-    def entry_match_key(e)
-      e.strip.upcase
     end
 
     def load_translation(locale)
@@ -73,54 +67,39 @@ module MESH
       gzipped_file = File.open(filename)
       file = Zlib::GzipReader.new(gzipped_file)
 
-      entries = []
-      original_heading = nil
-      natural_language_name = nil
-      summary = nil
       unique_id = nil
+      lines = []
       file.each_line do |line|
 
         case
 
-          when line.match(/^\*NEWRECORD$/)
-            unless unique_id.nil?
-              entries.sort!
-              entries.uniq!
-              if heading = find(unique_id)
-                heading.set_original_heading(original_heading, locale) unless original_heading.nil?
-                heading.set_natural_language_name(natural_language_name, locale) unless natural_language_name.nil?
-                heading.set_summary(summary, locale) unless summary.nil?
-                entries.each do |entry|
-                  heading.entries(locale) << entry
-                  add_heading_by_entry_word(heading, entry)
+          when line.start_with?('*NEWRECORD')
+            unless unique_id.nil? || lines.empty?
+              if heading = find_heading_by_unique_id(unique_id)
+                new_entries = heading.load_translation(lines, locale)
+                new_entries.each do |entry|
+                  @entries_by_term[entry.term.hash] = entry
+                  @entries_by_loose_match_term[entry.loose_match_term.hash] = entry
+                  entry_words = entry.term.downcase.split(/\W+/)
+                  hash = entry_words[0].hash
+                  @entries_by_first_word[hash] ||= Set.new
+                  @entries_by_first_word[hash] << entry
                 end
+              else
+                raise 'Translation provided for missing header'
               end
 
-              entries = []
-              original_heading = nil
-              summary = nil
               unique_id = nil
+              lines = []
             end
 
           when matches = line.match(/^UI = (.*)/)
             unique_id = matches[1]
 
-          when matches = line.match(/^MS = (.*)/)
-            summary = matches[1]
-
-          when matches = line.match(/^MH = (.*)/)
-            mh = matches[1]
-            original_heading = mh
-            entries << mh
-            librarian_parts = mh.match(/(.*), (.*)/)
-            natural_language_name = librarian_parts.nil? ? mh : "#{librarian_parts[2]} #{librarian_parts[1]}"
-
-          when matches = line.match(/^(?:PRINT )?ENTRY = ([^|]+)/)
-            entry = matches[1].chomp
-            entries << entry
-
         end
 
+        lines << line
+
       end
       @locales << locale
     end
@@ -139,9 +118,9 @@ module MESH
 
           when line.match(/^\*NEWRECORD$/)
             unless unique_id.nil?
-              if heading = find(unique_id)
+              if heading = find_heading_by_unique_id(unique_id)
                 wikipedia_links.each do |wl|
-                  wl[:score] = (wl[:score].to_f / heading.entries.length.to_f).round(2)
+                  wl[:score] = (wl[:score].to_f / heading.structured_entries.length.to_f).round(2)
                 end
                 heading.wikipedia_links = wikipedia_links
               end
@@ -165,86 +144,78 @@ module MESH
 
 
     def linkify_summaries &block
-      @headings.each do |h|
+      (0..@headings_last_position).each do |i|
+        h = @headings[i]
+      # @headings.each do |h|
         h.linkify_summary &block
       end
     end
 
-    # NO LONGER COVERED BY TESTS
-    # def translate(locale, tr)
-    #   return if @locales.include? locale
-    #   @headings.each_with_index do |h, i|
-    #     h.set_original_heading(tr.translate(h.original_heading), locale)
-    #     h.set_natural_language_name(tr.translate(h.natural_language_name), locale)
-    #     h.set_summary(tr.translate(h.summary), locale)
-    #     h.entries.each { |entry| h.entries(locale) << tr.translate(entry) }
-    #     h.entries(locale).sort!
-    #   end
-    #
-    #   @locales << locale
-    # end
-
-    def find(unique_id)
-      return @by_unique_id[unique_id]
+    def find_heading_by_unique_id(unique_id)
+      return @headings_by_unique_id[unique_id.hash]
+    end
+
+    def find_heading_by_tree_number(tree_number)
+      return @headings_by_tree_number[tree_number.hash]
     end
 
-    def find_by_tree_number(tree_number)
-      return @by_tree_number[tree_number]
+    def find_heading_by_main_heading(heading)
+      return @headings_by_original_heading[heading.hash]
     end
 
-    def find_by_original_heading(heading)
-      return @by_original_heading[heading]
+    def find_entry_by_term(term)
+      return @entries_by_term[term.hash]
     end
 
-    def find_by_entry(entry)
-      return @by_entry[entry_match_key(entry)]
+    def find_entry_by_loose_match(term)
+      return @entries_by_loose_match_term[Entry.loose_match(term).hash]
     end
 
-    def find_by_entry_word(word)
-      return @by_entry_word[word]
+    def find_entries_by_word(word)
+      return @entries_by_first_word[word.hash]
     end
 
     def where(conditions)
       matches = []
-      @headings.each do |heading|
+      (0..@headings_last_position).each do |i|
+      # @headings.each do |heading|
+        heading = @headings[i]
         matches << heading if heading.matches(conditions)
       end
       matches
     end
 
     def each
-      for i in 0 ... @headings.size
+      (0..@headings_last_position).each do |i|
+        # for i in 0 ... @headings.size
         yield @headings[i] if @headings[i].useful
       end
     end
 
-    def match_in_text(text)
+    def match_in_text (text)
       return [] if text.nil?
       downcased = text.downcase
-      candidate_headings = Set.new
-      downcased.split(/\W+/).uniq.each do |word|
-        candidate_headings.merge(find_by_entry_word(word))
+      candidate_entries = []
+      text_words = @@sw.clarify(downcased).split(/\W+/)
+      text_words.uniq!
+      text_words.each do |word|
+        entries_by_word = find_entries_by_word(word)
+        candidate_entries << entries_by_word.to_a
       end
+      candidate_entries.compact!
+      candidate_entries.flatten!
+      # candidate_entries.uniq! #30% in this uniq
+      candidate_entries.keep_if { |entry| entry.heading.useful }
+      # puts "\n\n****\n#{candidate_entries.length}\n*****\n\n"
       matches = []
-      candidate_headings.each do |heading|
-        next unless heading.useful
-        @locales.each do |locale|
-          heading.entries(locale).each do |entry|
-            if downcased.include? entry.downcase #This is a looser check than the regex but much, much faster
-              if /^[A-Z0-9]+$/ =~ entry
-                regex = /(^|\W)#{Regexp.quote(entry)}(\W|$)/
-              else
-                regex = /(^|\W)#{Regexp.quote(entry)}(\W|$)/i
-              end
-              text.to_enum(:scan, regex).map do |m,|
-                match = Regexp.last_match
-                matches << {heading: heading, matched: entry, index: match.offset(0)}
-              end
-            end
-          end
-        end
+      candidate_entries.each do |entry|
+        entry_matches = entry.match_in_text(text, downcased)
+        matches << entry_matches
       end
-      confirmed_matches = []
+
+      matches.compact!
+      matches.flatten!
+
       matches.combination(2) do |l, r|
         if (r[:index][0] >= l[:index][0]) && (r[:index][1] <= l[:index][1])
           #r is within l
@@ -257,6 +228,13 @@ module MESH
       matches.delete_if { |match| match[:delete] }
     end
 
+    private
+
+
+    def entry_match_key(e)
+      e.strip.upcase
+    end
+
 
   end