opener-polarity-tagger 3.4.0 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e7d99cc90032fe712bac6fa6952d1dc482d7ea5a86d9db9544f17e52d75d68f6
-  data.tar.gz: 8c8a5126f8a63a55bea7062c06d078bc87b0ff9286201956524755fb163e1172
+  metadata.gz: fe952e44b35deff6d30de2994ffff1555c6fb256915dd6931605f2141e29cb9c
+  data.tar.gz: 30cd612208d71ec9d2feea2fea625c445e6c5fdef1bd5f359bd160d4a8acb465
 SHA512:
-  metadata.gz: 185160b82b01008ed03ce4a26e0d93ca18b1aab91e20c57acfe7425fa103c557606e0151e161ad7d36dc7380b2f26c9daae9e9649e4b1f7b65ce29c98f270ccc
-  data.tar.gz: 73f6b42eba61db2004fff08f54a1636767e8e77e93ec817717530accfa9fcd3001097e630c043fea0d3afd462c42b755a03f274c4d74e427c1753cf70fb30648
+  metadata.gz: 94afe1918ac45c3fda4007588ee8a1c4f50ee6568a9bcf83c2b9de50090caf9bcc72494d4d731a2a3c6bbd143cc1a025ca071c18bc5379059316d293d973d1c7
+  data.tar.gz: 20fe782270ce4c1d1815acc9acfc9efd9870bd582435f86625bc5c9d92c09c2563fcd62cfc6a4c735a6696745f42bb253b6d5249c3222e9ad82fa196ffe19499

data/core/general-lexicons/EN-lexicon/Sentiment-English-general.xml

@@ -14,6 +14,15 @@
             <Domain/>
          </Sense>
       </LexicalEntry>
+      <LexicalEntry id="" partOfSpeech="noun">
+         <Lemma writtenForm="very comfortable"/>
+         <Sense>
+            <Confidence level="manual"/>
+            <MonolingualExternalRef/>
+            <Sentiment polarity="negative" strength="strong"/>
+            <Domain/>
+         </Sense>
+      </LexicalEntry>
       <LexicalEntry id="" partOfSpeech="noun">
          <Lemma writtenForm="abandonment"/>
          <Sense>

data/lib/opener/polarity_tagger/internal.rb

@@ -9,15 +9,16 @@ module Opener
       DESC        = 'VUA polarity tagger multilanguage'
       LAST_EDITED = '21may2014'
       VERSION     = '1.2'
+      N_WORDS     = 5
 
-      CACHE = LexiconsCache.new
+      CACHE       = LexiconsCache.new
 
       def initialize ignore_pos: false, **params
         @ignore_pos = ignore_pos
       end
 
       def run input, params = {}
-        kaf = KAF::Document.from_xml input
+        kaf         = KAF::Document.from_xml input
 
         @cache_keys = params[:cache_keys] ||= {}
         @cache_keys.merge! lang: kaf.language
@@ -25,31 +26,52 @@ module Opener
 
         raise Opener::Core::UnsupportedLanguageError, kaf.language if @map.blank?
 
-        kaf.terms.each do |t|
+        next_index = 0
+        kaf.terms.each_with_index do |t, index|
+          # skip terms when a multi_word_expression is found
+          next if next_index > index
           lemma = t.lemma&.downcase
           text  = t.text.to_s.downcase
           pos   = if @ignore_pos then nil else t.pos end
           attrs = Hashie::Mash.new
 
-          # text matching have priority as sometimes
-          # the lemma provided by Stanza is a different word
-          lexicon, polarity_pos = @map.by_polarity text, pos
-          lexicon, polarity_pos = @map.by_polarity lemma, pos if lexicon.polarity == 'unknown'
-
-          if l = @map.by_negator(text) || @map.by_negator(lemma)
-            lexicon, polarity_pos = l, nil
-            attrs.sentiment_modifier = 'shifter'
-          elsif l = @map.by_intensifier(text) || @map.by_intensifier(lemma)
-            lexicon, polarity_pos = l, nil
-            attrs.sentiment_modifier = 'intensifier'
-          elsif lexicon.polarity != 'unknown'
-            attrs.polarity = lexicon.polarity
+
+          polarity_pos = nil
+
+          if opts = @map.by_negator(text) || @map.by_negator(lemma)
+            lexicon, next_index = get_lexicon(opts, kaf, index)
+            attrs.sentiment_modifier = 'shifter' if lexicon
+          elsif opts = @map.by_intensifier(text) || @map.by_intensifier(lemma)
+            lexicon, next_index = get_lexicon(opts, kaf, index)
+            attrs.sentiment_modifier = 'intensifier' if lexicon
+          end
+
+          unless lexicon
+            # text matching have priority as sometimes
+            # the lemma provided by Stanza is a different word
+            [text, lemma].each do |word|
+              opts, polarity_pos = @map.by_polarity word, pos
+
+              if opts[:multi].size > 0 or opts[:single]
+                lexicon, next_index = get_lexicon opts, kaf, index
+                if lexicon
+                  attrs.polarity = lexicon.polarity
+                  break
+                end
+              end
+            end
           end
 
           if attrs.size > 0
-            attrs['lexicon-id'] = lexicon.id.to_s  if lexicon.id
-            attrs.resource      = lexicon.resource if lexicon.resource
+            attrs['lexicon-id'] = lexicon.id.to_s  if lexicon&.id
+            attrs.resource      = lexicon.resource if lexicon&.resource
             t.setPolarity attrs, polarity_pos
+            i = index
+            while i < next_index do
+              term = kaf.terms[i]
+              term.setPolarity attrs, polarity_pos
+              i += 1
+            end
           end
         end
 
@@ -58,6 +80,29 @@ module Opener
         kaf.to_xml
       end
 
+      def get_lexicon opts, kaf, index
+        if lexicon = identify_lexicon(kaf.terms[index, N_WORDS], opts.multi)
+          index = index + lexicon.lemma.strip.split(' ').size
+        else
+          lexicon = opts.single
+        end
+
+        [lexicon, index]
+      end
+
+      def identify_lexicon terms, lexicons
+        return unless lexicons.size > 0
+
+        lemma = terms.map{|t| t.lemma&.downcase }.join(' ')
+        text  = terms.map{|t| t.text&.downcase }.join(' ')
+
+        lexicons.each do |lexicon|
+          return lexicon if lemma =~ /^#{lexicon.lemma}($|\s)+/
+          return lexicon if text =~ /^#{lexicon.lemma}($|\s)+/
+        end
+        nil
+      end
+
     end
   end
 end

data/lib/opener/polarity_tagger/lexicon_map.rb

@@ -7,9 +7,7 @@ module Opener
       attr_reader :intensifiers
       attr_reader :with_polarity
 
-      UNKNOWN = Hashie::Mash.new polarity: 'unknown'
-
-      POS_ORDER     = 'NRVGAO'
+      POS_ORDER     = 'ONRVGA'
       DEFAULT_POS   = 'O'
       POS_SHORT_MAP = {
         adj:         'G',
@@ -45,43 +43,56 @@ module Opener
         @intensifiers[lemma]
       end
 
-      def by_polarity lemma, short_pos
-        l = @with_polarity[lemma+short_pos] if short_pos
-        return [l, short_pos] if l
+      def by_polarity lemma, identified_short_pos
+        hash = Hashie::Mash.new multi: (@with_polarity[lemma] || [])
+
+        if identified_short_pos and lexicon = @with_polarity[lemma+identified_short_pos]
+          hash[:single] = lexicon
+          return [hash, identified_short_pos]
+        end
 
         POS_ORDER.chars.each do |short_pos|
-          l = @with_polarity[lemma+short_pos]
-          return [l, short_pos] if l
+          if lexicon = @with_polarity[lemma+short_pos]
+            hash[:single] = lexicon
+            return [hash, identified_short_pos]
+          end
         end
 
-        [UNKNOWN, 'unknown']
+        [hash, 'unknown']
       end
 
       protected
 
-      def map_one_polarity l
-        poses = if l.poses.present? then l.poses else [l.pos] end
-        poses.each do |pos|
-          short_pos = POS_SHORT_MAP[pos&.to_sym] || DEFAULT_POS
-          @with_polarity[l.lemma+short_pos] = l
-        end
-      end
-
       def map lexicons
         return if blank?
 
-        lexicons.each do |l|
-          next if l.lemma.nil?
+        lexicons.each do |lexicon|
+          next if lexicon.lemma.nil?
 
-          case l.type
-          when 'polarityShifter' then @negators[l.lemma]     = l
-          when 'intensifier'     then @intensifiers[l.lemma] = l
-          else
-            if l.polarity
-              map_one_polarity l
-              l.variants&.each do |v|
-                map_one_polarity v
+          sub_lexicons = [lexicon]
+          sub_lexicons += lexicon.variants if lexicon.variants
+
+          sub_lexicons.each do |variant|
+            if variant.lemma.strip.include? ' '
+              lemma = variant.lemma.strip.split(' ').first
+              type = :multi
+            else
+              lemma = variant.lemma
+              type = :single
+            end
+
+            if ['polarityShifter', 'intensifier'].include? lexicon.type
+              var = @negators if lexicon.type == 'polarityShifter'
+              var = @intensifiers if lexicon.type == 'intensifier'
+
+              var[lemma] ||= Hashie::Mash.new multi: []
+              if type == :multi
+                var[lemma][:multi] << lexicon
+              else
+                var[lemma][:single] = lexicon
               end
+            else
+              map_one_polarity lemma, variant, lexicon if lexicon.polarity
             end
           end
         end
@@ -91,6 +102,19 @@ module Opener
         puts "#{@lang}: loaded #{@with_polarity.size} elements with polarity"
       end
 
+      def map_one_polarity lemma, hash, lexicon
+        poses = if hash.poses.present? then hash.poses else [hash.pos] end
+        poses.each do |pos|
+          short_pos = POS_SHORT_MAP[pos&.to_sym] || DEFAULT_POS
+          @with_polarity[lemma] ||= []
+          if hash.lemma.strip.include? ' '
+            @with_polarity[lemma] << lexicon
+          else
+            @with_polarity[lemma+short_pos] = lexicon
+          end
+        end
+      end
+
     end
   end
 end

data/lib/opener/polarity_tagger/version.rb

@@ -1,7 +1,7 @@
 module Opener
   class PolarityTagger
 
-    VERSION = '3.4.0'
+    VERSION = '3.5.0'
 
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opener-polarity-tagger
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.5.0
 platform: ruby
 authors:
 - development@olery.com
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-25 00:00:00.000000000 Z
+date: 2021-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opener-daemons
@@ -251,7 +251,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.7.8
+rubygems_version: 2.7.6
 signing_key:
 specification_version: 4
 summary: Polarity tagger for various languages.