opener-polarity-tagger 3.3.0 → 3.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcce63ff8ce9916a2387c5d0f48209b203eacbc3834eedcf3b5b4990084ac1ad
-  data.tar.gz: 495e79bfdec921d459ff78e50bdae28a2c6a0a8bc2aed150de27f49a7f99c423
+  metadata.gz: befa570d0fdb2774fee58f8f7f3154c4d944cee2865015411534d2c2c313837a
+  data.tar.gz: 4d95524761db77af92db9ece51dc5ff5bf66744445459a1c3c0b7c570ddb3267
 SHA512:
-  metadata.gz: e51288c630084ce9be780b022ae23012337622b6df30de91962f5c73fd2ef32054435247bf850924a40ca723c3fe1a71fc9f688455b1ea5141acf80959fbe37a
-  data.tar.gz: e8f00bd36a27aab5242d91e3f350d4da94d2661ef5565590e1793e02eb65075cf7dd789682b2cd0e06cf4b84bb21a5d5671d51f7e9d94ec1748cd1c8012f33ac
+  metadata.gz: be4cd1daf7cb8f906f2e91d1bba18e64bc713e5d13c6ec090877b4e079153b8fa818915fcc789f6df1bcfe4735b24af2e7a15dbab5517a7037bd6983eba612c8
+  data.tar.gz: dfd2058e9b98ea482d65e86d4a82d33865a1ef9cd5350deac4853c5e4b3c9cec39b37766b862a0f459c92644fd4e5f916f35ea4d1568ffdddde5d823b6b7b43b

data/core/general-lexicons/EN-lexicon/Sentiment-English-general.xml

@@ -14,6 +14,15 @@
             <Domain/>
          </Sense>
       </LexicalEntry>
+      <LexicalEntry id="" partOfSpeech="noun">
+         <Lemma writtenForm="very comfortable"/>
+         <Sense>
+            <Confidence level="manual"/>
+            <MonolingualExternalRef/>
+            <Sentiment polarity="negative" strength="strong"/>
+            <Domain/>
+         </Sense>
+      </LexicalEntry>
       <LexicalEntry id="" partOfSpeech="noun">
          <Lemma writtenForm="abandonment"/>
          <Sense>

data/lib/opener/polarity_tagger/internal.rb

@@ -9,46 +9,70 @@ module Opener
       DESC        = 'VUA polarity tagger multilanguage'
       LAST_EDITED = '21may2014'
       VERSION     = '1.2'
+      N_WORDS     = 5
 
-      CACHE = LexiconsCache.new
+      CACHE       = LexiconsCache.new
 
       def initialize ignore_pos: false, **params
         @ignore_pos = ignore_pos
       end
 
       def run input, params = {}
-        kaf = KAF::Document.from_xml input
+        kaf         = KAF::Document.from_xml input
 
         @cache_keys = params[:cache_keys] ||= {}
         @cache_keys.merge! lang: kaf.language
+        @cache_keys = @cache_keys.except :property_type
         @map = kaf.map = CACHE[**@cache_keys].lexicons
 
         raise Opener::Core::UnsupportedLanguageError, kaf.language if @map.blank?
 
-        kaf.terms.each do |t|
+        next_index = 0
+        kaf.terms.each_with_index do |t, index|
+          # skip terms when a multi_word_expression is found
+          next if next_index > index
           lemma = t.lemma&.downcase
           text  = t.text.to_s.downcase
           pos   = if @ignore_pos then nil else t.pos end
           attrs = Hashie::Mash.new
 
-          # text matching have priority as sometimes
-          # the lemma provided by Stanza is a different word
-          lexicon, polarity_pos = @map.by_polarity text, pos
-          lexicon, polarity_pos = @map.by_polarity lemma, pos if lexicon.polarity == 'unknown'
-
-          if l = @map.by_negator(text) || @map.by_negator(lemma)
-            lexicon, polarity_pos = l, nil
-            attrs.sentiment_modifier = 'shifter'
-          elsif l = @map.by_intensifier(text) || @map.by_intensifier(lemma)
-            lexicon, polarity_pos = l, nil
-            attrs.sentiment_modifier = 'intensifier'
-          elsif lexicon.polarity != 'unknown'
-            attrs.polarity = lexicon.polarity
+
+          polarity_pos = nil
+
+          if opts = @map.by_negator(text) || @map.by_negator(lemma)
+            lexicon, next_index = get_lexicon(opts, kaf, index)
+            attrs.sentiment_modifier = 'shifter' if lexicon
+          elsif opts = @map.by_intensifier(text) || @map.by_intensifier(lemma)
+            lexicon, next_index = get_lexicon(opts, kaf, index)
+            attrs.sentiment_modifier = 'intensifier' if lexicon
+          end
+
+          unless lexicon
+            # text matching have priority as sometimes
+            # the lemma provided by Stanza is a different word
+            [text, lemma].each do |word|
+              opts, polarity_pos = @map.by_polarity word, pos
+
+              if opts[:multi].size > 0 or opts[:single]
+                lexicon, next_index = get_lexicon opts, kaf, index
+                if lexicon
+                  attrs.polarity = lexicon.polarity
+                  break
+                end
+              end
+            end
           end
 
           if attrs.size > 0
-            attrs.resource = lexicon.resource if lexicon.resource
+            attrs['lexicon-id'] = lexicon.id.to_s  if lexicon&.id
+            attrs.resource      = lexicon.resource if lexicon&.resource
             t.setPolarity attrs, polarity_pos
+            i = index
+            while i < next_index do
+              term = kaf.terms[i]
+              term.setPolarity attrs, polarity_pos
+              i += 1
+            end
           end
         end
 
@@ -57,6 +81,29 @@ module Opener
         kaf.to_xml
       end
 
+      def get_lexicon opts, kaf, index
+        if lexicon = identify_lexicon(kaf.terms[index, N_WORDS], opts.multi)
+          index = index + lexicon.lemma.strip.split(' ').size
+        else
+          lexicon = opts.single
+        end
+
+        [lexicon, index]
+      end
+
+      def identify_lexicon terms, lexicons
+        return unless lexicons.size > 0
+
+        lemma = terms.map{|t| t.lemma&.downcase }.join(' ')
+        text  = terms.map{|t| t.text&.downcase }.join(' ')
+
+        lexicons.each do |lexicon|
+          return lexicon if lemma =~ /^#{Regexp.escape(lexicon.lemma)}($|\s)+/
+          return lexicon if text =~ /^#{Regexp.escape(lexicon.lemma)}($|\s)+/
+        end
+        nil
+      end
+
     end
   end
 end

data/lib/opener/polarity_tagger/lexicon_map.rb

@@ -7,9 +7,7 @@ module Opener
       attr_reader :intensifiers
       attr_reader :with_polarity
 
-      UNKNOWN = Hashie::Mash.new polarity: 'unknown'
-
-      POS_ORDER     = 'NRVGAO'
+      POS_ORDER     = 'ONRVGA'
       DEFAULT_POS   = 'O'
       POS_SHORT_MAP = {
         adj:         'G',
@@ -45,16 +43,22 @@ module Opener
         @intensifiers[lemma]
       end
 
-      def by_polarity lemma, short_pos
-        l = @with_polarity[lemma+short_pos] if short_pos
-        return [l, short_pos] if l
+      def by_polarity lemma, identified_short_pos
+        hash = Hashie::Mash.new multi: (@with_polarity[lemma] || [])
+
+        if identified_short_pos and lexicon = @with_polarity[lemma+identified_short_pos]
+          hash[:single] = lexicon
+          return [hash, identified_short_pos]
+        end
 
         POS_ORDER.chars.each do |short_pos|
-          l = @with_polarity[lemma+short_pos]
-          return [l, short_pos] if l
+          if lexicon = @with_polarity[lemma+short_pos]
+            hash[:single] = lexicon
+            return [hash, identified_short_pos]
+          end
         end
 
-        [UNKNOWN, 'unknown']
+        [hash, 'unknown']
       end
 
       protected
@@ -62,16 +66,33 @@ module Opener
       def map lexicons
         return if blank?
 
-        lexicons.each do |l|
-          next if l.lemma.nil?
+        lexicons.each do |lexicon|
+          next if lexicon.lemma.nil?
 
-          case l.type
-          when 'polarityShifter' then @negators[l.lemma]     = l
-          when 'intensifier'     then @intensifiers[l.lemma] = l
-          else
-            if l.polarity
-              short_pos = POS_SHORT_MAP[l.pos&.to_sym] || DEFAULT_POS
-              @with_polarity[l.lemma+short_pos] = l
+          sub_lexicons = [lexicon]
+          sub_lexicons += lexicon.variants if lexicon.variants
+
+          sub_lexicons.each do |variant|
+            if variant.lemma.strip.include? ' '
+              lemma = variant.lemma.strip.split(' ').first
+              type = :multi
+            else
+              lemma = variant.lemma
+              type = :single
+            end
+
+            if ['polarityShifter', 'intensifier'].include? lexicon.type
+              var = @negators if lexicon.type == 'polarityShifter'
+              var = @intensifiers if lexicon.type == 'intensifier'
+
+              var[lemma] ||= Hashie::Mash.new multi: []
+              if type == :multi
+                var[lemma][:multi] << lexicon
+              else
+                var[lemma][:single] = lexicon
+              end
+            else
+              map_one_polarity lemma, variant, lexicon if lexicon.polarity
             end
           end
         end
@@ -81,6 +102,19 @@ module Opener
         puts "#{@lang}: loaded #{@with_polarity.size} elements with polarity"
       end
 
+      def map_one_polarity lemma, hash, lexicon
+        poses = if hash.poses.present? then hash.poses else [hash.pos] end
+        poses.each do |pos|
+          short_pos = POS_SHORT_MAP[pos&.to_sym] || DEFAULT_POS
+          @with_polarity[lemma] ||= []
+          if hash.lemma.strip.include? ' '
+            @with_polarity[lemma] << lexicon
+          else
+            @with_polarity[lemma+short_pos] = lexicon
+          end
+        end
+      end
+
     end
   end
 end

data/lib/opener/polarity_tagger/lexicons_cache.rb

@@ -15,10 +15,11 @@ module Opener
       end
 
       def [] **params
+        existing = @cache[params]
+        return existing if existing and existing.from > UPDATE_INTERVAL.ago
+
         synchronize do
-          existing = @cache[params]
-          break existing if existing and existing.from > UPDATE_INTERVAL.ago
-          @cache[params] = cache_update existing, **params
+          @cache[params] = cache_update @cache[params], **params
         end
       end
       alias_method :get, :[]

data/lib/opener/polarity_tagger/version.rb

@@ -1,7 +1,7 @@
 module Opener
   class PolarityTagger
 
-    VERSION = '3.3.0'
+    VERSION = '3.5.3'
 
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opener-polarity-tagger
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.5.3
 platform: ruby
 authors:
 - development@olery.com
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-10 00:00:00.000000000 Z
+date: 2021-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opener-daemons
@@ -251,7 +251,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.7.8
+rubygems_version: 2.7.6
 signing_key:
 specification_version: 4
 summary: Polarity tagger for various languages.