opener-polarity-tagger 3.2.7 → 3.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e46a3cf60d76757018619e73dca7629c175c2bf091c3291b2935ef0826e7a08a
-  data.tar.gz: 0bb5b5ac8d605478791314260d6bd1a7d709722e1598a354236a62dd5741b0b0
+  metadata.gz: afcf1b19a55cbd0c37e6f6adfba85ad3f376cea5bcabd2c3d5d780d45eb5bd4e
+  data.tar.gz: 86f7473b8ae1cafa47bf78aa5a685280f6a0844065d264c0bb27f944ec3a1c92
 SHA512:
-  metadata.gz: f6759038cb51cfef7aa1dd14aef0d8dda9260dc93f0a532fb9dad03bb3d26bfc1c3f60b163f1e77358b9a80862a14aeb4ad8d01a3f1a6b40118d9896a6ef520d
-  data.tar.gz: 4bdb95248999e753e29b85ea172fdbea1489925bab48227de3018c46736068c0cd91eefa2867ee6f181bdc09a59a8250df4e445ec80d41b5b3cd216398b80b0d
+  metadata.gz: b2eadf5b3e97ff01a8934f03c48e01d0af98209bd37c0884d5dac6e714430eb23bbd878faf036369dca11eadad6f59acff45b9a2271559f95a7f1d1399e78adc
+  data.tar.gz: d7b70f216f1e10b5085a391c91ede0cb88aec903c892592ee5ab3269e821bf333d3a24ec1b474505688b532e64ece201967cbd6537dc554ef885e9971854c6d8

data/core/LexiconMod.py

@@ -45,9 +45,7 @@ def load_lexicons(language, path=None):
 def show_lexicons(language, path=None):
     if path is None:
       path = __module_dir
-    #lexicons, default_id, this_folder, folder_per_lang = load_lexicons(language, path)
-    LexiconSent(language,None,path)
-
+    lexicons, default_id, this_folder, folder_per_lang = load_lexicons(language, path)
     print
     print '#'*30
     print 'Available lexicons for',language

data/core/general-lexicons/EN-lexicon/Sentiment-English-general.xml

@@ -14,6 +14,15 @@
             <Domain/>
          </Sense>
       </LexicalEntry>
+      <LexicalEntry id="" partOfSpeech="noun">
+         <Lemma writtenForm="very comfortable"/>
+         <Sense>
+            <Confidence level="manual"/>
+            <MonolingualExternalRef/>
+            <Sentiment polarity="negative" strength="strong"/>
+            <Domain/>
+         </Sense>
+      </LexicalEntry>
       <LexicalEntry id="" partOfSpeech="noun">
          <Lemma writtenForm="abandonment"/>
          <Sense>

data/lib/opener/polarity_tagger/internal.rb

@@ -9,46 +9,70 @@ module Opener
       DESC        = 'VUA polarity tagger multilanguage'
       LAST_EDITED = '21may2014'
       VERSION     = '1.2'
+      N_WORDS     = 5
 
-      CACHE = LexiconsCache.new
+      CACHE       = LexiconsCache.new
 
       def initialize ignore_pos: false, **params
         @ignore_pos = ignore_pos
       end
 
       def run input, params = {}
-        kaf = KAF::Document.from_xml input
+        kaf         = KAF::Document.from_xml input
 
         @cache_keys = params[:cache_keys] ||= {}
         @cache_keys.merge! lang: kaf.language
+        @cache_keys = @cache_keys.except :property_type
         @map = kaf.map = CACHE[**@cache_keys].lexicons
 
         raise Opener::Core::UnsupportedLanguageError, kaf.language if @map.blank?
 
-        kaf.terms.each do |t|
+        next_index = 0
+        kaf.terms.each_with_index do |t, index|
+          # skip terms when a multi_word_expression is found
+          next if next_index > index
           lemma = t.lemma&.downcase
           text  = t.text.to_s.downcase
           pos   = if @ignore_pos then nil else t.pos end
           attrs = Hashie::Mash.new
 
-          lexicon, polarity_pos = @map.by_polarity lemma, pos
-          lexicon, polarity_pos = @map.by_polarity text, pos if lexicon.polarity == 'unknown'
 
-          if lexicon.polarity != 'unknown'
-            attrs.polarity = lexicon.polarity
-          end
-          if l = @map.by_negator(lemma) || @map.by_negator(text)
-            lexicon, polarity_pos = l, nil
-            attrs.sentiment_modifier = 'shifter'
+          polarity_pos = nil
+
+          if opts = @map.by_negator(text) || @map.by_negator(lemma)
+            lexicon, next_index = get_lexicon(opts, kaf, index)
+            attrs.sentiment_modifier = 'shifter' if lexicon
+          elsif opts = @map.by_intensifier(text) || @map.by_intensifier(lemma)
+            lexicon, next_index = get_lexicon(opts, kaf, index)
+            attrs.sentiment_modifier = 'intensifier' if lexicon
           end
-          if l = @map.by_intensifier(lemma) || @map.by_intensifier(text)
-            lexicon, polarity_pos = l, nil
-            attrs.sentiment_modifier = 'intensifier'
+
+          unless lexicon
+            # text matching have priority as sometimes
+            # the lemma provided by Stanza is a different word
+            [text, lemma].each do |word|
+              opts, polarity_pos = @map.by_polarity word, pos
+
+              if opts[:multi].size > 0 or opts[:single]
+                lexicon, next_index = get_lexicon opts, kaf, index
+                if lexicon
+                  attrs.polarity = lexicon.polarity
+                  break
+                end
+              end
+            end
           end
 
           if attrs.size > 0
-            attrs.resource = lexicon.resource if lexicon.resource
+            attrs['lexicon-id'] = lexicon.id.to_s  if lexicon&.id
+            attrs.resource      = lexicon.resource if lexicon&.resource
             t.setPolarity attrs, polarity_pos
+            i = index
+            while i < next_index do
+              term = kaf.terms[i]
+              term.setPolarity attrs, polarity_pos
+              i += 1
+            end
           end
         end
 
@@ -57,6 +81,29 @@ module Opener
         kaf.to_xml
       end
 
+      def get_lexicon opts, kaf, index
+        if lexicon = identify_lexicon(kaf.terms[index, N_WORDS], opts.multi)
+          index = index + lexicon.lemma.strip.split(' ').size
+        else
+          lexicon = opts.single
+        end
+
+        [lexicon, index]
+      end
+
+      def identify_lexicon terms, lexicons
+        return unless lexicons.size > 0
+
+        lemma = terms.map{|t| t.lemma&.downcase }.join(' ')
+        text  = terms.map{|t| t.text&.downcase }.join(' ')
+
+        lexicons.each do |lexicon|
+          return lexicon if lemma =~ /^#{lexicon.lemma}($|\s)+/
+          return lexicon if text =~ /^#{lexicon.lemma}($|\s)+/
+        end
+        nil
+      end
+
     end
   end
 end

data/lib/opener/polarity_tagger/lexicon_map.rb

@@ -7,9 +7,7 @@ module Opener
       attr_reader :intensifiers
       attr_reader :with_polarity
 
-      UNKNOWN = Hashie::Mash.new polarity: 'unknown'
-
-      POS_ORDER     = 'NRVGAO'
+      POS_ORDER     = 'ONRVGA'
       DEFAULT_POS   = 'O'
       POS_SHORT_MAP = {
         adj:         'G',
@@ -45,17 +43,22 @@ module Opener
         @intensifiers[lemma]
       end
 
-      def by_polarity lemma, short_pos
-        return [@with_polarity[lemma+short_pos] || UNKNOWN, short_pos] if short_pos
+      def by_polarity lemma, identified_short_pos
+        hash = Hashie::Mash.new multi: (@with_polarity[lemma] || [])
+
+        if identified_short_pos and lexicon = @with_polarity[lemma+identified_short_pos]
+          hash[:single] = lexicon
+          return [hash, identified_short_pos]
+        end
 
         POS_ORDER.chars.each do |short_pos|
-          if l = @with_polarity[lemma+short_pos]
-            puts "Found polarify #{l.polarity} for #{lemma} with PoS #{short_pos}"
-            return [l, short_pos]
+          if lexicon = @with_polarity[lemma+short_pos]
+            hash[:single] = lexicon
+            return [hash, identified_short_pos]
           end
         end
 
-        [UNKNOWN, 'unknown']
+        [hash, 'unknown']
       end
 
       protected
@@ -63,16 +66,33 @@ module Opener
       def map lexicons
         return if blank?
 
-        lexicons.each do |l|
-          next if l.lemma.nil?
+        lexicons.each do |lexicon|
+          next if lexicon.lemma.nil?
 
-          case l.type
-          when 'polarityShifter' then @negators[l.lemma]     = l
-          when 'intensifier'     then @intensifiers[l.lemma] = l
-          else
-            if l.polarity
-              short_pos = POS_SHORT_MAP[l.pos&.to_sym] || DEFAULT_POS
-              @with_polarity[l.lemma+short_pos] = l
+          sub_lexicons = [lexicon]
+          sub_lexicons += lexicon.variants if lexicon.variants
+
+          sub_lexicons.each do |variant|
+            if variant.lemma.strip.include? ' '
+              lemma = variant.lemma.strip.split(' ').first
+              type = :multi
+            else
+              lemma = variant.lemma
+              type = :single
+            end
+
+            if ['polarityShifter', 'intensifier'].include? lexicon.type
+              var = @negators if lexicon.type == 'polarityShifter'
+              var = @intensifiers if lexicon.type == 'intensifier'
+
+              var[lemma] ||= Hashie::Mash.new multi: []
+              if type == :multi
+                var[lemma][:multi] << lexicon
+              else
+                var[lemma][:single] = lexicon
+              end
+            else
+              map_one_polarity lemma, variant, lexicon if lexicon.polarity
             end
           end
         end
@@ -82,6 +102,19 @@ module Opener
         puts "#{@lang}: loaded #{@with_polarity.size} elements with polarity"
       end
 
+      def map_one_polarity lemma, hash, lexicon
+        poses = if hash.poses.present? then hash.poses else [hash.pos] end
+        poses.each do |pos|
+          short_pos = POS_SHORT_MAP[pos&.to_sym] || DEFAULT_POS
+          @with_polarity[lemma] ||= []
+          if hash.lemma.strip.include? ' '
+            @with_polarity[lemma] << lexicon
+          else
+            @with_polarity[lemma+short_pos] = lexicon
+          end
+        end
+      end
+
     end
   end
 end

data/lib/opener/polarity_tagger/lexicons_cache.rb

@@ -15,10 +15,11 @@ module Opener
       end
 
       def [] **params
+        existing = @cache[params]
+        return existing if existing and existing.from > UPDATE_INTERVAL.ago
+
         synchronize do
-          existing = @cache[params]
-          break existing if existing and existing.from > UPDATE_INTERVAL.ago
-          @cache[params] = cache_update existing, **params
+          @cache[params] = cache_update @cache[params], **params
         end
       end
       alias_method :get, :[]

data/lib/opener/polarity_tagger/version.rb

@@ -1,7 +1,7 @@
 module Opener
   class PolarityTagger
 
-    VERSION = '3.2.7'
+    VERSION = '3.5.2'
 
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opener-polarity-tagger
 version: !ruby/object:Gem::Version
-  version: 3.2.7
+  version: 3.5.2
 platform: ruby
 authors:
 - development@olery.com
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-09 00:00:00.000000000 Z
+date: 2021-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opener-daemons