opener-polarity-tagger 3.2.6 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a550043e18b17f1e12fd9466f5eee1dd2f806f97f38fe4b0e51e6ac4c5dd6ff1
-  data.tar.gz: ec57806e4fa7559082aeccf9bb0e1492ef77d93f86d411549f5513282e10ce64
+  metadata.gz: 5bc937332f3126f240a402051ede1ec3d50c753c65a09e147d4d99446be67bd7
+  data.tar.gz: deb5934edffe4496b83d57d12d0ad2c1dc0ebff77bfbaaa1509602f2a80ae7db
 SHA512:
-  metadata.gz: 1b53a75fccce4af956ab6b334406896ce6489967067e9abc1b529eb2ece9a1dbe167e8a567e6f938292d945a0e174d1c0b79305ce301e477549fd2161a03361c
-  data.tar.gz: 5b8bd573afe2eb9fb41002e8d62949c890893395d711ad98220df2e159220d54caac8d70f5c5134898e821213ea6c05e1d1b226a4c50336361fc33f528b6cdb7
+  metadata.gz: 6430b221c2b87e2e934f442535d4bfae2dae80b520e0703864b8ff3b3a407fd36954b0538fa6a1f4d0ae6bc52461379c9aa45e98c6021d6634b9759f1a1f9495
+  data.tar.gz: 2626fbe6a5d959eda1ff4c06a9b22408d81520ff0c1fbb26be573051df12fd391cee1049488b25bd47a7778b186914730902d9be8765121bc787bff7f4faf5b3

data/core/LexiconMod.py

@@ -45,9 +45,7 @@ def load_lexicons(language, path=None):
 def show_lexicons(language, path=None):
     if path is None:
       path = __module_dir
-    #lexicons, default_id, this_folder, folder_per_lang = load_lexicons(language, path)
-    LexiconSent(language,None,path)
-
+    lexicons, default_id, this_folder, folder_per_lang = load_lexicons(language, path)
     print
     print '#'*30
     print 'Available lexicons for',language

data/core/general-lexicons/EN-lexicon/Sentiment-English-general.xml

@@ -14,6 +14,15 @@
             <Domain/>
          </Sense>
       </LexicalEntry>
+      <LexicalEntry id="" partOfSpeech="noun">
+         <Lemma writtenForm="very comfortable"/>
+         <Sense>
+            <Confidence level="manual"/>
+            <MonolingualExternalRef/>
+            <Sentiment polarity="negative" strength="strong"/>
+            <Domain/>
+         </Sense>
+      </LexicalEntry>
       <LexicalEntry id="" partOfSpeech="noun">
          <Lemma writtenForm="abandonment"/>
          <Sense>

data/lib/opener/{polarity_tagger/kaf → kaf}/term.rb

@@ -18,6 +18,10 @@ module Opener
         @node.attr :lemma
       end
 
+      def text
+        @node.attr :text
+      end
+
       def pos
         @node.attr :pos
       end

data/lib/opener/polarity_tagger/internal.rb

@@ -1,6 +1,6 @@
 require_relative 'lexicons_cache'
 require_relative 'lexicon_map'
-require_relative 'kaf/document'
+require_relative '../kaf/document'
 
 module Opener
   class PolarityTagger
@@ -9,53 +9,98 @@ module Opener
       DESC        = 'VUA polarity tagger multilanguage'
       LAST_EDITED = '21may2014'
       VERSION     = '1.2'
+      N_WORDS     = 5
 
-      CACHE = LexiconsCache.new
+      CACHE       = LexiconsCache.new
 
       def initialize ignore_pos: false, **params
         @ignore_pos = ignore_pos
       end
 
-      def clear_cache lang: nil, environment:
-      end
-
       def run input, params = {}
-        @kaf = KAF::Document.from_xml input
+        kaf         = KAF::Document.from_xml input
 
         @cache_keys = params[:cache_keys] ||= {}
-        @cache_keys.merge! lang: @kaf.language
-        @map = @kaf.map = CACHE[**@cache_keys].lexicons
+        @cache_keys.merge! lang: kaf.language
+        @map = kaf.map = CACHE[**@cache_keys].lexicons
 
-        raise Opener::Core::UnsupportedLanguageError, @kaf.language if @map.blank?
+        raise Opener::Core::UnsupportedLanguageError, kaf.language if @map.blank?
 
-        @kaf.terms.each do |t|
+        next_index = 0
+        kaf.terms.each_with_index do |t, index|
+          # skip terms when a multi_word_expression is found
+          next if next_index > index
           lemma = t.lemma&.downcase
+          text  = t.text.to_s.downcase
           pos   = if @ignore_pos then nil else t.pos end
           attrs = Hashie::Mash.new
 
-          lexicon, polarity_pos = @map.by_polarity lemma, pos
 
-          if lexicon.polarity != 'unknown'
-            attrs.polarity = lexicon.polarity
-          end
-          if l = @map.by_negator(lemma)
-            lexicon, polarity_pos = l, nil
-            attrs.sentiment_modifier = 'shifter'
+          polarity_pos = nil
+
+          if opts = @map.by_negator(text) || @map.by_negator(lemma)
+            lexicon, next_index = get_lexicon(opts, kaf, index)
+            attrs.sentiment_modifier = 'shifter' if lexicon
+          elsif opts = @map.by_intensifier(text) || @map.by_intensifier(lemma)
+            lexicon, next_index = get_lexicon(opts, kaf, index)
+            attrs.sentiment_modifier = 'intensifier' if lexicon
           end
-          if l = @map.by_intensifier(lemma)
-            lexicon, polarity_pos = l, nil
-            attrs.sentiment_modifier = 'intensifier'
+
+          unless lexicon
+            # text matching have priority as sometimes
+            # the lemma provided by Stanza is a different word
+            [text, lemma].each do |word|
+              opts, polarity_pos = @map.by_polarity word, pos
+
+              if opts[:multi].size > 0 or opts[:single]
+                lexicon, next_index = get_lexicon opts, kaf, index
+                if lexicon
+                  attrs.polarity = lexicon.polarity
+                  break
+                end
+              end
+            end
           end
 
           if attrs.size > 0
-            attrs.resource = lexicon.resource if lexicon.resource
+            attrs['lexicon-id'] = lexicon.id.to_s  if lexicon&.id
+            attrs.resource      = lexicon.resource if lexicon&.resource
             t.setPolarity attrs, polarity_pos
+            i = index
+            while i < next_index do
+              term = kaf.terms[i]
+              term.setPolarity attrs, polarity_pos
+              i += 1
+            end
           end
         end
 
-        @kaf.add_linguistic_processor DESC, "#{LAST_EDITED}_#{VERSION}", 'terms'
+        kaf.add_linguistic_processor DESC, "#{LAST_EDITED}_#{VERSION}", 'terms'
+
+        kaf.to_xml
+      end
 
-        @kaf.to_xml
+      def get_lexicon opts, kaf, index
+        if lexicon = identify_lexicon(kaf.terms[index, N_WORDS], opts.multi)
+          index = index + lexicon.lemma.strip.split(' ').size
+        else
+          lexicon = opts.single
+        end
+
+        [lexicon, index]
+      end
+
+      def identify_lexicon terms, lexicons
+        return unless lexicons.size > 0
+
+        lemma = terms.map{|t| t.lemma&.downcase }.join(' ')
+        text  = terms.map{|t| t.text&.downcase }.join(' ')
+
+        lexicons.each do |lexicon|
+          return lexicon if lemma =~ /^#{lexicon.lemma}($|\s)+/
+          return lexicon if text =~ /^#{lexicon.lemma}($|\s)+/
+        end
+        nil
       end
 
     end

data/lib/opener/polarity_tagger/lexicon_map.rb

@@ -7,9 +7,7 @@ module Opener
       attr_reader :intensifiers
       attr_reader :with_polarity
 
-      UNKNOWN = Hashie::Mash.new polarity: 'unknown'
-
-      POS_ORDER     = 'NRVGAO'
+      POS_ORDER     = 'ONRVGA'
       DEFAULT_POS   = 'O'
       POS_SHORT_MAP = {
         adj:         'G',
@@ -45,17 +43,22 @@ module Opener
         @intensifiers[lemma]
       end
 
-      def by_polarity lemma, short_pos
-        return [@with_polarity[lemma+short_pos] || UNKNOWN, short_pos] if short_pos
+      def by_polarity lemma, identified_short_pos
+        hash = Hashie::Mash.new multi: (@with_polarity[lemma] || [])
+
+        if identified_short_pos and lexicon = @with_polarity[lemma+identified_short_pos]
+          hash[:single] = lexicon
+          return [hash, identified_short_pos]
+        end
 
         POS_ORDER.chars.each do |short_pos|
-          if l = @with_polarity[lemma+short_pos]
-            puts "Found polarify #{l.polarity} for #{lemma} with PoS #{short_pos}"
-            return [l, short_pos]
+          if lexicon = @with_polarity[lemma+short_pos]
+            hash[:single] = lexicon
+            return [hash, identified_short_pos]
           end
         end
 
-        [UNKNOWN, 'unknown']
+        [hash, 'unknown']
       end
 
       protected
@@ -63,16 +66,33 @@ module Opener
       def map lexicons
         return if blank?
 
-        lexicons.each do |l|
-          next if l.lemma.nil?
+        lexicons.each do |lexicon|
+          next if lexicon.lemma.nil?
 
-          case l.type
-          when 'polarityShifter' then @negators[l.lemma]     = l
-          when 'intensifier'     then @intensifiers[l.lemma] = l
-          else
-            if l.polarity
-              short_pos = POS_SHORT_MAP[l.pos&.to_sym] || DEFAULT_POS
-              @with_polarity[l.lemma+short_pos] = l
+          sub_lexicons = [lexicon]
+          sub_lexicons += lexicon.variants if lexicon.variants
+
+          sub_lexicons.each do |variant|
+            if variant.lemma.strip.include? ' '
+              lemma = variant.lemma.strip.split(' ').first
+              type = :multi
+            else
+              lemma = variant.lemma
+              type = :single
+            end
+
+            if ['polarityShifter', 'intensifier'].include? lexicon.type
+              var = @negators if lexicon.type == 'polarityShifter'
+              var = @intensifiers if lexicon.type == 'intensifier'
+
+              var[lemma] ||= Hashie::Mash.new multi: []
+              if type == :multi
+                var[lemma][:multi] << lexicon
+              else
+                var[lemma][:single] = lexicon
+              end
+            else
+              map_one_polarity lemma, variant, lexicon if lexicon.polarity
             end
           end
         end
@@ -82,6 +102,19 @@ module Opener
         puts "#{@lang}: loaded #{@with_polarity.size} elements with polarity"
       end
 
+      def map_one_polarity lemma, hash, lexicon
+        poses = if hash.poses.present? then hash.poses else [hash.pos] end
+        poses.each do |pos|
+          short_pos = POS_SHORT_MAP[pos&.to_sym] || DEFAULT_POS
+          @with_polarity[lemma] ||= []
+          if hash.lemma.strip.include? ' '
+            @with_polarity[lemma] << lexicon
+          else
+            @with_polarity[lemma+short_pos] = lexicon
+          end
+        end
+      end
+
     end
   end
 end

data/lib/opener/polarity_tagger/lexicons_cache.rb

@@ -15,10 +15,11 @@ module Opener
       end
 
       def [] **params
+        existing = @cache[params]
+        return existing if existing and existing.from > UPDATE_INTERVAL.ago
+
         synchronize do
-          existing = @cache[params]
-          break existing if existing and existing.from > UPDATE_INTERVAL.ago
-          @cache[params] = cache_update existing, **params
+          @cache[params] = cache_update @cache[params], **params
         end
       end
       alias_method :get, :[]

data/lib/opener/polarity_tagger/version.rb

@@ -1,7 +1,7 @@
 module Opener
   class PolarityTagger
 
-    VERSION = '3.2.6'
+    VERSION = '3.5.1'
 
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opener-polarity-tagger
 version: !ruby/object:Gem::Version
-  version: 3.2.6
+  version: 3.5.1
 platform: ruby
 authors:
 - development@olery.com
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-04 00:00:00.000000000 Z
+date: 2021-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opener-daemons
@@ -212,12 +212,12 @@ files:
 - core/poltagger-basic-multi.py
 - exec/polarity-tagger.rb
 - ext/hack/Rakefile
+- lib/opener/kaf/document.rb
+- lib/opener/kaf/term.rb
 - lib/opener/polarity_tagger.rb
 - lib/opener/polarity_tagger/cli.rb
 - lib/opener/polarity_tagger/external.rb
 - lib/opener/polarity_tagger/internal.rb
-- lib/opener/polarity_tagger/kaf/document.rb
-- lib/opener/polarity_tagger/kaf/term.rb
 - lib/opener/polarity_tagger/lexicon_map.rb
 - lib/opener/polarity_tagger/lexicons_cache.rb
 - lib/opener/polarity_tagger/public/markdown.css