opener-polarity-tagger 3.0.1 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 4b7da413e3b3dccd744b0fdab117ed827055c2cd
-  data.tar.gz: cfecf678253bad98f8f3a6e09350daf1d57deced
+SHA256:
+  metadata.gz: 1407a3f9dab798e58f92e033a6255151545d225c03a5e056a2f2dbb6878832ed
+  data.tar.gz: 9c79bb5e40a5882effff686d11f180f7d0ababbd5e2c9d11c0a4bab1a835eb22
 SHA512:
-  metadata.gz: 376581bc6d9a2e5864c66d943d817a1ee6596d5430b9be5966693fd3f6538fac5e9b9116089ec8ef4552142e121602bf9edcd37135b902319182b7acbf3160bb
-  data.tar.gz: 0d212c6acf29b3f969c90359342c0b9d20abf9862e0999c12f131a461ca67a58a588ee41c9c85bfd5faea1e8c9dd77bc3220503edcf5302b86ef93e62fcb5575
+  metadata.gz: 53b7b5649087d0ab9728df51ccf67dd7c6ff48eeaa0c6f4c3f74a257f07b610f29dc05b0919b1d2c379becb5b31f36c5609bca49583a3eb1d0b085df256ad9e7
+  data.tar.gz: f510909ce50427bb126773dedb60b21f3998fb80f49b9e602254e3adf1de1a658ce5d5a3b8ba7a452a78d764fa3960e455ce89d3d495c82147440e17780237ea

data/README.md

@@ -178,6 +178,7 @@ At least you need the following system setup:
 * lxml installed
 * libarchive, on Debian/Ubuntu based systems this can be installed using
   `sudo apt-get install libarchive-dev`
+* VUKafParserPy, install with sudo pip install 'https://github.com/opener-project/VU-kaf-parser/archive/v1.1.zip#egg=VUKafParserPy'
 
 Domain Adaption
 ---------------

data/bin/console

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require_relative '../lib/opener/polarity_tagger'
+
+begin
+  require 'awesome_print'
+  require 'pry'
+  binding.pry
+rescue LoadError
+  require 'irb'
+  IRB.start
+end

data/core/LexiconMod.py

@@ -61,7 +61,6 @@ def show_lexicons(language, path=None):
     print '#'*30
     print
 
-
 class LexiconSent:
 
     def __init__(self,language='nl',lexicon_id=None, path=None):
@@ -79,10 +78,8 @@ class LexiconSent:
 
         self.load_resources(language,lexicon_id, path)
 
-
         self.__load_lexicon_xml()
 
-
     def load_resources(self,language,my_id=None, path=None):
         if path is None:
           path = os.path.dirname(__file__)
@@ -100,14 +97,9 @@ class LexiconSent:
         self.filename = os.path.join(this_folder,folder_per_lang[language],lexicons[id_to_load][0])
         self.resource = lexicons[id_to_load][1]+" . "+lexicons[id_to_load][2]
 
-
-
-
-
     def getResource(self):
         return self.resource
 
-
     def convert_pos_to_kaf(self,pos):
         my_map = {}
         my_map['adj'] = 'G'
@@ -119,7 +111,6 @@ class LexiconSent:
         my_map['verb']= 'V'
         return my_map.get(pos.lower(),'O')
 
-
     def __load_lexicon_xml(self):
         logging.debug('Loading lexicon from the file'+self.filename)
         from collections import defaultdict
@@ -160,16 +151,12 @@ class LexiconSent:
         logging.debug('Loaded: '+str(len(self.intensifiers))+' intensifiers')
         logging.debug('Loaded: '+str(len(self.sentLex))+' elements with polarity')
 
-
-
     def isIntensifier(self,lemma):
         return lemma in self.intensifiers
 
-
     def isNegator(self,lemma):
       return lemma in self.negators
 
-
     def getPolarity(self,lemma,pos):
       if pos:
           return self.sentLex.get((lemma,pos),'unknown'),pos

data/lib/opener/polarity_tagger.rb

@@ -1,106 +1,27 @@
 require 'open3'
 require 'opener/core'
+require 'nokogiri'
+require 'hashie'
 
 require_relative 'polarity_tagger/version'
 require_relative 'polarity_tagger/cli'
+require_relative 'polarity_tagger/external'
+
+require_relative 'polarity_tagger/internal'
 
 module Opener
-  ##
-  # Ruby wrapper around the Python based polarity tagger.
-  #
-  # @!attribute [r] options
-  #  @return [Hash]
-  #
-  # @!attribute [r] args
-  #  @return [Array]
-  #
   class PolarityTagger
-    attr_reader :options, :args
 
-    ##
-    # @param [Hash] options
-    #
-    # @option options [Array] :args Collection of arbitrary arguments to pass
-    #  to the underlying kernel.
-    #
-    # @option options [String] :resource_path Path to the lexicons to use.
-    #
-    def initialize(options = {})
+    def initialize options = {}
       @args    = options.delete(:args) || []
       @options = options
+      @klass   = if ENV['LEGACY'] then External else Internal end
+      @proc    = @klass.new args: @args
     end
 
-    ##
-    # Returns a String containing the command to use for executing the kernel.
-    #
-    # @return [String]
-    #
-    def command
-      return "#{adjust_python_path} python -E #{kernel} #{lexicon_path} #{args.join(" ")}"
+    def run input
+      @proc.run input
     end
 
-    ##
-    # @return [String]
-    #
-    def lexicon_path
-      path = options[:resource_path] || ENV['RESOURCE_PATH'] ||
-        ENV['POLARITY_LEXICON_PATH']
-
-      return path ? "--lexicon-path #{path}" : nil
-    end
-
-    ##
-    # Processes the input and returns an Array containing the output of STDOUT,
-    # STDERR and an object containing process information.
-    #
-    # @param [String] input The text of which to detect the language.
-    # @return [Array]
-    #
-    def run(input)
-      stdout, stderr, process = capture(input)
-
-      raise stderr unless process.success?
-
-      return stdout
-    end
-
-    protected
-
-    ##
-    # @return [String]
-    #
-    def adjust_python_path
-      site_packages =  File.join(core_dir, 'site-packages')
-
-      "env PYTHONPATH=#{site_packages}:$PYTHONPATH"
-    end
-
-    ##
-    # capture3 method doesn't work properly with Jruby, so
-    # this is a workaround
-    #
-    def capture(input)
-      Open3.popen3(*command.split(" ")) {|i, o, e, t|
-        out_reader = Thread.new { o.read }
-        err_reader = Thread.new { e.read }
-        i.write input
-        i.close
-        [out_reader.value, err_reader.value, t.value]
-      }
-    end
-
-    ##
-    # @return [String]
-    #
-    def core_dir
-      return File.expand_path('../../../core', __FILE__)
-    end
-
-    ##
-    # @return [String]
-    #
-    def kernel
-      return File.join(core_dir, 'poltagger-basic-multi.py')
-    end
-  end # PolarityTagger
-end # Opener
+  end
+end

data/lib/opener/polarity_tagger/cli.rb

@@ -26,7 +26,7 @@ module Opener
       # @return [Slop]
       #
       def configure_slop
-        return Slop.new(:strict => false, :indent => 2, :help => true) do
+        Slop.new strict: false, indent: 2, help: true do
           banner 'Usage: polarity-tagger [OPTIONS] -- [PYTHON OPTIONS]'
 
           separator <<-EOF.chomp
@@ -62,6 +62,7 @@ Examples:
           end
         end
       end
-    end # CLI
-  end # PolarityTagger
-end # Opener
+
+    end
+  end
+end

data/lib/opener/polarity_tagger/external.rb

@@ -0,0 +1,105 @@
+module Opener
+  class PolarityTagger
+    ##
+    # Ruby wrapper around the Python based polarity tagger.
+    #
+    # @!attribute [r] options
+    #  @return [Hash]
+    #
+    # @!attribute [r] args
+    #  @return [Array]
+    #
+    class External
+
+      attr_reader :options, :args
+
+      ##
+      # @param [Hash] options
+      #
+      # @option options [Array] :args Collection of arbitrary arguments to pass
+      #  to the underlying kernel.
+      #
+      # @option options [String] :resource_path Path to the lexicons to use.
+      #
+      def initialize options = {}
+        @args    = options.delete(:args) || []
+        @options = options
+      end
+
+      ##
+      # Returns a String containing the command to use for executing the kernel.
+      #
+      # @return [String]
+      #
+      def command
+        return "#{adjust_python_path} python -E #{kernel} #{lexicon_path} #{args.join(" ")}"
+      end
+
+      ##
+      # @return [String]
+      #
+      def lexicon_path
+        path = options[:resource_path] || ENV['RESOURCE_PATH'] ||
+          ENV['POLARITY_LEXICON_PATH']
+
+        return path ? "--lexicon-path #{path}" : nil
+      end
+
+      ##
+      # Processes the input and returns an Array containing the output of STDOUT,
+      # STDERR and an object containing process information.
+      #
+      # @param [String] input The text of which to detect the language.
+      # @return [Array]
+      #
+      def run(input)
+        stdout, stderr, process = capture(input)
+
+        raise stderr unless process.success?
+        puts stderr if ENV['DEBUG']
+
+        return stdout
+      end
+
+      protected
+
+      ##
+      # @return [String]
+      #
+      def adjust_python_path
+        site_packages =  File.join(core_dir, 'site-packages')
+
+        "env PYTHONPATH=#{site_packages}:$PYTHONPATH"
+      end
+
+      ##
+      # capture3 method doesn't work properly with Jruby, so
+      # this is a workaround
+      #
+      def capture(input)
+        Open3.popen3(*command.split(" ")) {|i, o, e, t|
+          out_reader = Thread.new { o.read }
+          err_reader = Thread.new { e.read }
+          i.write input
+          i.close
+          [out_reader.value, err_reader.value, t.value]
+        }
+      end
+
+      ##
+      # @return [String]
+      #
+      def core_dir
+        File.expand_path '../../../../core', __FILE__
+      end
+
+      ##
+      # @return [String]
+      #
+      def kernel
+        File.join core_dir, 'poltagger-basic-multi.py'
+      end
+
+    end
+  end
+end

data/lib/opener/polarity_tagger/internal.rb

@@ -0,0 +1,57 @@
+require_relative 'lexicons_cache'
+require_relative 'lexicon_map'
+require_relative 'kaf/document'
+
+module Opener
+  class PolarityTagger
+    class Internal
+
+      DESC        = 'VUA polarity tagger multilanguage'
+      LAST_EDITED = '21may2014'
+      VERSION     = '1.2'
+
+      def initialize ignore_pos: false, **params
+        @cache = LexiconsCache.new
+
+        @ignore_pos = ignore_pos
+      end
+
+      def run input
+        @kaf = KAF::Document.from_xml input
+        @map = @kaf.map = @cache[@kaf.language]
+
+        negators = 0
+        @kaf.terms.each do |t|
+          lemma = t.lemma&.downcase
+          pos   = if @ignore_pos then nil else t.pos end
+          attrs = Hashie::Mash.new
+
+          lexicon, polarity_pos = @map.by_polarity lemma, pos
+
+          if lexicon.polarity != 'unknown'
+            attrs.polarity = lexicon.polarity
+          end
+          if l = @map.by_negator(lemma)
+            negators += 1
+            lexicon, polarity = l, nil
+            attrs.sentiment_modifier = 'shifter'
+          end
+          if l = @map.by_intensifier(lemma)
+            lexicon, polarity = l, nil
+            attrs.sentiment_modifier = 'intensifier'
+          end
+
+          if attrs.size > 0
+            attrs.resource = lexicon.resource if lexicon.resource
+            t.setPolarity attrs, polarity_pos
+          end
+        end
+
+        @kaf.add_linguistic_processor DESC, "#{LAST_EDITED}_#{VERSION}", 'terms'
+
+        @kaf.to_xml
+      end
+
+    end
+  end
+end

data/lib/opener/polarity_tagger/kaf/document.rb

@@ -0,0 +1,53 @@
+require_relative 'term'
+
+module Opener
+  module KAF
+    class Document
+
+      attr_reader :document
+      attr_reader :lexicons
+
+      attr_accessor :map
+
+      def initialize xml
+        @document = xml
+      end
+
+      def self.from_xml xml
+        new Nokogiri::XML xml
+      end
+
+      def language
+        @language ||= @document.at_xpath('KAF').attr 'xml:lang'
+      end
+
+      def terms
+        @terms ||= collection 'KAF/terms/term', Term
+      end
+
+      def add_linguistic_processor name, version, layer, timestamp: false
+        header  = @document.at('kafHeader') || @document.root.add_child('<kafHeader/>')
+        procs   = header.css('linguisticProcessors').find{ |l| l.attr(:layer) == layer }
+        procs ||= header.add_child("<linguisticProcessors layer='#{layer}'/>")
+        lp      = procs.add_child('<lp/>')
+        lp.attr(
+          timestamp: if timestamp then Time.now.iso8601 else '*' end,
+          version:   version,
+          name:      name,
+        )
+        lp
+      end
+
+      def to_xml
+        @document.to_xml indent: 2
+      end
+
+      protected
+
+      def collection query, wrapper
+        @document.xpath(query).map{ |node| wrapper.new self, node }
+      end
+
+    end
+  end
+end

data/lib/opener/polarity_tagger/kaf/term.rb

@@ -0,0 +1,35 @@
+module Opener
+  module KAF
+    class Term
+
+      attr_reader :document
+      attr_reader :node
+
+      def initialize document, node
+        @document = document
+        @node     = node
+      end
+
+      def id
+        @id ||= @node.attr :tid
+      end
+
+      def lemma
+        @node.attr :lemma
+      end
+
+      def pos
+        @node.attr :pos
+      end
+
+      def setPolarity attrs, polarity_pos
+        #In case there is no pos info, we use the polarityPos
+        @node[:pos] = polarity_pos if !pos and polarity_pos
+
+        sentiment = @node.add_child('<sentiment/>')
+        sentiment.attr attrs
+      end
+
+    end
+  end
+end

data/lib/opener/polarity_tagger/lexicon_map.rb

@@ -0,0 +1,82 @@
+module Opener
+  class PolarityTagger
+    class LexiconMap
+
+      attr_reader :resource
+      attr_reader :negators
+      attr_reader :intensifiers
+      attr_reader :with_polarity
+
+      POS_ORDER = 'NRVGAO'
+      UNKNOWN   = Hashie::Mash.new polarity: 'unknown'
+
+      def initialize lang:, lexicons:
+        @lang          = lang
+        @lexicons      = lexicons
+
+        @negators      = {}
+        @intensifiers  = {}
+        @with_polarity = {}
+        map lexicons
+      end
+
+      DEFAULT_POS = 'O'
+
+      POS_SHORT_MAP = {
+        adj:         'G',
+        adv:         'A',
+        noun:        'N',
+        propernoun:  'N',
+        other:       'O',
+        prep:        'P',
+        verb:        'V',
+        nil =>       DEFAULT_POS,
+        multi_word_expression: 'O',
+      }
+
+      def by_negator lemma
+        @negators[lemma]
+      end
+
+      def by_intensifier lemma
+        @intensifiers[lemma]
+      end
+
+      def by_polarity lemma, short_pos
+        return [@with_polarity[lemma+short_pos] || UNKNOWN, short_pos] if short_pos
+
+        POS_ORDER.chars.each do |short_pos|
+          if l = @with_polarity[lemma+short_pos]
+            puts "Found polarify #{l.polarity} for #{lemma} with PoS #{short_pos}"
+            return [l, short_pos]
+          end
+        end
+
+        [UNKNOWN, 'unknown']
+      end
+
+      protected
+
+      def map lexicons
+        lexicons.each do |l|
+          next if l.lemma.nil?
+
+          case l.type
+          when 'polarityShifter' then @negators[l.lemma]     = l
+          when 'intensifier'     then @intensifiers[l.lemma] = l
+          else
+            if l.polarity
+              short_pos = POS_SHORT_MAP[l.pos&.to_sym] || DEFAULT_POS
+              @with_polarity[l.lemma+short_pos] = l
+            end
+          end
+        end
+
+        puts "#{@lang}: loaded #{@negators.size} negators"
+        puts "#{@lang}: loaded #{@intensifiers.size} intensifiers"
+        puts "#{@lang}: loaded #{@with_polarity.size} elements with polarity"
+      end
+
+    end
+  end
+end

data/lib/opener/polarity_tagger/lexicons_cache.rb

@@ -0,0 +1,67 @@
+module Opener
+  class PolarityTagger
+    class LexiconsCache
+
+      def initialize
+        extend MonitorMixin
+
+        @url   = ENV['POLARITY_LEXICON_URL']
+        @path  = ENV['POLARITY_LEXICON_PATH']
+        @cache = {}
+      end
+
+      def [] lang
+        synchronize do
+          @cache[lang] ||= load_lexicons lang
+        end
+      end
+      alias_method :get, :[]
+
+      def load_lexicons lang
+        lexicons = if @url then load_from_url lang else load_from_path lang end
+
+        LexiconMap.new lang: lang, lexicons: lexicons
+      end
+
+      def load_from_url lang
+        url      = "#{@url}&language_code=#{lang}"
+        puts "#{lang}: loading lexicons from url #{url}"
+        lexicons = JSON.parse HTTPClient.new.get(url).body
+        lexicons = lexicons['data'].map{ |l| Hashie::Mash.new l }
+        lexicons
+      end
+
+      def load_from_path lang
+        @path  ||= 'core/general-lexicons'
+        dir      = "#{@path}/#{lang.upcase}-lexicon"
+        config   = Nokogiri::XML File.read "#{dir}/config.xml"
+        lexicons = []
+
+        config.css(:lexicon).each do |cl|
+          filename = cl.at(:filename).text
+          resource = cl.at(:resource).text
+          xml      = Nokogiri::XML File.read "#{dir}/#{filename}"
+          puts "#{lang}: loading lexicons from the file #{filename}"
+
+          lexicons.concat(xml.css(:LexicalEntry).map do |le|
+            Hashie::Mash.new(
+              resource:   resource,
+              identifier: le.attr(:id),
+              type:       le.attr(:type),
+              lemma:      le.at(:Lemma).attr(:writtenForm).downcase,
+              pos:        le.attr(:partOfSpeech)&.downcase,
+              aspect:     le.at(:Domain)&.attr(:aspect)&.downcase,
+              polarity:   le.at(:Sentiment).attr(:polarity),
+              strength:   le.at(:Sentiment).attr(:strength),
+              confidence_level:   le.at(:Confidence)&.attr(:level),
+              domain_conditional: le.at(:Domain)&.attr(:conditional) == 'yes',
+            )
+          end)
+        end
+
+        lexicons
+      end
+
+    end
+  end
+end

data/lib/opener/polarity_tagger/version.rb

@@ -1,5 +1,7 @@
 module Opener
   class PolarityTagger
-    VERSION = '3.0.1'
-  end # PolarityTagger
-end # Opener
+
+    VERSION = '3.1.0'
+
+  end
+end

data/opener-polarity-tagger.gemspec

@@ -33,11 +33,13 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'opener-webservice', '~> 2.1'
   gem.add_dependency 'opener-core', '~> 2.2'
 
+  gem.add_dependency 'hashie'
   gem.add_dependency 'rake'
   gem.add_dependency 'nokogiri'
   gem.add_dependency 'cliver'
   gem.add_dependency 'slop', '~> 3.5'
 
+  gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rspec', '~> 3.0'
   gem.add_development_dependency 'cucumber'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opener-polarity-tagger
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.1.0
 platform: ruby
 authors:
 - development@olery.com
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-01-12 00:00:00.000000000 Z
+date: 2020-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opener-daemons
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -137,17 +165,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Polarity tagger for various languages.
-email: 
+email:
 executables:
-- polarity-tagger-server
-- polarity-tagger-daemon
 - polarity-tagger
+- polarity-tagger-daemon
+- polarity-tagger-server
+- console
 extensions:
 - ext/hack/Rakefile
 extra_rdoc_files: []
 files:
 - LICENSE.txt
 - README.md
+- bin/console
 - bin/polarity-tagger
 - bin/polarity-tagger-daemon
 - bin/polarity-tagger-server
@@ -170,6 +200,12 @@ files:
 - ext/hack/Rakefile
 - lib/opener/polarity_tagger.rb
 - lib/opener/polarity_tagger/cli.rb
+- lib/opener/polarity_tagger/external.rb
+- lib/opener/polarity_tagger/internal.rb
+- lib/opener/polarity_tagger/kaf/document.rb
+- lib/opener/polarity_tagger/kaf/term.rb
+- lib/opener/polarity_tagger/lexicon_map.rb
+- lib/opener/polarity_tagger/lexicons_cache.rb
 - lib/opener/polarity_tagger/public/markdown.css
 - lib/opener/polarity_tagger/server.rb
 - lib/opener/polarity_tagger/version.rb
@@ -185,7 +221,7 @@ homepage: http://opener-project.github.com/
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -200,10 +236,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.7.8
+signing_key:
 specification_version: 4
 summary: Polarity tagger for various languages.
 test_files: []
-has_rdoc: