opener-property-tagger 3.0.5 → 3.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: db4ce8acc0c1dd88b2042f9a936012ea9a3c1448
-  data.tar.gz: 4ed777c8b215fb98dd3687689cb1c58d97787dd0
+SHA256:
+  metadata.gz: 168a0501dc6567285dd70b1a304fdc885b6d9493f9ca62863a428068b42d284e
+  data.tar.gz: 6e9dd2b446eca6e75d4644e5cb59ec8ad64ec6f7f88e79266142129ce5ee59eb
 SHA512:
-  metadata.gz: 5cd57abe32b4d88f8ed10618c9e8466c1628faebda89ad1ceeaa956c571fdc00a6069cea44a1a1570df2ebe0136fc6e34c142d1850972a1bc3e2f926c640d13d
-  data.tar.gz: e64b80100aac3dbfc3229e75692ef9ab16cbd362ca878e36f2ef5b492c456843fcb535c5f26105f6e334594a433c7ff9b050758068779803373aab76acac5e96
+  metadata.gz: 7a209b8da679c06ea5fce64a11142b22b8b7a3c4da2775f832f4266af152c624f8ddb9e81bf7cbb96d668babc303fedb1243988b243faa2c72a5b7438c47c9b6
+  data.tar.gz: 621b9a11a6c2230aee345fc4d072b8636d1c63b6533960103de475355a54baac7924867eb59176850d7b0aaaccdbc1a4a6eccf9af62652bbaf6f26097163778d

data/lib/opener/property_tagger.rb

@@ -1,12 +1,18 @@
 require 'open3'
 require 'slop'
 require 'oga'
+require 'monitor'
+require 'httpclient'
+require 'hashie'
+require 'json'
 
 require 'rexml/document'
 require 'rexml/formatters/pretty'
 
 require_relative 'property_tagger/version'
 require_relative 'property_tagger/cli'
+require_relative 'property_tagger/aspects_cache'
+require_relative 'property_tagger/remote_aspects_cache'
 require_relative 'property_tagger/processor'
 
 module Opener
@@ -41,36 +47,36 @@ module Opener
     # @return [String]
     #
     def path
-      path = options[:resource_path] || ENV['RESOURCE_PATH'] ||
+      return @path if @path
+
+      @path = options[:resource_path] || ENV['RESOURCE_PATH'] ||
         ENV['PROPERTY_TAGGER_LEXICONS_PATH']
+      return unless @path
 
-      unless path
-        raise ArgumentError, 'No lexicon path provided'
-      end
+      @path = File.expand_path @path
+    end
 
-      return File.expand_path(path)
+    def remote_url
+      @remote_url ||= ENV['PROPERTY_TAGGER_LEXICONS_URL']
     end
 
     ##
-    # Processes the input and returns an Array containing the output of STDOUT,
-    # STDERR and an object containing process information.
+    # Processes the input KAF document.
     #
-    # @param [String] input The text of which to detect the language.
-    # @return [Array]
+    # @param [String] input
+    # @return [String]
     #
-    def run(input)
-      output = process(input)
+    def run input
+      timestamp = !options[:no_time]
 
-      return output
+      Processor.new(input,
+        url:       remote_url,
+        path:      path,
+        timestamp: timestamp,
+        pretty:    options[:pretty],
+      ).process
     end
 
-    protected
-
-    def process(input)
-      processor = Processor.new(input, path, !options[:no_time])
-
-      return processor.process
-    end
-  end # PolarityTagger
-end # Opener
+  end
+end
 

data/lib/opener/property_tagger/aspects_cache.rb

@@ -0,0 +1,47 @@
+module Opener
+  class PropertyTagger
+    ##
+    # Thread-safe cache for storing the contents of aspect files.
+    #
+    class AspectsCache
+      include MonitorMixin
+
+      def initialize
+        super
+
+        @cache = {}
+      end
+
+      ##
+      # Returns the aspects for the given file path. If the aspects don't exist
+      # they are first loaded into the cache.
+      #
+      # @param [String] path
+      #
+      def [](path)
+        synchronize do
+          @cache[path] = load_aspects(path) unless @cache.key?(path)
+        end
+      end
+
+      alias_method :get, :[]
+
+      ##
+      # Loads the aspects of the given path.
+      #
+      # @param [String] path
+      #
+      def load_aspects(path)
+        mapping = Hash.new { |hash, key| hash[key] = [] }
+
+        File.foreach(path) do |line|
+          lemma, pos, aspect = line.chomp.split("\t")
+
+          mapping[lemma.to_sym] << aspect
+        end
+
+        return mapping
+      end
+    end # AspectsCache
+  end # PropertyTagger
+end # Opener

data/lib/opener/property_tagger/cli.rb

@@ -56,10 +56,13 @@ Examples:
 
           on :'no-time', 'Disables adding of timestamps'
 
+          on :ugly, 'Disables pretty formatting of XML (faster)'
+
           run do |opts, args|
             tagger = PropertyTagger.new(
               :args    => args,
-              :no_time => opts[:'no-time']
+              :no_time => opts[:'no-time'],
+              :pretty  => !opts[:ugly]
             )
 
             input  = STDIN.tty? ? nil : STDIN.read

data/lib/opener/property_tagger/processor.rb

@@ -4,15 +4,37 @@ module Opener
     # Class that applies property tagging to a given input KAF file.
     #
     class Processor
-      attr_accessor :document, :aspects_path, :language, :aspects, :terms,
-        :timestamp
 
-      def initialize(file, aspects_path, timestamp = true)
-        @document     = Oga.parse_xml(file)
-        @aspects_path = aspects_path
-        @timestamp    = timestamp
+      attr_accessor :document
+      attr_accessor :aspects, :aspects_path, :aspects_url
+      attr_accessor :timestamp, :pretty
 
+      ##
+      # Global cache used for storing loaded aspects.
+      #
+      # @return [Opener::PropertyTagger::AspectsCache.new]
+      #
+      ASPECTS_CACHE        = AspectsCache.new
+      REMOTE_ASPECTS_CACHE = RemoteAspectsCache.new
+
+      ##
+      # @param [String|IO] file The KAF file/input to process.
+      # @param [String] aspects_path Path to the aspects.
+      # @param [TrueClass|FalseClass] timestamp Add timestamps to the KAF.
+      # @param [TrueClass|FalseClass] pretty Enable pretty formatting, disabled
+      #  by default due to the performance overhead.
+      #
+      def initialize file, url: nil, path: nil, timestamp: true, pretty: false
+        @document     = Oga.parse_xml file
         raise 'Error parsing input. Input is required to be KAF' unless is_kaf?
+        @timestamp    = timestamp
+        @pretty       = pretty
+
+        @remote       = !url.nil?
+        @aspects_path = path
+        @aspects_url  = url
+
+        @aspects = if @remote then REMOTE_ASPECTS_CACHE[language] else ASPECTS_CACHE[aspects_file] end
       end
 
       ##
@@ -20,64 +42,45 @@ module Opener
       # @return [String]
       #
       def process
-        @language = get_language
-        @aspects  = load_aspects
-        @terms    = get_terms
-
         existing_aspects = extract_aspects
 
         add_features_layer
         add_properties_layer
 
-        index = 1
+        existing_aspects.each_with_index do |(key, value), index|
+          index += 1
 
-        existing_aspects.each_pair do |key,value|
           add_property(key, value, index)
-          index += 1
         end
 
         add_linguistic_processor
 
-        return pretty_print(document)
-      end
-
-      ##
-      # Loads the aspects from the txt file
-      # @return [Hash]
-      #
-      def load_aspects
-        aspects_hash = {}
-
-        File.foreach(aspects_file) do |line|
-          lemma, pos, aspect = line.gsub("\n", "").split("\t")
-
-          aspects_hash[lemma.to_sym] = [] unless aspects_hash[lemma.to_sym]
-          aspects_hash[lemma.to_sym] << aspect
-        end
-
-        return aspects_hash
+        return pretty ? pretty_print(document) : document.to_xml
       end
 
       ##
       # Get the language of the input file.
+      #
       # @return [String]
       #
-      def get_language
-        document.at_xpath('KAF').get('xml:lang')
+      def language
+        return @language ||= document.at_xpath('KAF').get('xml:lang')
       end
 
       ##
       # Get the terms from the input file
       # @return [Hash]
       #
-      def get_terms
-        terms_hash = {}
+      def terms
+        unless @terms
+          @terms = {}
 
-        document.xpath('KAF/terms/term').each do |term|
-          terms_hash[term.get('tid').to_sym] = term.get('lemma')
+          document.xpath('KAF/terms/term').each do |term|
+            @terms[term.get('tid').to_sym] = term.get('lemma')
+          end
         end
 
-        return terms_hash
+        return @terms
       end
 
       ##
@@ -93,7 +96,7 @@ module Opener
         # lemmas) belong to a property.
         max_ngram = 2
 
-        uniq_aspects = {}
+        uniq_aspects = Hash.new { |hash, key| hash[key] = [] }
 
         while current_token < terms.count
           (0..max_ngram).each do |tam_ngram|
@@ -107,7 +110,6 @@ module Opener
                 properties.uniq.each do |property|
                   next if !property or property.strip.empty?
 
-                  uniq_aspects[property.to_sym] = [] unless uniq_aspects[property.to_sym]
                   uniq_aspects[property.to_sym] << [ids,ngram]
                 end
               end
@@ -223,9 +225,13 @@ module Opener
         return !!document.at_xpath('KAF')
       end
 
+      ##
+      # @return [String]
+      #
       def aspects_file
-        return File.expand_path("#{aspects_path}/#{language}.txt", __FILE__)
+        @aspects_file ||= File.expand_path "#{aspects_path}/#{language}.txt", __FILE__
       end
-    end # Processor
-  end # PropertyTagger
-end # Opener
+
+    end
+  end
+end

data/lib/opener/property_tagger/remote_aspects_cache.rb

@@ -0,0 +1,40 @@
+module Opener
+  class PropertyTagger
+    ##
+    # Thread-safe cache for storing the contents of remote aspects.
+    #
+    class RemoteAspectsCache
+
+      include MonitorMixin
+
+      def initialize
+        super
+
+        @url   = ENV['PROPERTY_TAGGER_LEXICONS_URL']
+        @cache = {}
+      end
+
+      def [] lang
+        synchronize do
+          @cache[lang] ||= load_aspects lang
+        end
+      end
+      alias_method :get, :[]
+
+      def load_aspects lang
+        mapping  = Hash.new{ |hash, key| hash[key] = [] }
+        url      = "#{@url}&language_code=#{lang}"
+        lexicons = JSON.parse HTTPClient.new.get(url).body
+        lexicons = lexicons['data'].map{ |l| Hashie::Mash.new l }
+        puts "#{lang}: loaded aspects from #{url}"
+
+        lexicons.each do |l|
+          mapping[l.lemma.to_sym] << l.aspect
+        end
+
+        return mapping
+      end
+
+    end
+  end
+end

data/lib/opener/property_tagger/version.rb

@@ -1,5 +1,7 @@
 module Opener
   class PropertyTagger
-    VERSION = '3.0.5'
-  end # PropertyTagger
-end # Opener
+
+    VERSION = '3.2.1'
+
+  end
+end

data/opener-property-tagger.gemspec

@@ -28,9 +28,12 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'opener-webservice', '~> 2.1'
   gem.add_dependency 'opener-core', '~> 2.2'
 
-  gem.add_dependency 'oga'
+  gem.add_dependency 'oga', ['~> 1.0', '>= 1.3.1']
+  gem.add_dependency 'httpclient'
+  gem.add_dependency 'hashie'
 
   gem.add_development_dependency 'rspec', '~> 3.0'
   gem.add_development_dependency 'cucumber'
   gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'benchmark-ips', '~> 2.0'
 end

data/task/test.rake

@@ -2,5 +2,6 @@ desc 'Runs the tests'
 task :test => [:lexicons] do
   ENV['RESOURCE_PATH'] = File.expand_path('../../tmp/lexicons/hotel', __FILE__)
 
-  sh('cucumber features')
+  sh 'cucumber features'
+  sh 'rspec spec --order random'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opener-property-tagger
 version: !ruby/object:Gem::Version
-  version: 3.0.5
+  version: 3.2.1
 platform: ruby
 authors:
 - development@olery.com
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-01-26 00:00:00.000000000 Z
+date: 2020-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opener-daemons
@@ -54,6 +54,40 @@ dependencies:
         version: '2.2'
 - !ruby/object:Gem::Dependency
   name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -108,8 +142,22 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 description: Property tagger for hotels in Dutch and English.
-email: 
+email:
 executables:
 - property-tagger
 - property-tagger-daemon
@@ -125,9 +173,11 @@ files:
 - config.ru
 - exec/property-tagger.rb
 - lib/opener/property_tagger.rb
+- lib/opener/property_tagger/aspects_cache.rb
 - lib/opener/property_tagger/cli.rb
 - lib/opener/property_tagger/processor.rb
 - lib/opener/property_tagger/public/markdown.css
+- lib/opener/property_tagger/remote_aspects_cache.rb
 - lib/opener/property_tagger/server.rb
 - lib/opener/property_tagger/version.rb
 - lib/opener/property_tagger/views/index.erb
@@ -139,7 +189,7 @@ homepage: http://opener-project.github.com/
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -154,10 +204,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.7.8
+signing_key:
 specification_version: 4
 summary: Property tagger for hotels in Dutch and English.
 test_files: []
-has_rdoc: