opener-property-tagger 3.3.1 → 3.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d732e35937583e6fbad1452c2dc72f894df668979a9259c19f4a84a9557b8971
-  data.tar.gz: 6999b3814921c0ec9cefd59b096c89b18c4ac4e9c5a9529f0fa3141c463ea306
+  metadata.gz: 0bdc478005838e629b52e310437c5cac89158b8f33429d477c7096f1210c3249
+  data.tar.gz: a8fec4e6229cfd39f1cc0f11c3a8007c2c40442fd6ed2f00a558d2cbd8f01c8d
 SHA512:
-  metadata.gz: 0d1d7714bfeab24e5505ea3d442b423ab3fe6daa26dfe214b412add729e2544543d0ccadf078bec3d852bb3fa2b1ec541b131a064b9a2598e38bebbeb58a6061
-  data.tar.gz: 24a4b3b4dcd85edd9249b16e79673487a174c58e7c826820c4b1a67aed56a8f92ec4e228e0d773598b3ed6ce14b5196a2163b48c63646d3442014fcfc2af561c
+  metadata.gz: a88d97bd1fff6f57c956024d7d47affe13179a02e068129ddda1be0c577324eafee18c271e465d68dc06a99d7cdd78288e6094f307ab4503ee68c963e618bafc
+  data.tar.gz: 6e601c83e09931821e6d00c03114fbaabb72f8c72781f667eea384b7beb916adc3ccc42f83dbd15434779e2dd3e5f574bcb8ef7be022f3154f0afdabec0107b2

data/lib/opener/property_tagger.rb

@@ -1,6 +1,6 @@
 require 'open3'
 require 'slop'
-require 'oga'
+require 'nokogiri'
 require 'monitor'
 require 'httpclient'
 require 'hashie'

data/lib/opener/property_tagger/file_aspects_cache.rb

@@ -36,7 +36,7 @@ module Opener
         mapping = Hash.new { |hash, key| hash[key] = [] }
 
         File.foreach(path) do |line|
-          lemma, pos, aspect = line.chomp.split("\t")
+          lemma, _pos, aspect = line.chomp.split("\t")
 
           mapping[lemma.to_sym] << aspect
         end

data/lib/opener/property_tagger/processor.rb

@@ -25,16 +25,17 @@ module Opener
       #  by default due to the performance overhead.
       #
       def initialize file, params: {}, url: nil, path: nil, timestamp: true, pretty: false
-        @document     = Oga.parse_xml file
+        @document     = Nokogiri.XML file
         raise 'Error parsing input. Input is required to be KAF' unless is_kaf?
         @timestamp    = timestamp
         @pretty       = pretty
 
         @params       = params
-        @cache_keys   = params[:cache_keys] || {lang: language}
         @remote       = !url.nil?
         @aspects_path = path
         @aspects_url  = url
+        @cache_keys   = params[:cache_keys]
+        @cache_keys.merge! lang: @document.root.attr('xml:lang')
 
         @aspects = if @remote then REMOTE_ASPECTS_CACHE[**@cache_keys].aspects else FILE_ASPECTS_CACHE[aspects_file] end
       end
@@ -57,70 +58,64 @@ module Opener
 
         add_linguistic_processor
 
-        return pretty ? pretty_print(document) : document.to_xml
+        pretty ? pretty_print(document) : document.to_xml
       end
 
-      ##
-      # Get the language of the input file.
-      #
-      # @return [String]
-      #
       def language
-        return @language ||= document.at_xpath('KAF').get('xml:lang')
+        @language ||= document.at_xpath('KAF').attr('xml:lang')
       end
 
-      ##
-      # Get the terms from the input file
-      # @return [Hash]
-      #
       def terms
         unless @terms
           @terms = {}
 
           document.xpath('KAF/terms/term').each do |term|
-            @terms[term.get('tid').to_sym] = term.get('lemma')
+            @terms[term.attr('tid').to_sym] = { lemma: term.attr('lemma'), text: term.attr('text')}
           end
         end
 
-        return @terms
+        @terms
       end
 
       ##
       # Check which terms belong to an aspect (property)
+      # Text have priority over Lemmas, overriding if there is a conflict
       # @return [Hash]
       #
       def extract_aspects
-        term_ids  = terms.keys
-        lemmas    = terms.values
+        term_ids     = terms.keys
+        lemmas       = terms.values
+        uniq_aspects = Hash.new { |hash, key| hash[key] = [] }
 
-        current_token = 0
-        # Use of n-grams to determine if a unigram (1 lemma) or bigram (2
-        # lemmas) belong to a property.
-        max_ngram = 2
+        [:lemma, :text].each do |k|
+          current_token = 0
+          # Use of n-grams to determine if a unigram (1 lemma) or bigram (2
+          # lemmas) belong to a property.
+          max_ngram = 2
 
-        uniq_aspects = Hash.new { |hash, key| hash[key] = [] }
 
-        while current_token < terms.count
-          (0..max_ngram).each do |tam_ngram|
-            if current_token + tam_ngram <= terms.count
-              ngram = lemmas[current_token..current_token+tam_ngram].join(" ").downcase
+          while current_token < terms.count
+            (0..max_ngram).each do |tam_ngram|
+              if current_token + tam_ngram <= terms.count
+                ngram = lemmas[current_token..current_token+tam_ngram].map{|a| a[k] }.join(" ").downcase
 
-              if aspects[ngram.to_sym]
-                properties = aspects[ngram.to_sym]
-                ids        = term_ids[current_token..current_token+tam_ngram]
+                if aspects[ngram.to_sym]
+                  properties = aspects[ngram.to_sym]
+                  ids        = term_ids[current_token..current_token+tam_ngram]
 
-                properties.uniq.each do |property|
-                  next if !property or property.strip.empty?
+                  properties.uniq.each do |property|
+                    next if !property or property.strip.empty?
 
-                  uniq_aspects[property.to_sym] << [ids,ngram]
+                    uniq_aspects[property.to_sym] << [ids,ngram] unless uniq_aspects[property.to_sym].include? [ids,ngram]
+                  end
                 end
               end
             end
+            current_token += 1
           end
-          current_token += 1
         end
 
-        return Hash[uniq_aspects.sort]
+        Hash[uniq_aspects.sort]
       end
 
       ##
@@ -143,22 +138,21 @@ module Opener
       def add_property(key, value, index)
         property_node = new_node("property", "KAF/features/properties")
 
-        property_node.set('lemma', key.to_s)
-        property_node.set('pid', "p#{index.to_s}")
+        property_node['lemma'] = key.to_s
+        property_node['pid']   = "p#{index.to_s}"
 
         references_node = new_node("references", property_node)
 
         value.uniq.each do |v|
-          comment = Oga::XML::Comment.new(:text => " #{v.last} ")
-
-          references_node.children << comment
+          comm_node = Nokogiri::XML::Comment.new(references_node, " #{v.last} ")
+          references_node.add_child comm_node
 
           span_node = new_node("span", references_node)
 
           v.first.each do |val|
-            target_node = new_node("target", span_node)
+            target_node       = new_node("target", span_node)
 
-            target_node.set('id', val.to_s)
+            target_node['id'] = val.to_s
           end
         end
       end
@@ -169,19 +163,19 @@ module Opener
         version     = '2.0'
 
         node = new_node('linguisticProcessors', 'KAF/kafHeader')
-        node.set('layer', 'features')
+        node['layer'] = 'features'
 
         lp_node = new_node('lp', node)
 
-        lp_node.set('version', "#{last_edited}-#{version}")
-        lp_node.set('name', description)
+        lp_node['version'] = "#{last_edited}-#{version}"
+        lp_node['name']    = description
 
         if timestamp
           format = '%Y-%m-%dT%H:%M:%S%Z'
 
-          lp_node.set('timestamp', Time.now.strftime(format))
+          lp_node['timestamp'] = Time.now.strftime(format)
         else
-          lp_node.set('timestamp', '*')
+          lp_node['timestamp'] = '*'
         end
       end
 
@@ -200,7 +194,7 @@ module Opener
         formatter.compact = true
         formatter.write(doc, out)
 
-        return out.strip
+        out.strip
       end
 
       protected
@@ -212,11 +206,11 @@ module Opener
           parent_node = parent
         end
 
-        node = Oga::XML::Element.new(:name => tag)
+        node = Nokogiri::XML::Element.new(tag, document)
 
-        parent_node.children << node
+        parent_node.add_child node
 
-        return node
+        node
       end
 
       ##
@@ -224,7 +218,7 @@ module Opener
       # @return [Boolean]
       #
       def is_kaf?
-        return !!document.at_xpath('KAF')
+        !!document.at_xpath('KAF')
       end
 
       ##

data/lib/opener/property_tagger/remote_aspects_cache.rb

@@ -7,6 +7,8 @@ module Opener
 
       include MonitorMixin
 
+      UPDATE_INTERVAL = (ENV['CACHE_EXPIRE_MINS']&.to_i || 5).minutes
+
       def initialize
         super
 
@@ -16,13 +18,9 @@ module Opener
 
       def [] **params
         synchronize do
-          if existing = @cache[params]
-            existing.tap do
-              Thread.new{ @cache[params] = cache_update existing, **params }
-            end
-          else
-            @cache[params] = cache_update **params
-          end
+          existing = @cache[params]
+          break existing if existing and existing.from > UPDATE_INTERVAL.ago
+          @cache[params] = cache_update existing, **params
         end
       end
       alias_method :get, :[]
@@ -31,7 +29,11 @@ module Opener
         from     = Time.now
         lexicons = load_aspects cache: existing, **params
 
-        return existing if existing and lexicons.blank?
+        if existing and lexicons.blank?
+          existing.from = from
+          return existing
+        end
+
         Hashie::Mash.new(
           aspects: lexicons,
           from:    from,
@@ -39,8 +41,8 @@ module Opener
       end
 
       def load_aspects lang:, cache:, **params
-        url      = "#{@url}&language_code=#{lang}&#{params.to_query}"
-        url     += "&if_updated_since=#{cache.from.iso8601}" if cache
+        url  = "#{@url}&language_code=#{lang}&#{params.to_query}"
+        url += "&if_updated_since=#{cache.from.utc.iso8601}" if cache
         puts "#{lang}: loading aspects from #{url}"
 
         lexicons = JSON.parse HTTPClient.new.get(url).body

data/lib/opener/property_tagger/version.rb

@@ -1,7 +1,7 @@
 module Opener
   class PropertyTagger
 
-    VERSION = '3.3.1'
+    VERSION = '3.3.6'
 
   end
 end

data/opener-property-tagger.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'opener-webservice', '~> 2.1'
   gem.add_dependency 'opener-core', '~> 2.2'
 
-  gem.add_dependency 'oga', ['~> 1.0', '>= 1.3.1']
+  gem.add_dependency 'nokogiri'
   gem.add_dependency 'httpclient'
   gem.add_dependency 'hashie'
   gem.add_dependency 'activesupport'

metadata

@@ -1,170 +1,164 @@
 --- !ruby/object:Gem::Specification
 name: opener-property-tagger
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 3.3.6
 platform: ruby
 authors:
 - development@olery.com
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-07 00:00:00.000000000 Z
+date: 2020-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: opener-daemons
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
-  name: opener-daemons
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
 - !ruby/object:Gem::Dependency
+  name: opener-webservice
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
-  name: opener-webservice
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
 - !ruby/object:Gem::Dependency
+  name: opener-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
-  name: opener-core
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
 - !ruby/object:Gem::Dependency
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.1
-  name: oga
-  prerelease: false
+        version: '0'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: '0'
 - !ruby/object:Gem::Dependency
+  name: httpclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: httpclient
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: hashie
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: hashie
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: activesupport
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-  name: rspec
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
+  name: cucumber
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: cucumber
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: benchmark-ips
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-  name: benchmark-ips
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -219,7 +213,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.7.9
+rubygems_version: 2.7.8
 signing_key:
 specification_version: 4
 summary: Property tagger for hotels in Dutch and English.