opener-property-tagger 3.3.5 → 3.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4498897f273195326764dae62875d3901919f69014df57d0919d88b0755ee777
-  data.tar.gz: e0c3c1cfcc32cf260024c2c4b5fee4afb7e6a2b4e46ba690499e8b151e959e52
+  metadata.gz: 181a83398a21579d625df2d6dfbb5e8d03759e2e285c40f3e6fe519f73f270b7
+  data.tar.gz: f2548258178f538a8f7e32e0617fd5f8bea8399373399529fab6316906739543
 SHA512:
-  metadata.gz: 0f0557b5cb1604b6edec19c6d5c9b2d3cc892792d7562209d619cfe0f1f09c04fa0a340f3197e8f5963d0dbefbfb890d18e9051f82705eaa346f20682b724eb1
-  data.tar.gz: 828fb63990b968d4511548b5ed6b05a44574cc9b0ed933745ddb7053cc642400ddf1ff7966f50a55b6dc84223a1fbbb8882c9cb28850dd3a8e98e550e9453183
+  metadata.gz: e3817f4e33c9aedc67d767e4e194a76a7b3d5f87331e7722916dd3f84c4475f51f04763a9dfc7561092eab623aeee2f90c2ab7b4b2a8caf2b99e34708ebe3995
+  data.tar.gz: bb42edeff3f5225972d475c4522da2b91b99484edb94d2a14b646ea7712964e0f22858819e7b0affb4b59ecfd87f4823035ab13d804a76c4b377684bfb7ea7c1

data/lib/opener/property_tagger/file_aspects_cache.rb

@@ -33,12 +33,17 @@ module Opener
       # @param [String] path
       #
       def load_aspects(path)
-        mapping = Hash.new { |hash, key| hash[key] = [] }
+        mapping = Hash.new{ |hash, key| hash[key] = [] }
 
-        File.foreach(path) do |line|
-          lemma, _pos, aspect = line.chomp.split("\t")
+        File.foreach path do |line|
+          lemma, pos, aspect = line.chomp.split("\t")
+          l = Hashie::Mash.new(
+            lemma:  lemma,
+            pos:    pos,
+            aspect: aspect,
+          )
 
-          mapping[lemma.to_sym] << aspect
+          mapping[l.lemma.to_sym] << l
         end
 
         return mapping

data/lib/opener/property_tagger/processor.rb

@@ -6,7 +6,8 @@ module Opener
     class Processor
 
       attr_accessor :document
-      attr_accessor :aspects, :aspects_path, :aspects_url
+      attr_accessor :aspects_path, :aspects_url
+      attr_accessor :aspects, :lexicons
       attr_accessor :timestamp, :pretty
 
       ##
@@ -31,12 +32,13 @@ module Opener
         @pretty       = pretty
 
         @params       = params
-        @cache_keys   = params[:cache_keys] || {lang: language}
         @remote       = !url.nil?
         @aspects_path = path
         @aspects_url  = url
+        @cache_keys   = params[:cache_keys] || {}
+        @cache_keys.merge! lang: @document.root.attr('xml:lang')
 
-        @aspects = if @remote then REMOTE_ASPECTS_CACHE[**@cache_keys].aspects else FILE_ASPECTS_CACHE[aspects_file] end
+        @lexicons = if @remote then REMOTE_ASPECTS_CACHE[**@cache_keys].aspects else FILE_ASPECTS_CACHE[aspects_file] end
       end
 
       ##
@@ -44,15 +46,13 @@ module Opener
       # @return [String]
       #
       def process
-        existing_aspects = extract_aspects
-
         add_features_layer
         add_properties_layer
 
-        existing_aspects.each_with_index do |(key, value), index|
+        extract_aspects.each.with_index do |(lemma, values), index|
           index += 1
 
-          add_property(key, value, index)
+          add_property lemma, values, index
         end
 
         add_linguistic_processor
@@ -76,36 +76,41 @@ module Opener
         @terms
       end
 
+      # Use of n-grams to determine if a unigram (1 lemma) or bigram (2
+      # lemmas) belong to a property.
+      MAX_NGRAM = 2
+
       ##
       # Check which terms belong to an aspect (property)
+      # Text have priority over Lemmas, overriding if there is a conflict
       # @return [Hash]
       #
       def extract_aspects
-        term_ids     = terms.keys
+        all_term_ids = terms.keys
         lemmas       = terms.values
-        uniq_aspects = Hash.new { |hash, key| hash[key] = [] }
+        uniq_aspects = Hash.new{ |hash, lemma| hash[lemma] = [] }
 
         [:lemma, :text].each do |k|
           current_token = 0
-          # Use of n-grams to determine if a unigram (1 lemma) or bigram (2
-          # lemmas) belong to a property.
-          max_ngram = 2
-
 
           while current_token < terms.count
-            (0..max_ngram).each do |tam_ngram|
-              if current_token + tam_ngram <= terms.count
-                ngram = lemmas[current_token..current_token+tam_ngram].map{|a| a[k] }.join(" ").downcase
-
-                if aspects[ngram.to_sym]
-                  properties = aspects[ngram.to_sym]
-                  ids        = term_ids[current_token..current_token+tam_ngram]
-
-                  properties.uniq.each do |property|
-                    next if !property or property.strip.empty?
-
-                    uniq_aspects[property.to_sym] << [ids,ngram] unless uniq_aspects[property.to_sym].include? [ids,ngram]
-                  end
+            (0..MAX_NGRAM).each do |tam_ngram|
+              next unless current_token + tam_ngram <= terms.count
+
+              ngram = lemmas[current_token..current_token+tam_ngram].map{ |a| a[k] }.join(" ").downcase
+
+              @lexicons[ngram.to_sym]&.each do |l|
+                properties = if l.aspects.present? then l.aspects else [l.aspect] end
+                properties.each do |p|
+                  next if p.blank?
+                  term_ids = all_term_ids[current_token..current_token+tam_ngram]
+                  next if uniq_aspects[p.to_sym].find{ |v| v.term_ids == term_ids }
+
+                  uniq_aspects[p.to_sym] << Hashie::Mash.new(
+                    term_ids: term_ids,
+                    ngram:    ngram,
+                    lexicon:  l,
+                  )
                 end
               end
             end
@@ -133,24 +138,25 @@ module Opener
         new_node("properties", "KAF/features")
       end
 
-      def add_property(key, value, index)
+      def add_property lemma, values, index
         property_node = new_node("property", "KAF/features/properties")
 
-        property_node['lemma'] = key.to_s
+        property_node['lemma'] = lemma.to_s
         property_node['pid']   = "p#{index.to_s}"
 
         references_node = new_node("references", property_node)
 
-        value.uniq.each do |v|
-          comm_node = Nokogiri::XML::Comment.new(references_node, " #{v.last} ")
+        values.each do |v|
+          comm_node = Nokogiri::XML::Comment.new(references_node, " #{v.ngram} ")
           references_node.add_child comm_node
 
-          span_node = new_node("span", references_node)
+          span_node = new_node 'span', references_node
 
-          v.first.each do |val|
-            target_node       = new_node("target", span_node)
+          v.term_ids.each do |id|
+            target_node       = new_node 'target', span_node
 
-            target_node['id'] = val.to_s
+            target_node['id'] = id.to_s
+            target_node['lexicon-id'] = v.lexicon.id if v.lexicon.id
           end
         end
       end

data/lib/opener/property_tagger/remote_aspects_cache.rb

@@ -17,10 +17,12 @@ module Opener
       end
 
       def [] **params
+        existing = @cache[params]
+        return existing if existing and existing.from > UPDATE_INTERVAL.ago
+        params[:contract_ids] = nil unless params[:contract_ids]
+
         synchronize do
-          existing = @cache[params]
-          break existing if existing and existing.from > UPDATE_INTERVAL.ago
-          @cache[params] = cache_update existing, **params
+          @cache[params] = cache_update @cache[params], **params
         end
       end
       alias_method :get, :[]
@@ -49,7 +51,10 @@ module Opener
         lexicons = lexicons['data'].map{ |l| Hashie::Mash.new l }
         mapping  = Hash.new{ |hash, key| hash[key] = [] }
         lexicons.each do |l|
-          mapping[l.lemma.to_sym] << l.aspect
+          mapping[l.lemma.to_sym] << l
+          l.variants&.each do |v|
+            mapping[v.lemma.to_sym] << l
+          end
         end
 
         mapping

data/lib/opener/property_tagger/version.rb

@@ -1,7 +1,7 @@
 module Opener
   class PropertyTagger
 
-    VERSION = '3.3.5'
+    VERSION = '3.4.2'
 
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opener-property-tagger
 version: !ruby/object:Gem::Version
-  version: 3.3.5
+  version: 3.4.2
 platform: ruby
 authors:
 - development@olery.com
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-10 00:00:00.000000000 Z
+date: 2021-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opener-daemons
@@ -212,8 +212,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.6
+rubygems_version: 3.2.14
 signing_key:
 specification_version: 4
 summary: Property tagger for hotels in Dutch and English.