browser_web_data_entity_sumarization 1.0.0beta1 → 1.0.0beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b24bd4b2cd4ae8c908f6549710531303a49c36e5
-  data.tar.gz: 0c48cc2e23703a28dbe262c3535703b87698f63f
+  metadata.gz: df328fc4a3b8ad15cd85f3ef8cc226e8090c0890
+  data.tar.gz: c72c4d29438b7cee335d2d72475f478f0c7da0cd
 SHA512:
-  metadata.gz: 786bfb65d8250af636e9a1187cc7530738e734252d405c53bc10c0f0c03ba0ea4953fca5b32613817c50c4c67200df09ea31ff96f0a2ec8f0c481b53a965e00b
-  data.tar.gz: 3448f20b4aa7b7bd457da33479435fd557ef14c87f13264cf3a99286ecd053511cbf341f0ae2f1cb75a516fe5bf4d9e73bd6a0e46b7de701d93d1938f6851c12
+  metadata.gz: 5824a7f65f83928f79d5099534c892b8f2728c9c24fab480572da8b124c48b53d1d17014d87b09f72e1b18f76f85ff4ddc493f1ab598e782094e18afa23bd126
+  data.tar.gz: cd38b64882daa6f91e3dee375ddc80709b0f69f695c4b117e225b2b1dc88a5ca2f657939a883bc40580d463fe8e1b9b190d8bc3f9f3cc07b43b9251d04496448

data/lib/browser_web_data_entity_sumarization/entity_sumarization_predicates_similarity.rb

@@ -22,6 +22,7 @@ module BrowserWebData
 
         load_identical_predicates
         load_different_predicates
+        load_counts
       end
 
       ###
@@ -32,7 +33,7 @@ module BrowserWebData
       # @return [String] key
       def self.get_key(predicates)
         predicates = [predicates] unless predicates.is_a?(Array)
-        "<#{predicates.join('><')}>" if predicates && !predicates.empty?
+        "<#{predicates.sort.join('><')}>" if predicates && !predicates.empty?
       end
 
       ###
@@ -52,9 +53,12 @@ module BrowserWebData
       #
       # @param [Array<String>] predicates
       def identify_identical_predicates(predicates, identical_limit = @identical_limit)
-        @temp_counts ||= {}
+        combination = predicates.take(IMPORTANCE_TO_IDENTIFY_MAX_COUNT).map { |p| p.to_sym }.combination(2)
+        five_times_count = combination.size / 20
+
+        combination.each_with_index { |values, i|
+
 
-        predicates.combination(2).each { |values|
 
           already_mark_same = find_identical(values)
           already_mark_different = find_different(values)
@@ -65,16 +69,16 @@ module BrowserWebData
             # automatically became identical
             unless try_auto_identical(values)
 
-              unless @temp_counts[values[0]]
-                @temp_counts[values[0]] = @query.get_count_of_identical_predicates(values[0])
+              unless @counts[values[0]]
+                @counts[values[0]] = @query.get_count_of_identical_predicates(values[0])
               end
 
-              unless @temp_counts[values[1]]
-                @temp_counts[values[1]] = @query.get_count_of_identical_predicates(values[1])
+              unless @counts[values[1]]
+                @counts[values[1]] = @query.get_count_of_identical_predicates(values[1])
               end
 
-              x = @temp_counts[values[0]]
-              y = @temp_counts[values[1]]
+              x = @counts[values[0]]
+              y = @counts[values[1]]
               z = @query.get_count_of_identical_predicates(values)
 
               identical_level = z / [x, y].max
@@ -86,12 +90,15 @@ module BrowserWebData
                 add_different(values)
               end
             end
+          end
 
+          if @console_output && ( i == 0 || (i+1) % five_times_count == 0 )
+            puts "#{Time.now.localtime} | #{(((i+1)/combination.size) * 100).round(2)}% | [#{(i+1)}/#{combination.size}]"
           end
 
-          true
         }
 
+        store_counts
       end
 
       ###
@@ -235,6 +242,13 @@ module BrowserWebData
         end
       end
 
+      def load_counts
+        unless @counts
+          file_path = "#{@results_dir_path}/counts.json"
+          @counts = ensure_load_json(file_path, {})
+        end
+      end
+
       def store_identical_properties
         File.write("#{@results_dir_path}/different_predicates.json", JSON.generate(@different_predicates))
       end
@@ -243,6 +257,11 @@ module BrowserWebData
         File.write("#{@results_dir_path}/different_predicates.json", JSON.generate(@different_predicates))
       end
 
+      def store_counts
+        File.write("#{@results_dir_path}/counts.json", JSON.generate(@counts))
+      end
+
+
       def ensure_load_json(file_path, def_val, json_params = {})
         if File.exists?(file_path)
           file_data = File.read(file_path).force_encoding('utf-8')

data/lib/browser_web_data_entity_sumarization/entity_sumarization_statistics.rb

@@ -36,7 +36,7 @@ module BrowserWebData
         @console_output = console_output
 
         @query = SPARQLRequest.new
-        @predicates_similarity = PredicatesSimilarity.new(@results_dir_path)
+        @predicates_similarity = PredicatesSimilarity.new(@results_dir_path, console_output)
       end
 
       ###
@@ -251,25 +251,38 @@ module BrowserWebData
       # to one result knowledge base file.
       #
       # @param [String] type Type from http://mappings.dbpedia.org/server/ontology/classes/
-      # @param [Fixnum] best_count Define max count of properties that will be assign to entity class type.
       # @param [TrueClass, FalseClass] identify_identical Flag for process identify and group identical properties as one item.
-      def generate_knowledge_base(type, best_count = 20, identify_identical = true)
+      def generate_knowledge_base(type, identify_identical = true)
         puts "_____ #{type} _____" if @console_output
         files = Dir.glob("#{@results_dir_path}/#{type}/*.json")
         type = type.to_s.to_sym
 
         knowledge_data = {type => []}
 
-        files.each { |file_path|
-          puts "- calculate #{file_path}" if @console_output
-          file_data = JSON.parse(File.read(file_path).force_encoding('utf-8'), symbolize_names: true)
+        global_properties = get_global_statistic_by_type(type) || {}
+
+        if identify_identical
+          try_this_identical = {}
 
-          if identify_identical
+          files.each { |file_path|
+            file_data = JSON.parse(File.read(file_path).force_encoding('utf-8'), symbolize_names: true)
             file_data[:nif_data].each { |data|
-              all_properties = data[:properties][type].keys + ((data[:strict_properties]||{})[type] || {}).keys.uniq
-              @predicates_similarity.identify_identical_predicates(all_properties)
+              try_this_identical.merge!(data[:properties][type]) { |_, x, y| x + y }
             }
+          }
+
+          try_this_identical.merge!(global_properties) { |_, x, y| x + y }
+
+          if try_this_identical.size > 0
+            try_this_identical = Hash[try_this_identical.sort_by { |_,v|v}.reverse]
+            puts "- prepare to identify identical: total count #{try_this_identical.size}" if @console_output
+            @predicates_similarity.identify_identical_predicates(try_this_identical.keys)
           end
+        end
+
+        puts "- calculate: files count #{files.size}" if @console_output
+        files.each { |file_path|
+          file_data = JSON.parse(File.read(file_path).force_encoding('utf-8'), symbolize_names: true)
 
           file_data[:nif_data].each { |found|
 
@@ -309,10 +322,6 @@ module BrowserWebData
           end
         }
 
-        global_properties = get_global_statistic_by_type(type) || {}
-        if identify_identical
-          @predicates_similarity.identify_identical_predicates(global_properties.keys)
-        end
 
         if global_properties.size > 0
           max_count = global_properties.max_by { |_, count| count }[1].to_f
@@ -331,7 +340,11 @@ module BrowserWebData
           hash
         }
 
-        knowledge_data[type] = knowledge_data[type].sort_by { |hash| hash[:score] }.reverse.take(best_count)
+        knowledge_data[type] = knowledge_data[type].keep_if { |hash|
+          hash[:score] > 0
+        }.sort_by { |hash|
+          hash[:score]
+        }.reverse
 
         if identify_identical
           @predicates_similarity.reduce_identical
@@ -383,7 +396,7 @@ module BrowserWebData
       # @param [String] path
       #
       # @return [Hash] classes
-      def get_all_classes(path = File.join(__dir__,'../knowledge/classes_hierarchy.json'))
+      def get_all_classes(path = File.join(__dir__, '../knowledge/classes_hierarchy.json'))
         data = ensure_load_json(path, {})
         HashHelper.recursive_map_keys(data)
       end

data/lib/browser_web_data_entity_sumarization/version.rb

@@ -1,3 +1,3 @@
 module EntitySumarization
-  VERSION = '1.0.0beta1'
+  VERSION = '1.0.0beta2'
 end

data/lib/config/entity_sumarization_config.rb

@@ -8,6 +8,8 @@ module BrowserWebData
 
     IDENTICAL_PROPERTY_LIMIT = 0.8
 
+    IMPORTANCE_TO_IDENTIFY_MAX_COUNT = 250
+
     NO_SENSE_PROPERTIES = %w(
       http://xmlns.com/foaf/0.1/primaryTopic
       http://dbpedia.org/ontology/wikiPageRedirects

data/lib/utils/cache_helper.rb

@@ -55,6 +55,33 @@ module CacheHelper
     HashHelper.recursive_symbolize_keys(hash)
   end
 
+  ###
+  # The method helps to update knowledge by key in yield block.
+  #
+  # @param [String] key Key of stored knowledge.
+  #
+  # @yield param actual_data
+  # @yield return new_data
+  def self.update_knowledge(key)
+    dir_path = "#{File.dirname(File.expand_path('..', __FILE__))}/knowledge"
+    file_path = "#{dir_path}/#{StringHelper.get_clear_file_path(key)}.json"
+
+    hash = {}
+    if !File.exists?(file_path)
+
+      if block_given?
+        hash = yield hash
+        File.open(file_path, 'w') { |f| f.puts hash.to_json } unless hash.empty?
+      end
+    else
+      old_hash = JSON.parse(File.read(file_path).force_encoding('UTF-8'), symbolize_names: true)
+      hash = yield old_hash
+      File.open(file_path, 'w') { |f| f.puts hash.to_json } unless hash.empty?
+    end
+
+    HashHelper.recursive_symbolize_keys(hash)
+  end
+
   ###
   # The method helps to get build in knowledge by key.
   #

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: browser_web_data_entity_sumarization
 version: !ruby/object:Gem::Version
-  version: 1.0.0beta1
+  version: 1.0.0beta2
 platform: ruby
 authors:
 - Marek Filteš
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-04-12 00:00:00.000000000 Z
+date: 2017-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement