browser_web_data_entity_sumarization 1.0.0beta1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b24bd4b2cd4ae8c908f6549710531303a49c36e5
+  data.tar.gz: 0c48cc2e23703a28dbe262c3535703b87698f63f
+SHA512:
+  metadata.gz: 786bfb65d8250af636e9a1187cc7530738e734252d405c53bc10c0f0c03ba0ea4953fca5b32613817c50c4c67200df09ea31ff96f0a2ec8f0c481b53a965e00b
+  data.tar.gz: 3448f20b4aa7b7bd457da33479435fd557ef14c87f13264cf3a99286ecd053511cbf341f0ae2f1cb75a516fe5bf4d9e73bd6a0e46b7de701d93d1938f6851c12

data/lib/browser_web_data_entity_sumarization.rb

@@ -0,0 +1,23 @@
+#encoding: utf-8
+require 'sparql/client'
+require 'benchmark'
+require 'json'
+
+module BrowserWebData
+  module EntitySumarization
+
+  end
+end
+
+Dir.glob(File.dirname(__FILE__) + '/utils/*.rb').each { |file| require file }
+Dir.glob(File.dirname(__FILE__) + '/config/*.rb').each { |file| require file }
+
+# Require all gem scripts by their relative names
+Dir[File.dirname(__FILE__) + '/browser_web_data_entity_sumarization/**/*.rb'].each do |file|
+  require(file.gsub('\\', '/').split('/lib/').last[0..-4])
+end
+
+
+
+
+

data/lib/browser_web_data_entity_sumarization/entity_sumarization_nif_parser.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+
+###
+# Core project module
+module BrowserWebData
+
+  ###
+  # Project logic module
+  module EntitySumarization
+
+    ###
+    # The class include helpers to retrieve structured nif data from nif lines.
+    class NIFLineParser
+      include BrowserWebData::EntitySumarizationConfig
+
+      ###
+      # The method apply scan to recognize resource uri from given nif dataset line.
+      #
+      # @param [String] line
+      #
+      # @return [String] resource_uri
+      # @example resource_uri: "http://dbpedia.org/resource/Captain_EO"
+      def self.parse_resource_uri(line)
+        (line.scan(SCAN_REGEXP[:scan_resource])[0])[0].split('?').first
+      end
+
+      ###
+      # The method apply scan to recognize link, anchor, indexes and section from given nif dataset group of 7 lines.
+      #
+      # @param [Array<String>] lines_group
+      #
+      # @return [Hash] nif_data
+      # @example nif_data:
+      # {
+      #   link: "http://dbpedia.org/resource/Science_fiction_film",
+      #   anchor: "science fiction film",
+      #   indexes:  ["33", "53"],
+      #   section: "paragraph_0_419"
+      # }
+      def self.parse_line_group(lines_group)
+        begin_index = lines_group[2].scan(SCAN_REGEXP[:begin_index])[0]
+        end_index = lines_group[3].scan(SCAN_REGEXP[:end_index])[0]
+        target_resource_link = lines_group[5].scan(SCAN_REGEXP[:target_resource_link])[0]
+        section = lines_group[4].scan(SCAN_REGEXP[:section])[0]
+        anchor = lines_group[6].scan(SCAN_REGEXP[:anchor])[0]
+
+        {
+          link: target_resource_link[1].force_encoding('utf-8'),
+          anchor: anchor[1].force_encoding('utf-8'),
+          indexes: [begin_index[1], end_index[1]],
+          section: section[0].split('=')[1]
+        }
+      end
+
+    end
+
+  end
+
+end

data/lib/browser_web_data_entity_sumarization/entity_sumarization_predicate.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+
+###
+# Core project module
+module BrowserWebData
+
+  ###
+  # Project logic module
+  module EntitySumarization
+
+
+    ###
+    # The class include helper methods.
+    # (todo definition of predicate instance)
+    class Predicate
+      include BrowserWebData::EntitySumarizationConfig
+
+      ###
+      # The method helps identify unimportant predicate by constants.
+      #
+      # @param [String] property
+      #
+      # @return [TrueClass, FalseClass] result
+      def self.unimportant?(property)
+        property = property.to_s
+        NO_SENSE_PROPERTIES.include?(property) || COMMON_PROPERTIES.include?(property)
+      end
+
+    end
+
+  end
+
+end

data/lib/browser_web_data_entity_sumarization/entity_sumarization_predicates_similarity.rb

@@ -0,0 +1,263 @@
+# encoding: utf-8
+
+###
+# Core project module
+module BrowserWebData
+
+  ###
+  # Project logic module
+  module EntitySumarization
+
+    ###
+    # The class include methods to identify identical predicates
+    class PredicatesSimilarity
+      include BrowserWebData::EntitySumarizationConfig
+
+      def initialize(results_dir_path, identical_limit = IDENTICAL_PROPERTY_LIMIT, console_output = false)
+        @results_dir_path = results_dir_path
+        @console_output = console_output
+        @identical_limit = identical_limit
+
+        @query = SPARQLRequest.new
+
+        load_identical_predicates
+        load_different_predicates
+      end
+
+      ###
+      # The method return key of identical predicates
+      #
+      # @param [Array<String>] predicates
+      #
+      # @return [String] key
+      def self.get_key(predicates)
+        predicates = [predicates] unless predicates.is_a?(Array)
+        "<#{predicates.join('><')}>" if predicates && !predicates.empty?
+      end
+
+      ###
+      # The method return identical predicates by key
+      #
+      # @param [String] key
+      #
+      # @return [Array<String>] predicates
+      def self.parse_key(key)
+        key.to_s.scan(SCAN_REGEXP[:identical_key]).reduce(:+)
+      end
+
+      ###
+      # The method verify every combination of two predicates.
+      # Method store identify combination in two files identical_predicates.json and different_predicates.json
+      # files contains Array of combination keys
+      #
+      # @param [Array<String>] predicates
+      def identify_identical_predicates(predicates, identical_limit = @identical_limit)
+        @temp_counts ||= {}
+
+        predicates.combination(2).each { |values|
+
+          already_mark_same = find_identical(values)
+          already_mark_different = find_different(values)
+
+          if already_mark_same.nil? && already_mark_different.nil?
+
+            # in case of dbpedia ontology vs. property
+            # automatically became identical
+            unless try_auto_identical(values)
+
+              unless @temp_counts[values[0]]
+                @temp_counts[values[0]] = @query.get_count_of_identical_predicates(values[0])
+              end
+
+              unless @temp_counts[values[1]]
+                @temp_counts[values[1]] = @query.get_count_of_identical_predicates(values[1])
+              end
+
+              x = @temp_counts[values[0]]
+              y = @temp_counts[values[1]]
+              z = @query.get_count_of_identical_predicates(values)
+
+              identical_level = z / [x, y].max
+
+              if identical_level >= identical_limit
+                puts "     - result[#{identical_level}] z[#{z}] x[#{x}] y[#{y}] #{values.inspect}" if @console_output
+                add_identical(values)
+              else
+                add_different(values)
+              end
+            end
+
+          end
+
+          true
+        }
+
+      end
+
+      ###
+      # The method helps to recognize if is already marked as identical properties
+      #
+      # @param [Array<String>, String] value
+      #
+      # @return [String, NilClass]
+      def find_identical(value)
+        raise RuntimeError.new('No support identify identical for more than 2 predicates.') if value.is_a?(Array) && value.size >2
+
+        predicates_key = case value
+                           when Array
+                             value = value.map { |v| PredicatesSimilarity.get_key(v) }
+                             @identical_predicates.find { |p|
+                               p[value[0]] && p[value[1]]
+                             }
+                           else
+                             value = PredicatesSimilarity.get_key(value)
+                             @identical_predicates.find { |p|
+                               p[value]
+                             }
+                         end
+
+        PredicatesSimilarity.parse_key(predicates_key)
+      end
+
+      ###
+      # The method helps to recognize if is already marked as different properties
+      #
+      # @param [Array<String>, String] value
+      #
+      # @return [String, NilClass]
+      def find_different(value)
+        raise RuntimeError.new('No support identify identical for more than 2 predicates.') if value.is_a?(Array) && value.size >2
+
+        key = case value
+                when Array
+                  value = value.map { |v| PredicatesSimilarity.get_key(v) }
+                  @different_predicates.find { |p| p[value[0]] && p[value[1]] }
+                else
+                  value = PredicatesSimilarity.get_key(value)
+                  @different_predicates.find { |p| p[value] }
+              end
+
+        PredicatesSimilarity.parse_key(key)
+      end
+
+      def add_identical(values)
+        values = values.map { |p| p.to_s }.uniq.sort
+        group_key = PredicatesSimilarity.get_key(values)
+
+        unless @identical_predicates.include?(group_key)
+          @identical_predicates << group_key
+          store_identical_properties
+        end
+      end
+
+      def add_different(values)
+        values = values.map { |p| p.to_s }.uniq.sort
+        group_key = PredicatesSimilarity.get_key(values)
+
+        unless @different_predicates.include?(group_key)
+          @different_predicates << group_key
+
+          @new_diff_counter ||= 0
+          @new_diff_counter += 1
+
+          if @new_diff_counter > 100
+            store_different_predicates
+            @new_diff_counter = 0
+          end
+
+        end
+      end
+
+      def try_auto_identical(values)
+        group_key = PredicatesSimilarity.get_key(values)
+
+        temp = values.map { |val| val.to_s.split('/').last }.uniq
+        if temp.size == 1 && group_key['property/'] && group_key['ontology/']
+          add_identical(values)
+          true
+        else
+          false
+        end
+      end
+
+
+      ###
+      # The method helps to reduce identical predicates by join of common predicate
+      def reduce_identical
+        new_identical = []
+
+        @identical_predicates.each { |key|
+          values = PredicatesSimilarity.parse_key(key)
+          next if new_identical.find { |v| !(v & values).empty? }
+
+          ## find nodes with values predicates
+          values = recursive_find_identical(key, values)
+
+          new_identical << values.uniq.sort
+        }
+
+        @identical_predicates = new_identical.map { |v| PredicatesSimilarity.get_key(v) }
+
+        store_identical_properties
+      end
+
+      def recursive_find_identical(keys, values)
+        keys = [keys] unless keys.is_a?(Array)
+
+        @identical_predicates.each { |this_key|
+          next if keys.include?(this_key)
+          temp = PredicatesSimilarity.parse_key(this_key)
+
+          unless (temp & values).empty?
+            keys << this_key
+            return recursive_find_identical(keys, (values + temp).uniq)
+          end
+        }
+
+        values
+      end
+
+
+      private
+
+
+      def load_identical_predicates
+        unless @identical_predicates
+          file_path = "#{@results_dir_path}/identical_predicates.json"
+          @identical_predicates = ensure_load_json(file_path, [])
+        end
+      end
+
+      def load_different_predicates
+        unless @different_predicates
+          file_path = "#{@results_dir_path}/different_predicates.json"
+          @different_predicates = ensure_load_json(file_path, [])
+        end
+      end
+
+      def store_identical_properties
+        File.write("#{@results_dir_path}/different_predicates.json", JSON.generate(@different_predicates))
+      end
+
+      def store_different_predicates
+        File.write("#{@results_dir_path}/different_predicates.json", JSON.generate(@different_predicates))
+      end
+
+      def ensure_load_json(file_path, def_val, json_params = {})
+        if File.exists?(file_path)
+          file_data = File.read(file_path).force_encoding('utf-8')
+          if file_data.size >= 2 # '[]'
+            JSON.parse(file_data, json_params)
+          else
+            def_val
+          end
+        else
+          def_val
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/browser_web_data_entity_sumarization/entity_sumarization_statistics.rb

@@ -0,0 +1,609 @@
+# encoding: utf-8
+
+###
+# Core project module
+module BrowserWebData
+
+  ###
+  # Project logic module
+  module EntitySumarization
+
+    ###
+    # Statistic class allow to find, collect and generate knowledge of entity sumarization.
+    # Entity sumarization is based on use dataset of NLP Interchange Format (NIF).
+    # For example datasets from http://wiki.dbpedia.org/nif-abstract-datasets
+    # Knowledge is generate by information in DBpedia.
+    class Statistic
+      include BrowserWebData::EntitySumarizationConfig
+
+      attr_reader :nif_file_path, :results_dir_path
+
+      ###
+      # Create new instance.
+      #
+      # @param [String] nif_dataset_path
+      # @param [String] results_dir_path
+      # @param [TrueClass, FalseClass] console_output Allow puts info to console. Default is false.
+      def initialize(nif_dataset_path, results_dir_path = File.join(__dir__, '../../results'), console_output = false)
+        nif_dataset_path = nif_dataset_path.gsub('\\', '/')
+        results_dir_path = results_dir_path.gsub('\\', '/').chomp('/')
+
+        return false unless File.exists?(nif_dataset_path)
+        return false unless File.exists?(results_dir_path)
+
+        @nif_file_path = nif_dataset_path.gsub('\\', '/')
+        @results_dir_path = results_dir_path.gsub('\\', '/').chomp('/')
+        @console_output = console_output
+
+        @query = SPARQLRequest.new
+        @predicates_similarity = PredicatesSimilarity.new(@results_dir_path)
+      end
+
+      ###
+      # The method find resource links in given nif file dataset.
+      #
+      # @param [Hash] params
+      # @option params [Array<String>, String] :entity_types Types from http://mappings.dbpedia.org/server/ontology/classes/
+      # @option params [Fixnum] :entity_count Best ranked resources by every entity type.
+      # @option params [Fixnum] :best_score_count Count of result predicates to keep.
+      # @option params [FalseClass, TruesClass] :demand_reload
+      # @option params [FalseClass, TruesClass] :identity_identical_predicates
+      def create_by_nif_dataset(params)
+        params[:entity_types] = [params[:entity_types]] unless params[:entity_types].is_a?(Array)
+
+        generate_statistics_from_nif(params[:entity_types], params[:entity_count], params[:demand_reload])
+
+        params[:entity_types].each { |type|
+          generate_knowledge_base(type, params[:best_score_count], params[:identity_identical_predicates])
+        }
+      end
+
+      ###
+      # The method return list of best ranked resources by required entity types.
+      #
+      # @param [Array<String>, String] entity_types Types from http://mappings.dbpedia.org/server/ontology/classes/
+      # @param [Fixnum] count Count of best ranked resources
+      #
+      # @return [Hash] resources
+      def get_best_ranked_resources(entity_types, count = 10)
+        resources = {}
+        entity_types = [entity_types] unless entity_types.is_a?(Array)
+
+        entity_types.each { |type|
+          top_ranked_entities = @query.get_resources_by_dbpedia_page_rank(type, count)
+
+          top_ranked_entities.each { |solution|
+            resources[solution.entity.value] = {type: type, rank: solution.rank.value.to_f}
+          }
+        }
+
+        resources
+      end
+
+      ###
+      # The method find links in given nif dataset. After find collect relations #find_relations.
+      # For each resource generate file in @results_dir_path.
+      #
+      # @param [Array<String>, String] entity_types Types from http://mappings.dbpedia.org/server/ontology/classes/
+      # @param [Fixnum] count Count of best ranked resources
+      # @param [FalseClass, TruesClass] demand_reload
+      def generate_statistics_from_nif(entity_types, count = 10, demand_reload = false)
+        resources = get_best_ranked_resources(entity_types, count)
+
+        resources = keep_unloaded(resources) unless demand_reload
+
+        actual_resource_data = []
+        lines_group = []
+
+        begin
+          time_start = Time.now
+          nif_file = File.open(@nif_file_path, 'r')
+          line = nif_file.readline
+
+          until nif_file.eof?
+            line = nif_file.readline
+
+            if lines_group.size == 7
+              # evaulate group (7 lines)
+              this_resource_uri = NIFLineParser.parse_resource_uri(lines_group[0])
+
+              if resources.keys.include?(this_resource_uri)
+                # process group, is requested
+                resource_uri = this_resource_uri
+                actual_resource_data << NIFLineParser.parse_line_group(lines_group)
+
+              elsif !actual_resource_data.empty?
+                # resource changed, process actual_resource_data
+                resource_hash = resources.delete(resource_uri)
+                type = resource_hash[:type]
+
+                this_time = (Time.now - time_start).round(2)
+                puts "\n#{resource_uri}\n- nif found in #{this_time}\n- resources to find #{resources.size}" if @console_output
+
+                result_relations = find_relations(resource_uri, actual_resource_data, type)
+                generate_result_file(resource_uri, type, result_relations, this_time)
+
+                break if resources.empty?
+
+                actual_resource_data = []
+                time_start = Time.now
+              end
+
+              # start new group
+              lines_group = [line]
+            else
+
+              # join line to group
+              lines_group << line
+            end
+
+          end
+
+        ensure
+          nif_file.close if nif_file && !nif_file.closed?
+        end
+      end
+
+      ###
+      # The method helps to recollect relations by already generated result files.
+      #
+      # @param [Array<String>, String] entity_types Types from http://mappings.dbpedia.org/server/ontology/classes/
+      # @param [Fixnum] count Count of best ranked resources
+      def refresh_statistics_in_files(entity_types, count = 10)
+        resources = get_best_ranked_resources(entity_types, count)
+
+        resources = keep_loaded(resources)
+
+        resources.each { |resource_uri, resource_info|
+          puts "_____ #{resource_uri} _____" if @console_output
+
+          update_nif_file_properties(resource_uri, resource_info[:type]) { |link|
+            get_predicates_by_link(resource_uri, link, resource_info[:type])
+          }
+        }
+
+      end
+
+
+      ###
+      # The method find predicates by given link.
+      # Find strict predicates that are in relation: <resource> ?predicate <link> .
+      # Find predicates that are in relation: ?subject a <type> . ?subject ?predicate <link>
+      #
+      # @param [String] resource_uri Resource for which will be find strict properties
+      # @param [String] link Link that has some importance to resource or entity type.
+      # @param [String] type Type from http://mappings.dbpedia.org/server/ontology/classes/
+      #
+      # @return [Hash] result
+      def get_predicates_by_link(resource_uri, link, type)
+        properties = {type => {}}
+        strict_properties = {type => {}}
+
+        @query.get_all_predicates_by_subject_object(resource_uri, link).each { |solution|
+          predicate = solution.to_h
+          property = predicate[:property].to_s.force_encoding('utf-8')
+
+          next if Predicate.unimportant?(property)
+
+          count = @query.get_count_predicate_by_entity(type, property)[0].to_h[:count].to_f
+          strict_properties[type][property] = count if count > 0
+        }
+
+        @query.get_all_predicates_by_object(link).each { |solution|
+          predicate = solution.to_h
+          property = predicate[:property].to_s.force_encoding('utf-8')
+
+          next if Predicate.unimportant?(property) || strict_properties[type][property]
+
+          count = @query.get_count_predicate_by_entity(type, property)[0].to_h[:count].to_f
+          properties[type][property] = count if count > 0
+        }
+
+
+        {properties: properties, strict_properties: strict_properties}
+      end
+
+      ###
+      # The method helps to store founded information from nif for given resource.
+      #
+      # @param [String] resource_uri
+      # @param [String] type Type from http://mappings.dbpedia.org/server/ontology/classes/
+      # @param [Hsah] result_relations Hash generated by method #find_relations
+      # @option result_relations [Hash] :sections Contains key 'section_type' value 'position'
+      # @option result_relations [Array<Hash>] :relations Hashes generated by method #get_predicates_by_link
+      #
+      # @param [Float] this_time Relative time of find in nif dataset.
+      def generate_result_file(resource_uri, type, result_relations, this_time)
+        section_degradation = result_relations[:sections].map { |section_type, position|
+          index = result_relations[:sections].keys.index(section_type)
+
+          # recognize value of degradation by relative position paragraphs in document
+          position[:degradation] = 1 - ((index / result_relations[:sections].size) / 10.0)
+
+          {section_type => position}
+        }.reduce(:merge)
+
+        total_size = section_degradation.max_by { |_, v| v[:to] }[1][:to].to_f
+
+        result_nif_data = result_relations[:relations].map { |relation|
+          paragraph_position = section_degradation[relation[:section]]
+
+          # weight is lowest by relative distance from document start
+          position_weight = (1 - ((relation[:indexes][0].to_i) / total_size))
+          # weight is also degraded by index of paragraph
+          relation[:weight] = (position_weight * paragraph_position[:degradation]).round(4)
+
+          relation
+        }
+
+        result = {
+          process_time: {nif_find: this_time, relations_find: result_relations[:time]},
+          resource_uri: resource_uri,
+          nif_data: result_nif_data
+        }
+
+        result_path = get_resource_file_path(resource_uri, type)
+        File.open(result_path, 'w:utf-8') { |f| f << JSON.pretty_generate(result) }
+      end
+
+      ###
+      # The method process all generated result files from nif dataset (by entity class type)
+      # to one result knowledge base file.
+      #
+      # @param [String] type Type from http://mappings.dbpedia.org/server/ontology/classes/
+      # @param [Fixnum] best_count Define max count of properties that will be assign to entity class type.
+      # @param [TrueClass, FalseClass] identify_identical Flag for process identify and group identical properties as one item.
+      def generate_knowledge_base(type, best_count = 20, identify_identical = true)
+        puts "_____ #{type} _____" if @console_output
+        files = Dir.glob("#{@results_dir_path}/#{type}/*.json")
+        type = type.to_s.to_sym
+
+        knowledge_data = {type => []}
+
+        files.each { |file_path|
+          puts "- calculate #{file_path}" if @console_output
+          file_data = JSON.parse(File.read(file_path).force_encoding('utf-8'), symbolize_names: true)
+
+          if identify_identical
+            file_data[:nif_data].each { |data|
+              all_properties = data[:properties][type].keys + ((data[:strict_properties]||{})[type] || {}).keys.uniq
+              @predicates_similarity.identify_identical_predicates(all_properties)
+            }
+          end
+
+          file_data[:nif_data].each { |found|
+
+            properties = found[:properties][type.to_sym]
+            strict_properties = (found[:strict_properties] ||{})[type] || {}
+            weight = found[:weight]
+
+            strict_properties.each { |property, count|
+              property = property.to_s
+              value = count.to_i * weight
+
+              prepare_property_to_knowledge(property, knowledge_data[type]) { |from_knowledge|
+                old_score = from_knowledge[:score] * from_knowledge[:counter]
+                from_knowledge[:counter] += 1
+                (old_score + value) / from_knowledge[:counter]
+              }
+            }
+
+            properties.each { |property, count|
+              property = property.to_s
+              value = count.to_i * weight
+
+              prepare_property_to_knowledge(property, knowledge_data[type]) { |from_knowledge|
+                old_score = from_knowledge[:score] * from_knowledge[:counter]
+                from_knowledge[:counter] += 1
+                (old_score + value) / from_knowledge[:counter]
+              }
+            }
+          }
+
+          unless knowledge_data[type].empty?
+            max_weight = knowledge_data[type].max_by { |data| data[:score] }[:score]
+            knowledge_data[type] = knowledge_data[type].map { |hash|
+              hash[:score] = (hash[:score] / max_weight).round(4)
+              hash
+            }
+          end
+        }
+
+        global_properties = get_global_statistic_by_type(type) || {}
+        if identify_identical
+          @predicates_similarity.identify_identical_predicates(global_properties.keys)
+        end
+
+        if global_properties.size > 0
+          max_count = global_properties.max_by { |_, count| count }[1].to_f
+          global_properties.each { |property, count|
+
+            value = count / max_count
+
+            prepare_property_to_knowledge(property, knowledge_data[type]) { |from_knowledge|
+              from_knowledge[:score] > 0 ? ((from_knowledge[:score] + value) / 2.0).round(4) : value.round(4)
+            }
+          }
+        end
+
+        knowledge_data[type].map! { |hash|
+          hash.delete(:counter)
+          hash
+        }
+
+        knowledge_data[type] = knowledge_data[type].sort_by { |hash| hash[:score] }.reverse.take(best_count)
+
+        if identify_identical
+          @predicates_similarity.reduce_identical
+        end
+
+        update_knowledge_base(knowledge_data)
+      end
+
+      ###
+      # The method generate simple statistics that contain all predicates that links to literal.
+      # Predicates are grouped by entity class type and also contains count of total occurrence.
+      # Predicates find from best ranked resources.
+      #
+      # @param [String] type Type from http://mappings.dbpedia.org/server/ontology/classes/
+      # @param [Fixnum] count Count of best ranked resources
+      def generate_literal_statistics(type = nil, count = 10)
+        unless type
+          type = get_all_classes
+        end
+
+        type = [type] unless type.is_a?(Array)
+
+        type.each_with_index { |entity_type, index|
+          all_properties = {}
+          puts "#{__method__} - start process entity type: #{entity_type} [#{(index / type.size.to_f).round(2)}]" if @console_output
+          entity_type = entity_type.to_s.to_sym
+
+          get_best_ranked_resources(entity_type, count).each { |resource, _|
+            properties = @query.get_all_predicates_by_subject(resource.to_s, true).map { |solution_prop|
+              solution_prop[:property].to_s
+            } || []
+
+            properties.uniq.each { |prop|
+              next if Predicate.unimportant?(prop)
+              all_properties[entity_type] ||= {}
+              all_properties[entity_type][prop] ||= 0
+              all_properties[entity_type][prop] += 1
+            }
+
+          }
+
+          update_global_statistic(all_properties)
+        }
+      end
+
+      ###
+      # The method load all defined entity class types by http://mappings.dbpedia.org/server/ontology/classes/
+      #
+      # @param [String] path
+      #
+      # @return [Hash] classes
+      def get_all_classes(path = File.join(__dir__,'../knowledge/classes_hierarchy.json'))
+        data = ensure_load_json(path, {})
+        HashHelper.recursive_map_keys(data)
+      end
+
+
+      private
+
+      ###
+      # The method helps to continue of process find links in nif dataset.
+      #
+      # @param [String] resource_uri
+      # @param [Hash] actual_resource_data Part data extracted from nif dataset for given resource_uri
+      # @param [String] type Type from http://mappings.dbpedia.org/server/ontology/classes/
+      #
+      # @return [Hash] out
+      def find_relations(resource_uri, actual_resource_data, type)
+        out = {
+          sections: {},
+          relations: []
+        }
+
+        puts "- properties to find size[#{actual_resource_data.size}]" if @console_output
+
+        time = Benchmark.realtime {
+          out[:relations] = actual_resource_data.map! { |resource_data|
+            section_group = resource_data[:section].scan(SCAN_REGEXP[:group])
+
+            type_key = resource_data[:section].force_encoding('utf-8')
+
+            out[:sections][type_key] ||= {
+              type: section_group[0][0],
+              from: section_group[0][1].to_i,
+              to: section_group[0][2].to_i,
+            }
+
+            result = get_predicates_by_link(resource_uri, resource_data[:link], type)
+
+            resource_data[:properties] = result[:properties]
+            resource_data[:strict_properties] = result[:strict_properties]
+
+            resource_data
+          }.compact || []
+        }
+
+        out[:time] = time.round(2)
+
+        puts "- properties found in #{out[:time]}" if @console_output
+
+        out
+      end
+
+
+      ###
+      # The method generate file path for given resource URI.
+      # Also ensure to exist sub directory by resource entity type.
+      #
+      # @param [String] resource_uri
+      # @param [String] type
+      #
+      # @return [String] resource_file_path
+      def get_resource_file_path(resource_uri, type)
+        type = type.split('/').last
+        resource_name = resource_uri.split('/').last
+
+        dir_path = "#{@results_dir_path}/#{type}"
+        Dir.mkdir(dir_path) unless Dir.exist?(dir_path)
+
+        "#{dir_path}/#{StringHelper.get_clear_file_path(resource_name)}.json"
+      end
+
+      ###
+      # The method helps update found predicates to stored links.
+      #
+      # @param [String] resource_uri
+      # @param [String] type
+      #
+      # @return [Array<Hash>] old_data, new_data
+      def update_nif_file_properties(resource_uri, type)
+        if block_given?
+          path = get_resource_file_path(resource_uri, type)
+          old_data = ensure_load_json(path, {}, symbolize_names: true)
+
+          new_data = old_data.dup
+
+          time = Benchmark.realtime {
+            new_data[:nif_data] = old_data[:nif_data].map { |hash|
+              actual_link = hash[:link].to_sym
+
+              result = yield actual_link
+
+              hash[:strict_properties] = result[:strict_properties] if result[:strict_properties]
+              hash[:properties] = result[:properties] if result[:properties]
+
+              hash
+            }
+          }
+
+          new_data[:process_time][:relations_find] = time.round(2)
+
+          File.write(path, JSON.pretty_generate(new_data))
+          return old_data, new_data
+        end
+      end
+
+      ###
+      # The method in yield block give founded hash for required property.
+      # This hash contains counter, score and also all identical properties.
+      # At the end update score that was get from yield block as return value.
+      #
+      # @param [String] property
+      # @param [Array<Hash>] this_knowledge_data
+      #
+      # @yield param found
+      # @yield return score
+      def prepare_property_to_knowledge(property, this_knowledge_data)
+        property = property.to_s
+
+        this_knowledge_data ||= []
+        found = this_knowledge_data.find { |data| data[:predicates].include?(property) }
+
+        if found.nil? || found.empty?
+          # add new
+
+          identical_properties = @predicates_similarity.find_identical(property)
+
+          found = {
+            counter: 0,
+            score: 0.0,
+            predicates: identical_properties || [property.to_s]
+          }
+
+          this_knowledge_data << found
+        end
+
+        new_score = yield found
+
+
+        found[:score] = new_score
+      end
+
+      ###
+      # The method delete all resources that already has created result file
+      #
+      # @param [Hash{resource=>type}] resources
+      def keep_unloaded(resources)
+        resources.delete_if { |resource, values|
+          dir_path = "#{@results_dir_path}/#{values[:type]}"
+          resource_name = resource.split('/').last
+          File.exists?("#{dir_path}/#{StringHelper.get_clear_file_path(resource_name)}.json")
+        }
+      end
+
+      ###
+      # The method keep all resources that already has created result file
+      #
+      # @param [Hash{resource=>type}] resources
+      def keep_loaded(resources)
+        resources.keep_if { |resource, values|
+          dir_path = "#{@results_dir_path}/#{values[:type]}"
+          resource_name = resource.split('/').last
+          File.exists?("#{dir_path}/#{StringHelper.get_clear_file_path(resource_name)}.json")
+        }
+      end
+
+      ###
+      # The method allow to update knowledge base by every entity class type.
+      #
+      # @param [Hash] new_data
+      def update_knowledge_base(new_data)
+        path = "#{@results_dir_path}/knowledge_base.json"
+        old_data = ensure_load_json(path, {}, symbolize_names: true)
+        File.write(path, JSON.pretty_generate(old_data.merge(new_data)))
+      end
+
+      ###
+      # The method allow to update global statistic by every entity class type.
+      #
+      # @param [Hash] new_data
+      def update_global_statistic(new_data)
+        path = "#{@results_dir_path}/global_statistic.json"
+        old_data = ensure_load_json(path, {}, symbolize_names: true)
+        File.write(path, JSON.pretty_generate(old_data.merge(new_data)))
+      end
+
+      ###
+      # The method returns global properties for given entity class type.
+      #
+      # @param [String] type Type from http://mappings.dbpedia.org/server/ontology/classes/
+      #
+      # @return [Hash] global_statistic_by_type
+      def get_global_statistic_by_type(type)
+        type = type.to_s.to_sym
+        path = "#{@results_dir_path}/global_statistic.json"
+        data = ensure_load_json(path, {}, symbolize_names: true)
+        data[type]
+      end
+
+      ###
+      # The method helps to load json file.
+      #
+      # @param [String] file_path
+      # @param [String] def_val If no exist file add values as default.
+      # @param [Hash] json_params JSON.parse params
+      #
+      # @return [Object] json
+      def ensure_load_json(file_path, def_val, json_params = {})
+        if File.exists?(file_path)
+          file_data = File.read(file_path).force_encoding('utf-8')
+          if file_data.size >= 2 # '[]'
+            JSON.parse(file_data, json_params)
+          else
+            def_val
+          end
+        else
+          def_val
+        end
+      end
+
+
+    end
+  end
+
+end