dnks_clustering_indexes 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2184949a035f273d5fc42f6bc36896a2845982a9d2d3734a8d649455c1d29304
+  data.tar.gz: 2c9926ecedaf81280c9f9cf2c6eca1ec083a791ea1012820655a3938fd620aab
+SHA512:
+  metadata.gz: 8be08145c482785e3dca7a53a58156dc18736ca65834474f35326c8c5d871aaeee0e048f063584efeeb0e5c53b0223a1d612dac6a281cbb3492a4c69e85bd89a
+  data.tar.gz: c9e8f30d24c5f867a5bb8e04f0aa97f3428299257b503076e79d3d453339f30eba2792445ffe5779dcd3eba425018ce5fd4d4207b6bd2a6b0ab42037a26a71fe

data/lib/calculations_utility_methods.rb

@@ -0,0 +1,109 @@
+require "bigdecimal"
+
+module Calculations
+  # numbers used for big calculations
+  def self.number(n)
+    BigDecimal(n, 10)
+  end
+
+  def self.mode_of_array(arr)
+    mode = nil
+    frequency_table = {}
+    max_frequency = 0
+
+    arr.each do |elem|
+      frequency_table[elem] ||= 0
+      frequency_table[elem] += 1
+      max_frequency, mode = frequency_table[elem], elem if frequency_table[elem] > max_frequency
+    end
+    mode
+  end
+
+  # return array without element with index from params
+  def self.array_except(array, index)
+    arr_dup = array.dup
+    arr_dup.delete_at(index)
+    arr_dup
+  end
+
+  module Distance
+    # distance between two "points"
+    def self.euclidean(p1, p2)
+      sum = Calculations.number(0.0)
+      p1.each_index { |i| sum += (Calculations.number(p2[i]) - Calculations.number(p1[i]))**2 }
+      Math.sqrt(sum)
+    end
+  end
+
+  module Cluster
+    def self.center(multi_array)
+      ret = []
+      (0...multi_array.first.length).each { |j|
+        sum = Calculations.number(0.0)
+        (0...multi_array.length).each { |i|
+          sum += Calculations.number(multi_array[i][j])
+        }
+        ret.push(Calculations.number(sum / multi_array.length))
+      }
+      ret
+    end
+
+    def self.distances_to_point(multi_array, point)
+      multi_array.map { |x| Calculations::Distance.euclidean(x, point) }
+    end
+
+    # find nearest centroid in array "centroids" to point "point",
+    # except centroid with index "index_of_cluster" in "centroids" array
+    # return nearest centroids index
+    def self.nearest_to_point(centroids, index_of_centroid_to_skip, point)
+      min = nil
+      (0...centroids.length).each do |i|
+        next if i == index_of_centroid_to_skip
+        distance = Calculations::Distance.euclidean(centroids[i], point)
+        min = [distance, i] if min.nil? || distance <= min[0]
+      end
+      min[1]
+    end
+  end
+
+  class DistanceMatrix
+    def initialize(data_array)
+      @distances_matrix = euclidean_distances_matrix(data_array)
+    end
+
+    def show
+      @distances_matrix.each_with_index do |row, i|
+        puts "X#{i}: #{row}"
+      end
+    end
+
+    def data
+      @distances_matrix.sum([])
+    end
+
+    def euclidean_distances_matrix(arr)
+      (0...arr.length - 1).to_a.map do |i|
+        ((i + 1)...arr.length).to_a.map { |j|
+          Calculations::Distance.euclidean(arr[i], arr[j])
+        }
+      end
+    end
+
+    def total_sum
+      data.sum(Calculations.number(0.0))
+    end
+
+    def max_distance
+      data.max
+    end
+
+    def get(point_1_index, point_2_index)
+      return 0 if point_1_index == point_2_index
+      point_1_index, point_2_index = point_2_index, point_1_index if point_1_index > point_2_index
+      puts "i = #{point_1_index}, j = #{point_2_index}"
+      puts "i = #{point_1_index}, j = #{point_2_index - point_1_index - 1}"
+      @distances_matrix[point_1_index][point_2_index - point_1_index - 1]
+    end
+
+  end
+end

data/lib/clustered_data_set.rb

@@ -0,0 +1,61 @@
+require_relative "calculations_utility_methods"
+require_relative "clustering_indexes"
+
+class ClusteredDataSet
+
+  attr_reader :clusters, :centroids, :num_objects, :num_features, :k
+
+  def data
+    clusters.sum([])
+  end
+
+  def dataset_barycenter
+    Calculations::Cluster.center(data)
+  end
+
+  def initialize(param)
+    build_by_kmeans_clusters(param) if param[0].instance_of?(KMeansClusterer::Cluster)
+    build_by_multi_array(param) if param[0].instance_of?(Array)
+  end
+
+  # build by already clustered dataset in multi array representation
+  def build_by_multi_array(marr)
+    @data_rows = []
+    @clusters = marr
+    @num_objects = data.length
+    @num_features = @clusters[0][0].length
+    @k = clusters.length
+    @centroids = @clusters.map { |cluster| Calculations::Cluster.center(cluster) }
+    puts "Built by multi array!"
+    self
+  end
+
+  # build by already clustered KmeansClusterer::Cluster
+  def build_by_kmeans_clusters(clusters)
+    @data_rows = []
+    @clusters = []
+    clusters.each { |x| @clusters.insert(x.id, kmeans_points_to_array(x.points)) }
+    @num_objects = data.length
+    @num_features = @clusters[0][0].length
+    @k = clusters.length
+    @centroids = @clusters.map { |cluster| Calculations::Cluster.center(cluster) }
+    puts "Built by kmeans clusterer! For k = #{@k}"
+    self
+  end
+
+  def kmeans_points_to_array(points)
+    points.map { |point| point.data.to_a }
+  end
+
+  def show
+    @clusters.each_with_index do |cl, i|
+      puts "Cluster #{i} :"
+      puts cl.to_s
+    end
+  end
+
+  def calculate_index_by_name(index_name)
+    index_calculator = ClusteringIndexes.new(self)
+    index_calculator.calculate_index_by_name(index_name)
+  end
+end

data/lib/clustering_indexes.rb

@@ -0,0 +1,111 @@
+require "bigdecimal"
+class ClusteringIndexes
+  def initialize(cds)
+    @cds = cds
+  end
+
+  def calculate_index_by_name(index_name)
+    case index_name
+    when Constants::Indexes::SILHOUETTE
+      silhouette_index
+    when Constants::Indexes::CALINSKI_HARABASZ
+      calinski_harabasz_index
+    when Constants::Indexes::C
+      c_index
+    when Constants::Indexes::DUNN
+      dunn_index
+    when Constants::Indexes::DAVIES_BOULDIN
+      davies_bouldin_index
+    when Constants::Indexes::PBM
+      pbm_index
+    end
+  end
+
+  private
+
+  def silhouette_index
+    return 1.0 if @cds.k < 2
+    silhouette_scores_sum = 0
+    silhouette_scores_count = 0
+    (0...@cds.k).each { |i|
+      silhouette_scores_sum += @cds.clusters[i].map { |point|
+        nearest_cluster_index = Calculations::Cluster.nearest_to_point(@cds.centroids, i, point)
+        a = Calculations::Cluster.distances_to_point(@cds.clusters[i], point).sum(0.0)
+        b = Calculations::Cluster.distances_to_point(@cds.clusters[nearest_cluster_index], point).sum(0.0)
+        a /= (@cds.clusters[i].length - 1)
+        b /= @cds.clusters[nearest_cluster_index].length
+        silhouette_scores_count += 1
+        a.nan? || b.nan? ? 0 : (b - a) / [a, b].max
+      }.sum(0.0)
+    }
+    silhouette_scores_sum / silhouette_scores_count
+  end
+
+  def davies_bouldin_index
+    r = (0...@cds.k).to_a.map { |i|
+      Calculations::Cluster.distances_to_point(@cds.clusters[i], @cds.centroids[i]).sum(0.0) / @cds.clusters[i].length
+    }
+    rc = (0...@cds.k).to_a.map { |i| [r[i], @cds.centroids[i]] }
+    (0...@cds.k).to_a.map do |i|
+      rc_dup = rc.dup
+      rc_dup.delete_at(i)
+      ret = rc_dup.map do |x|
+        sum = (0.0 + rc[i][0] + x[0])
+        div = Calculations::Distance.euclidean(rc[i][1], x[1])
+        sum / div
+      end.max
+      ret
+    end.sum(0.0) / @cds.k
+  end
+
+  def dunn_index
+    inter_min = (0...@cds.k).to_a.map do |i|
+      (0...@cds.clusters[i].length).to_a.map do |j|
+        Calculations::Cluster.distances_to_point(Calculations.array_except(@cds.clusters, i).sum([]), @cds.clusters[i][j]).min
+      end.min
+    end.min
+    intra_max = (0...@cds.k).to_a.map do |i|
+      (0...@cds.clusters[i].length).to_a.map do |j|
+        Calculations::Cluster.distances_to_point(@cds.clusters[i], @cds.clusters[i][j]).max
+      end.max
+    end.max
+    inter_min / intra_max
+  end
+
+  def calinski_harabasz_index
+    top_part = (0...@cds.k).to_a.map { |i|
+      @cds.clusters[i].length * (Calculations::Distance.euclidean(@cds.centroids[i], @cds.dataset_barycenter)**2)
+    }.sum(0.0) / (@cds.k - 1)
+
+    bot_part = (0...@cds.k).to_a.map { |i|
+      (0...@cds.clusters[i].length).to_a.map { |j|
+        Calculations::Distance.euclidean(@cds.clusters[i][j], @cds.centroids[i])**2
+      }.sum(0.0)
+    }.sum(0.0) / (@cds.num_objects - @cds.k)
+    top_part / bot_part
+  end
+
+  def c_index
+    # calculate d
+    d = (0...@cds.k).to_a.map do |i|
+      m = Calculations::DistanceMatrix.new(@cds.clusters[i])
+      m.total_sum
+    end.sum(0.0)
+    r = (0...@cds.k).to_a.map { |i| @cds.clusters[i].length * (@cds.clusters[i].length - 1) }.sum(0.0) / 2
+    matrix = Calculations::DistanceMatrix.new(@cds.data)
+    sorted_matrix = matrix.data.sort
+    d_min = sorted_matrix.first(r).sum(0.0)
+    d_max = sorted_matrix.last(r).sum(0.0)
+    (d - d_min) / (d_max - d_min)
+  end
+
+  def pbm_index
+    d_b = Calculations::DistanceMatrix.new(@cds.centroids).max_distance
+    num_arr = (0...@cds.k).to_a
+    sum_dist_to_centers =
+      num_arr.map { |i| Calculations::Cluster.distances_to_point(@cds.clusters[i], @cds.centroids[i]).sum(Calculations.number(0.0)) }
+    e_w = sum_dist_to_centers.sum(Calculations.number(0.0))
+    e_t = Calculations::Cluster.distances_to_point(@cds.data, @cds.dataset_barycenter).sum(0.0)
+    (d_b * (e_t / e_w) / @cds.k)**2
+  end
+end

data/lib/constants.rb

@@ -0,0 +1,19 @@
+module Constants
+  module Indexes
+    SILHOUETTE = "silhouette"
+    C = "c"
+    DAVIES_BOULDIN = "davies_bouldin"
+    DUNN = "dunn"
+    CALINSKI_HARABASZ = "calinski_harabasz"
+    PBM = "pbm"
+  end
+
+  CALCULATION_RULES = {
+    "silhouette" => "max",
+    "c" => "min",
+    "davies_bouldin" => "min",
+    "dunn" => "max",
+    "calinski_harabasz" => "max",
+    "pbm" => "max"
+  }
+end

data/lib/index_calculator.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require "kmeans-clusterer"
+require_relative "constants"
+require_relative "clustered_data_set"
+class IndexCalculator
+  attr_reader :data, :labels, :max_k, :k_matrix, :clustered_data_sets
+
+  def initialize(data, labels, max_k = 2)
+    @data = data
+    @labels = labels
+    @max_k = max_k
+    @k_matrix = []
+    @k_matrix_header = {}
+    @clustered_data_sets = [0, 0]
+    @optimal_k_indexes = {}
+  end
+
+  def clear_clustered_datasets
+    @clustered_data_sets = [0, 0] if @clustered_data_sets.length > 2
+  end
+
+  def load_data_from_file(filename)
+    if File.exist?(filename)
+      @data = []
+      clear_clustered_datasets
+      read_from_file_to_data(filename)
+      puts "Data was successfully loaded from file!"
+    else
+      puts "Build failed!\nFile with filename '#{filename}' doesnt exist!"
+      puts "Current working directory is '#{Dir.pwd}'"
+    end
+  end
+
+  def build_by_custom_clusters(multi_array)
+    clear_clustered_datasets
+    (2..@max_k).each { |k|
+      @clustered_data_sets.insert(k, ClusteredDataSet.new(multi_array))
+    }
+  end
+
+  def build_by_kmeans_clustering
+    clear_clustered_datasets
+    (2..@max_k).each { |k|
+      kmeans = KMeansClusterer.run k, @data, labels: @labels, runs: 5
+      @clustered_data_sets.insert(k, ClusteredDataSet.new((kmeans.clusters)))
+    }
+  end
+
+  def get_indexes_by_name(str)
+    @k_matrix[@k_matrix_header[str]] if @k_matrix_header.include?(str)
+  end
+
+  def calculate_index_k(index_name)
+    # nullify if index was calculated before, else add it to rhe and of array
+    if @k_matrix_header.has_key?(index_name)
+      @k_matrix[@k_matrix_header[index_name]] = [0, 0]
+    else
+      @k_matrix_header[index_name] = @k_matrix_header.size
+      @k_matrix.push([0, 0])
+    end
+    (2..@max_k).each { |k|
+      @k_matrix[@k_matrix_header[index_name]].push(@clustered_data_sets[k].calculate_index_by_name(index_name))
+    }
+  end
+
+  def calculate_all_indexes
+    Constants::Indexes.constants.each { |ce| calculate_index_k(Constants::Indexes.const_get(ce)) }
+  end
+
+  def optimal_k_indexes(index_name)
+    calculate_index_k(index_name) if @k_matrix.nil? || !@k_matrix_header.include?(index_name)
+    @optimal_k_indexes[index_name] ||= send("#{Constants::CALCULATION_RULES[index_name]}_k_index", index_name)
+  end
+
+  def find_all_optimal_k_indexes
+    Constants::Indexes.constants.each { |ce| optimal_k_indexes(Constants::Indexes.const_get(ce)) }
+  end
+
+  def optimal_k_index
+    calculate_all_indexes
+    find_all_optimal_k_indexes
+    all_k_values = @optimal_k_indexes.sum([])
+    Calculations.mode_of_array all_k_values
+  end
+
+  def show_optimal_k_indexes_table
+    best_k = optimal_k_index
+    puts "Optimal k(number of cluster) by different indexes"
+    @optimal_k_indexes.each { |k, v| puts "Best k for #{k} index: #{v}" }
+    s_for_plural = best_k.one? ? "" : "S"
+    puts "THE BEST VALUE#{s_for_plural} OF CLUSTER NUMBER#{s_for_plural} #{best_k.one? ? "IS" : "ARE"}: #{best_k.one? ? best_k[0] : best_k}"
+    str = ""
+    @optimal_k_indexes.each { |k, v| str += "#{k}, " if v == best_k }
+    puts "Indexes which calculated best k: #{str[0..-3]}"
+  end
+
+  def min_k_index(index_name)
+    header_index = @k_matrix_header[index_name]
+    k_indexes = @k_matrix[header_index]
+    best_k = 2
+    min = k_indexes[2]
+    (2...k_indexes.length).each { |k|
+      if k_indexes[k] < min
+        best_k = k
+        min = k_indexes[k]
+      end
+    }
+    best_value = k_indexes[best_k]
+    best_ks = []
+    (2...k_indexes.length).each { |k| best_ks.push k if k_indexes[k] == best_value }
+    best_ks
+  end
+
+  def max_k_index(index_name)
+    header_index = @k_matrix_header[index_name]
+    k_indexes = @k_matrix[header_index]
+    best_k = 2
+    max = k_indexes[2]
+    (2...k_indexes.length).each { |k|
+      if k_indexes[k] > max
+        best_k = k
+        max = k_indexes[k]
+      end
+    }
+    best_value = k_indexes[best_k]
+    best_ks = []
+    (2...k_indexes.length).each { |k| best_ks.push k if k_indexes[k] == best_value }
+    best_ks
+  end
+
+  def show_data_table
+    puts "Dataset :"
+    puts @data
+  end
+
+  def show_index_table(index_name)
+    # make meta
+    calculate_index_k(index_name) if @k_matrix.nil? || !@k_matrix_header.include?(index_name)
+    puts "#{index_name} index scores:"
+    puts_index_table index_name
+  end
+
+  def puts_index_table(index_name)
+    (2...@k_matrix[@k_matrix_header[index_name]].length).each { |k|
+      puts "k = #{k}\t score = #{@k_matrix[@k_matrix_header[index_name]][k]}"
+    }
+  end
+
+  private
+
+  def read_from_file_to_data(filename)
+    File.foreach(filename) { |line| @data.push(line.chomp.split.map(&:to_f)) }
+  end
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: dnks_clustering_indexes
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Danil Kosenko
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2022-06-13 00:00:00.000000000 Z
+dependencies: []
+description: Gem to calculate clustering indexes
+email: danilkos2013@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/calculations_utility_methods.rb
+- lib/clustered_data_set.rb
+- lib/clustering_indexes.rb
+- lib/constants.rb
+- lib/index_calculator.rb
+homepage: https://rubygems.org/gems/dnks_clustering_indexes
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key: 
+specification_version: 4
+summary: Clustering indexes
+test_files: []