db_clustering 0.1.0 → 0.1.1

data/lib/db_clustering.rb

@@ -0,0 +1,34 @@
+#
+# Algorithms
+#
+require 'algorithms/density_based/dbscan'
+
+
+#
+# Datasource Adapters
+#
+require 'datasource_adapters/active_record'
+require 'datasource_adapters/in_memory'
+
+
+#
+# Distance Metrics
+#
+require 'distance_metrics/average_difference'
+require 'distance_metrics/cosine_similarity'
+require 'distance_metrics/euclidean_distance'
+require 'distance_metrics/pearson_correlation'
+
+
+#
+# Generators
+#
+require 'generators/datasource/active_record'
+
+
+#
+# Models
+#
+require 'models/cluster'
+require 'models/point'
+require 'models/vector'

data/lib/distance_metrics/average_difference.rb

@@ -0,0 +1,28 @@
+module DbClustering
+  module DistanceMetrics
+    class AverageDifference
+      include Math
+
+      def initialize(min_dimensions: 1)
+        @min_dimensions = min_dimensions
+      end
+
+      def distance(vector1, vector2)
+        vector1_array = vector1.array_for_comparison(vector2)
+        vector2_array = vector2.array_for_comparison(vector1)
+
+        if vector1_array.count != vector2_array.count
+          raise "Vectors with different sizes cannot be compared"
+        end
+
+        if vector1_array.count < @min_dimensions
+          return Float::INFINITY
+        end
+
+        sum = vector1_array.map.with_index{ |x, i| (x - vector2_array[i]).abs }.reduce(&:+)
+        sum / vector1_array.count.to_f
+      end
+
+    end
+  end
+end

data/lib/distance_metrics/cosine_similarity.rb

@@ -0,0 +1,43 @@
+module DbClustering
+  module DistanceMetrics
+    class CosineSimilarity
+      include Math
+
+      def initialize(min_dimensions: 1)
+        @min_dimensions = min_dimensions
+      end
+
+      def distance(vector1, vector2)
+        1.0 - correlation(vector1, vector2)
+      end
+
+      def correlation(vector1, vector2)
+        vector1_array = vector1.array_for_comparison(vector2)
+        vector2_array = vector2.array_for_comparison(vector1)
+
+        if vector1_array.count != vector2_array.count
+          raise "Vectors with different sizes cannot be compared"
+        end
+
+        if vector1_array.count < @min_dimensions
+          return Float::INFINITY
+        end
+
+        # see here for calculation formula: https://en.wikipedia.org/wiki/Cosine_similarity
+        numerator = 0
+        vector1_array.count.times do |i|
+          numerator += vector1_array[i] * vector2_array[i]
+        end
+
+
+
+        left_sqrt = sqrt(vector1_array.reduce(0) { |sum, v1i| sum + v1i ** 2 })
+        right_sqrt = sqrt(vector2_array.reduce(0) { |sum, v2i| sum + v2i ** 2 })
+        denominator = left_sqrt * right_sqrt
+
+        numerator.to_f / denominator
+      end
+
+    end
+  end
+end

data/lib/distance_metrics/euclidean_distance.rb

@@ -0,0 +1,32 @@
+module DbClustering
+  module DistanceMetrics
+    class EuclideanDistance
+      include Math
+
+      def initialize(min_dimensions: 1)
+        @min_dimensions = min_dimensions
+      end
+
+      def distance(vector1, vector2)
+        vector1_array = vector1.array_for_comparison(vector2)
+        vector2_array = vector2.array_for_comparison(vector1)
+
+        if vector1_array.count != vector2_array.count
+          raise "Vectors with different sizes cannot be compared"
+        end
+
+        if vector1_array.count < @min_dimensions
+          return Float::INFINITY
+        end
+
+        # see here for calculation formula: http://en.wikipedia.org/wiki/Euclidean_distance
+        sum = 0
+        vector1_array.count.times do |i|
+          sum += (vector1_array[i] - vector2_array[i]) ** 2
+        end
+        sqrt sum
+      end
+
+    end
+  end
+end

data/lib/distance_metrics/pearson_correlation.rb

@@ -0,0 +1,44 @@
+module DbClustering
+  module DistanceMetrics
+    class PearsonCorrelation
+      include Math
+
+      def initialize(min_dimensions: 1)
+        @min_dimensions = min_dimensions
+      end
+
+      def distance(vector1, vector2)
+        1.0 - correlation(vector1, vector2)
+      end
+
+      def correlation(vector1, vector2)
+        vector1_array = vector1.array_for_comparison(vector2)
+        vector2_array = vector2.array_for_comparison(vector1)
+
+        if vector1_array.count != vector2_array.count
+          raise "Vectors with different sizes cannot be compared"
+        end
+
+        if vector1_array.count < @min_dimensions
+          return Float::INFINITY
+        end
+
+        # see here for calculation formula: http://en.wikipedia.org/wiki/Pearson_product-moment_correlation_coefficient
+        v1_mean = vector1_array.reduce(:+) / vector1_array.count.to_f
+        v2_mean = vector2_array.reduce(:+) / vector2_array.count.to_f
+
+        numerator = 0
+        vector1_array.count.times do |i|
+          numerator += (vector1_array[i] - v1_mean) * (vector2_array[i] - v2_mean)
+        end
+
+        left_sqrt = sqrt(vector1_array.reduce(0) { |sum, v1i| sum + (v1i - v1_mean) ** 2 })
+        right_sqrt = sqrt(vector2_array.reduce(0) { |sum, v2i| sum + (v2i - v2_mean) ** 2 })
+        denominator = left_sqrt * right_sqrt
+
+        numerator.to_f / denominator
+      end
+
+    end
+  end
+end

data/lib/models/cluster.rb

@@ -0,0 +1,18 @@
+module DbClustering
+  module Models
+    class Cluster
+
+      attr_accessor :points
+
+      def initialize
+        @points = []
+      end
+
+      def add(point)
+        @points << point
+        point.cluster = self
+      end
+
+    end
+  end
+end

data/lib/models/point.rb

@@ -0,0 +1,41 @@
+module DbClustering
+  module Models
+    class Point
+
+      attr_accessor :cluster, :is_noise, :datasource_point
+
+      def initialize(datasource_point)
+        @is_noise = false
+        @cluster = nil
+        @datasource_point = datasource_point
+      end
+
+      def vector
+        vector_object = @datasource_point.clustering_vector
+
+        if vector_object.is_a?(Hash) || vector_object.is_a?(Array)
+          DbClustering::Models::Vector.new(object: vector_object)
+        else
+          raise "clustering_vector method needs to result to a Hash or an Array object"
+        end
+      end
+
+      def visited?
+        self.is_noise || !self.cluster.nil?
+      end
+
+      def is_edge_point?
+        self.is_noise && !self.cluster.nil?
+      end
+
+      def is_core_point?
+        !self.is_noise && !self.cluster.nil?
+      end
+
+      def is_noise_point?
+        self.is_noise && self.cluster.nil?
+      end
+
+    end
+  end
+end

data/lib/models/vector.rb

@@ -0,0 +1,30 @@
+module DbClustering
+  module Models
+    class Vector
+
+      attr_reader :hash
+
+      def initialize(object:)
+        if object.is_a?(Hash)
+          @hash = object
+        else
+          @array = object
+        end
+      end
+
+      def array_for_comparison(other_vector)
+        if @hash
+          if other_vector
+            shared_keys = @hash.keys & other_vector.hash.keys
+            @hash.select{ |k,v| shared_keys.include?(k) }.sort.map{ |arr| arr.last }
+          else
+            @hash.values
+          end
+        else
+          @array
+        end
+      end
+
+    end
+  end
+end

data/spec/algorithms/density_based/dbscan_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+require 'support/test_model'
+
+describe DbClustering::Algorithms::Dbscan do
+  describe "initialization" do
+    before(:each) do
+      @dataset = DatasetHelper.normal_distribution
+
+      @in_memory_datasource = DbClustering::DatasourceAdapters::InMemory.new(array: @dataset)
+      @average_difference_metric = DbClustering::DistanceMetrics::AverageDifference.new
+
+      @dbscan = DbClustering::Algorithms::Dbscan.new(datasource: @in_memory_datasource, distance_metric: @average_difference_metric)
+    end
+
+    it "should initialize successfully" do
+      expect(@dbscan).to be_a(DbClustering::Algorithms::Dbscan)
+    end
+  end
+
+  describe "#cluster" do
+    before(:each) do
+      @clusters_count = 10
+      @dataset = DatasetHelper.normal_distribution(vector_size: 10, clusters: @clusters_count, datapoints: 100)
+
+      @in_memory_datasource = DbClustering::DatasourceAdapters::InMemory.new(array: @dataset)
+      @average_difference_metric = DbClustering::DistanceMetrics::AverageDifference.new
+
+      @dbscan = DbClustering::Algorithms::Dbscan.new(datasource: @in_memory_datasource, distance_metric: @average_difference_metric)
+      @dbscan.cluster(max_distance: 10, min_neighbors: 5)
+    end
+
+    it "changes all points to clustered or noise – not both" do
+      @dbscan.datasource.iterate_all_points do |point|
+        expect(point.is_core_point? || point.is_edge_point? || point.is_noise).to eq(true)
+        expect(point.is_core_point? && point.is_edge_point?).to eq(false)
+        expect(point.is_core_point? && point.is_noise_point?).to eq(false)
+        expect(point.is_edge_point? && point.is_noise_point?).to eq(false)
+      end
+    end
+
+    it "visits all points" do
+      @in_memory_datasource.iterate_all_points do |point|
+        expect(point.visited?).to eq(true)
+      end
+    end
+
+    it "finds all clusters" do
+      expect(@dbscan.clusters.count).to eq(@clusters_count)
+    end
+  end
+
+  describe "#expand_cluster" do
+    pending "should expand cluster with one point and missing points in cluster"
+    pending "should expand cluster with several points and missing points in cluster"
+    pending "should expand cluster with several points and without missing points in cluster"
+  end
+end

data/spec/datasource_adapters/in_memory_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+require 'simple-random'
+
+describe DbClustering::DatasourceAdapters::InMemory, type: :model do
+
+  # describe "#initialize" do
+  #   it "initializes with an array" do
+  #     expect(DbClustering::DatasourceAdapters::InMemory.new(array: [])).to be_a(DbClustering::DatasourceAdapters::InMemory)
+  #   end
+  # end
+  #
+  # describe "#iterate_all_points" do
+  #   before(:each) do
+  #     @in_memory = DbClustering::DatasourceAdapters::InMemory.new(array: (1..100).to_a)
+  #   end
+  #
+  #   it "iterates through all points" do
+  #     x = 0
+  #     @in_memory.iterate_all_points { |p| x += 1 }
+  #     expect(x).to eq(100)
+  #   end
+  # end
+
+  describe "#neighbors" do
+    before(:each) do
+      @dataset = DatasetHelper.normal_distribution(vector_size: 16, clusters: 8, datapoints: 80)
+
+      @in_memory = DbClustering::DatasourceAdapters::InMemory.new(array: @dataset)
+      @first_point = DbClustering::Models::Point.new(@dataset.first)
+    end
+
+    context "average difference" do
+      before(:each) do
+        @average_difference = DbClustering::DistanceMetrics::AverageDifference.new
+      end
+
+      it "finds all neighbors" do
+        neighbors = @in_memory.neighbors(point: @first_point, distance_metric: @average_difference, max_distance: 10)
+        expect(neighbors.count).to eq(10)
+        expect(neighbors.first).to be_a(DbClustering::Models::Point)
+      end
+    end
+
+    context "cosine similarity" do
+      before(:each) do
+        @cosine_similarity = DbClustering::DistanceMetrics::CosineSimilarity.new
+      end
+
+      it "finds all neighbors" do
+        neighbors = @in_memory.neighbors(point: @first_point, distance_metric: @cosine_similarity, max_distance: 0.25)
+        expect(neighbors.count).to eq(40)
+        expect(neighbors.first).to be_a(DbClustering::Models::Point)
+      end
+    end
+
+    context "euclidean distance" do
+      before(:each) do
+        @euclidean_distance = DbClustering::DistanceMetrics::EuclideanDistance.new
+      end
+
+      it "finds all neighbors" do
+        neighbors = @in_memory.neighbors(point: @first_point, distance_metric: @euclidean_distance, max_distance: 50)
+        expect(neighbors.count).to eq(10)
+        expect(neighbors.first).to be_a(DbClustering::Models::Point)
+      end
+    end
+
+    context "pearson correlation" do
+      before(:each) do
+        @pearson_correlation = DbClustering::DistanceMetrics::PearsonCorrelation.new
+      end
+
+      it "finds all neighbors" do
+        neighbors = @in_memory.neighbors(point: @first_point, distance_metric: @pearson_correlation, max_distance: 0.705)
+        expect(neighbors.count).to eq(10)
+        expect(neighbors.first).to be_a(DbClustering::Models::Point)
+      end
+    end
+
+  end
+
+end

data/spec/distance_metrics/average_difference_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+describe DbClustering::DistanceMetrics::AverageDifference, type: :model do
+
+  before(:each) do
+    @average_difference = DbClustering::DistanceMetrics::AverageDifference.new
+  end
+
+  describe "#distance" do
+
+    context "using array object" do
+
+      it "works with 6 dimensional examples" do
+        a1 = [-100, -50, 0, 10, 20, 30]
+        a2 = [-100, -50, 0, 20, 30, 40]
+
+        expect_distance(a1, a2, 5.0)
+
+        a1[0] = 100
+        expect_distance(a1, a2, 38.333333333333336)
+
+        a1[1] = 50
+        expect_distance(a1, a2, 55)
+
+        a1[3] = 20
+        expect_distance(a1, a2, 53.333333333333333)
+
+        a1[4] = 30
+        expect_distance(a1, a2, 51.666666666666664)
+
+        a1[5] = 40
+        expect_distance(a1, a2, 50)
+      end
+
+    end
+  end
+
+  def expect_distance(object1, object2, distance)
+    vector1 = DbClustering::Models::Vector.new(object: object1)
+    vector2 = DbClustering::Models::Vector.new(object: object2)
+    expect(@average_difference.distance(vector1, vector2)).to be_within(0.001).of(distance)
+  end
+
+end