same_same 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in samesame.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,44 @@
+PATH
+  remote: .
+  specs:
+    same_same (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.0.9)
+    colored (1.2)
+    columnize (0.3.6)
+    debugger (1.6.0)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.2.1)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.2.2)
+    diff-lcs (1.2.4)
+    method_source (0.8.1)
+    pry (0.9.12.2)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    pry-debugger (0.2.2)
+      debugger (~> 1.3)
+      pry (~> 0.9.10)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+    slop (3.4.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  colored
+  pry-debugger
+  rspec
+  same_same!

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Julian Russell
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+samesame
+========
+
+Ruby version of clustering algorithms from "Algorithms of the Intelligent Web"
+
+## Status
+
+Pretty much direct port of the sameple code from the book (Java).
+
+### Todo
+  * **Expand specs**. The basics have specs, but some of the computation specs are just testing the thing doesn't blow up, NOT that the calculation is right. Lots of higher level code doesn't have any specs
+  * **Refactor**. Some of the classes and methods are filthy. Things like `Cluster` are just thin wrappers that delegate to arrays.
+  * **Push in more data and see what happens**
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'same_same'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install samesame
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/examples/dbscan_digg.rb

@@ -0,0 +1,25 @@
+require 'same_same'
+require 'same_same/dendrogram_printer'
+require 'csv'
+
+digg_rows = CSV.read("../spec/fixtures/digg_stories.csv", headers: true)
+digg_data = digg_rows.map {|row|
+  SameSame::DataPoint.new( row["title"], 
+    %w(category topic description).map {|key|
+      row[key]
+    }.join(" ").downcase.split(/\s+/)
+  )
+}
+
+distance       = SameSame::CosineDistance.new
+vector_builder = SameSame::DbscanTermFrequencyVectors.new
+algo = SameSame::DbscanAlgorithm.new(
+  points: digg_data,
+  eps: 0.7,
+  min_points: 2,
+  vector_calculator: vector_builder,
+  distance: distance)
+
+clusters = algo.cluster
+
+SameSame::DendrogramPrinter.new.print_clusters( clusters )

data/examples/dbscan_lines.rb

@@ -0,0 +1,35 @@
+require 'same_same'
+require 'same_same/dendrogram_printer'
+require 'csv'
+
+csv = CSV.read("../spec/fixtures/lines.csv", headers: true)
+
+# , row['price']
+groups = csv.group_by {|row| [row['categories']].join("-")}
+
+fragments = groups.map {|group_key, group|
+  [group_key, group.map {|row|
+    SameSame::DataPoint.new( [row["id"], row["name"]].map {|t| t.gsub(/\s+/, ' ')}.join(": "), 
+      %w(name price).map {|key|
+        row[key]
+      }.join(" ").downcase.split(/\s+/)
+    )
+  }]
+}
+
+distance       = SameSame::CosineDistance.new
+vector_builder = SameSame::DbscanTermFrequencyVectors.new
+
+fragments.each do |key, group|
+  if group.size > 1
+    algo = SameSame::DbscanAlgorithm.new(
+      points: group,
+      eps: 0.3,
+      min_points: 2,
+      vector_calculator: vector_builder,
+      distance: distance)
+
+    clusters = algo.cluster
+    SameSame::DendrogramPrinter.new.print_clusters( clusters.select {|c| c.name != "Noise"} )
+  end
+end

data/examples/rock_digg.rb

@@ -0,0 +1,20 @@
+require 'same_same'
+require 'same_same/dendrogram_printer'
+require 'csv'
+
+digg_rows = CSV.read("../spec/fixtures/digg_stories.csv", headers: true)
+digg_data = digg_rows.map {|row|
+  SameSame::DataPoint.new( row["title"], 
+    %w(category topic description).map {|key|
+      row[key]
+    }.join(" ").downcase.split(/\s+/)
+  )
+}
+
+
+k = 2
+th = 0.2
+algo = SameSame::RockAlgorithm.new(datapoints: digg_data, k: k, th: th)
+dnd = algo.cluster
+
+SameSame::DendrogramPrinter.new.print_last(dnd)

data/examples/rock_lines.rb

@@ -0,0 +1,31 @@
+require 'same_same'
+require 'same_same/dendrogram_printer'
+require 'csv'
+
+csv = CSV.read("../spec/fixtures/lines.csv", headers: true)
+
+groups = csv.group_by {|row| [row['categories'], row['price']].join("-")}
+
+fragments = groups.map {|group_key, group|
+  [group_key, group.map {|row|
+    SameSame::DataPoint.new( [row["id"], row["name"]].map {|t| t.gsub(/\s+/, ' ')}.join(": "), 
+      %w(name price categories).map {|key|
+        row[key]
+      }.join(" ").downcase.split(/\s+/) + [row["name"].downcase.gsub(/\s+/, ' ')]
+    )
+  }]
+}
+
+k = 4
+th = 0.4
+fragments.each do |key, group|
+  if group.size > 1
+    algo = SameSame::RockAlgorithm.new(datapoints: group, k: k, th: th)
+    dnd = algo.cluster
+    if dnd.non_singelton_leaves?
+      SameSame::DendrogramPrinter.new.print_last(dnd)
+    end
+  end
+end
+
+

data/lib/same_same.rb

@@ -0,0 +1,15 @@
+require 'same_same/cluster'
+require 'same_same/cosine_distance'
+require 'same_same/data_point'
+require 'same_same/dbscan_algorithm'
+require 'same_same/dbscan_numeric_vectors'
+require 'same_same/dbscan_term_frequency_vectors'
+require 'same_same/jaquard_coefficient'
+require 'same_same/rock_algorithm'
+require 'same_same/symmetrical_matrix'
+require 'same_same/term_frequency_builder'
+require 'same_same/version'
+
+module Samesame
+  # Your code goes here...
+end

data/lib/same_same/cluster.rb

@@ -0,0 +1,27 @@
+module SameSame
+  class Cluster
+    # note to self - unless I need to implement a heap
+    # more, i'm just wrapping an array and delegating...
+    attr_accessor :datapoints, :name
+
+    def initialize( dp, name = nil )
+      self.datapoints = dp
+      self.name = name
+    end
+
+    def +( other )
+      names = [name, other.name].compact
+      new_name = names.empty? ? nil : names.join("+")
+      Cluster.new( datapoints + other.datapoints, new_name )
+    end
+
+    def size
+      datapoints.size
+    end
+
+    def inject(i, &block)
+      datapoints.inject(i, &block)
+    end
+
+  end
+end

data/lib/same_same/cluster_similarity.rb

@@ -0,0 +1,10 @@
+module SameSame
+
+  class ClusterSimilarity < Struct.new( :cluster_key, :goodness )
+    include Comparable
+    def <=>(other)
+      goodness <=> other.goodness
+    end
+  end
+
+end

data/lib/same_same/cosine_distance.rb

@@ -0,0 +1,27 @@
+require 'same_same/cosine_similarity'
+
+module SameSame
+  class CosineDistance
+
+    attr_accessor :cosin
+
+    def initialize
+      self.cosin = CosineSimilarity.new
+    end
+
+    def distance(x, y)
+      sim = cosin.sim(x, y)
+
+      if sim < 0.0
+        throw new ArgumentError(
+                "Can't use this value to calculate distance." +
+                "x[]=" + x.inspect + 
+                ", y[]=" + y.inspect +
+                ", cosin.sim(x,y)=" + sim)
+      end
+
+      1.0 - sim
+    end
+  end
+
+end

data/lib/same_same/cosine_similarity.rb

@@ -0,0 +1,22 @@
+module SameSame
+
+  class CosineSimilarity
+
+    def similarity(x, y)
+      sim(*TermFrequencyBuilder.build_vectors( x.data, y.data ))
+    end
+
+    def sim(v1, v2)
+      dot_product(v1, v2) / (norm(v1) * norm(v2))
+    end
+    
+    def dot_product(v1, v2)
+      v1.zip(v2).map {|val1,val2| val1 * val2}.inject(:+)
+    end
+
+    def norm(vector)
+      Math.sqrt( vector.map {|val| val ** 2}.inject(:+) )
+    end
+  end
+
+end

data/lib/same_same/data_point.rb

@@ -0,0 +1,12 @@
+module SameSame
+  class DataPoint < Struct.new( :id, :data )
+    def empty?
+      data.empty?
+    end
+
+    def size
+      data.size
+    end
+
+  end
+end

data/lib/same_same/dbscan_algorithm.rb

@@ -0,0 +1,135 @@
+require 'same_same/dbscan_neighborhood'
+require 'same_same/dbscan_clusters'
+
+module SameSame
+
+  # Implementation of DBSCAN clustering algorithm. 
+  # 
+  # Algorithm parameters:
+  # 
+  #  * Eps - threshold value to determine point neighbors. Two points are 
+  #    neighbors if the distance between them does not exceed this threshold value. 
+  #  * MinPts - minimum number of points in any cluster.
+  #
+  # Choice of parameter values depends on the data.
+  #
+  # Point types:
+  #
+  #   * Core point - point that belongs to the core of the cluster. It has at least
+  #     MinPts neighboring points.
+  #   * Border point - is a neighbor to at least one core point but it doesn't 
+  #     have enough neighbors to be a core point.
+  #   * Noise point - is a point that doesn't belong to any cluster because it is
+  #     not close to any of the core points.
+  #
+  class DbscanAlgorithm
+    attr_accessor :points
+
+    # Sets of points. Initially all points will be assigned into 
+    # Unclassified points set.
+    attr_accessor :dbscan_clusters
+    
+    # Number of points that should exist in the neighborhood for a point
+    # to be a core point.
+    #
+    # Best value for this parameter depends on the data set.
+    attr_accessor :min_points
+    
+    attr_accessor :neighborhood
+    
+
+    # Initializes algorithm with all data that it needs.
+    # 
+    #  * points            - points to cluster
+    #  * eps               - distance threshold value
+    #  * min_points        - number of neighbors for point to be considered a
+    #                        core point.
+    #  * distance          - distance measure to use (defaults to Cosine)
+    #  * vector_calculator - calculates the vectors to use for distance comparison.
+    #                        defaults to DbscanNumericVectors which compares just
+    #                        the numeric attributes of the datapoint.
+    #                        Alternatively use DbscanTermFrequency.
+    def initialize(attrs = {})
+      self.points       = attrs.fetch(:points)
+      self.min_points   = attrs.fetch(:min_points)
+      distance          = attrs[:distance] || CosineDistance.new
+      vector_calculator = attrs[:vector_calculator] || DbscanNumericVectors.new
+      
+      self.neighborhood = DbscanNeighborhood.new( distance: distance,
+                                                  eps: attrs.fetch(:eps),
+                                                  points: points,
+                                                  vector_calculator: vector_calculator )
+
+      # all points start as unclassifed
+      self.dbscan_clusters     = DbscanClusters.new( points )
+    end
+
+    def cluster
+      cluster_id = dbscan_clusters.get_next_cluster_id
+      points.each do |p|
+        if dbscan_clusters.unclassified?(p)
+          if create_cluster(p, cluster_id)
+            cluster_id = dbscan_clusters.get_next_cluster_id
+          end
+        end
+      end
+      
+      dbscan_clusters.to_clusters
+    end
+
+
+    def create_cluster( p, cluster_id)
+      neighbors = neighborhood.neighbors_of p
+      if neighbors.size < min_points
+        # Assign point into "Noise" group. 
+        # It will have a chance to become a border point later on.
+        dbscan_clusters.assign_to_noise(p)
+        # return false to indicate that we didn't create any cluster
+        return false
+      end
+
+      # All points are reachable from the core point...
+      dbscan_clusters.assign_points(neighbors, cluster_id)
+      
+      # Remove point itself.
+      neighbors.delete(p)
+      
+      # Process the rest of the neighbors...
+      while !neighbors.empty?
+        # pick the first neighbor
+        neighbor = neighbors.first
+        
+        # process neighbor
+        neighbors_neighbors = neighborhood.neighbors_of neighbor
+                
+        if neighbors_neighbors.size < min_points
+          # do nothing. The neighbor is just a border point.
+        else      
+          # neighbor is another core point.  
+          neighbors_neighbors.each do |neighbors_neighbor|
+          
+            if dbscan_clusters.noise?(neighbors_neighbor)
+              # It's a border point. We know that it doesn't have 
+              # enough neighbors to be a core point. Just add it 
+              # to the cluster.
+              dbscan_clusters.assign_point(neighbors_neighbor, cluster_id)
+            elsif dbscan_clusters.unclassified?(neighbors_neighbor)
+              
+              # We don't know if this point has enough neighbors
+              # to be a core point... add it to the list of points
+              # to be checked.
+              neighbors.add(neighbors_neighbor)
+                
+              # And assign it to the cluster
+              dbscan_clusters.assign_point(neighbors_neighbor, cluster_id)
+            end
+          end
+        end
+        
+        neighbors.delete neighbor
+      end
+      true
+    end
+  
+  end
+end