same_same 0.0.1

data/spec/same_same/dbscan_algorithm_spec.rb

@@ -0,0 +1,72 @@
+require 'csv'
+require 'same_same'
+require 'same_same/dendrogram_printer'
+
+# WARNING: This is just really testing that they don't blow up.
+describe SameSame::RockAlgorithm do
+  let(:digg_data) {
+    digg_data = CSV.read("spec/fixtures/digg_stories.csv", headers: true)
+    i = 0
+    digg_data.map {|row|
+      i = i + 1
+      SameSame::DataPoint.new( "#{i}: #{row["title"]}", 
+        %w(title category topic description).map {|key|
+          row[key]
+        }.join(" ").downcase.split(/\s+/).select {|w| w.size > 3}
+      )
+    }
+  }
+
+  let(:line_data) {
+    csv = CSV.read("spec/fixtures/lines.csv", headers: true)
+
+    # , csv['price']
+    groups = csv.group_by {|csv| [csv['categories']].join("-")}
+
+    groups.map {|key, group|
+      [key, group.map {|row|
+        SameSame::DataPoint.new( [row["id"], row["name"]].map {|t| t.gsub(/\s+/, ' ')}.join(": "), 
+          %w(name price).map {|key|
+            row[key]
+          }.join(" ").downcase.split(/\s+/)
+        )
+      }]
+    }
+  }
+
+  it "works on the digg data" do
+    distance       = SameSame::CosineDistance.new
+    vector_builder = SameSame::DbscanTermFrequencyVectors.new
+    
+    algo = SameSame::DbscanAlgorithm.new(
+      points: digg_data,
+      eps: 0.7,
+      min_points: 2,
+      vector_calculator: vector_builder,
+      distance: distance)
+
+    clusters = algo.cluster
+
+    #SameSame::DendrogramPrinter.new.print_clusters( clusters )
+  end
+
+  it "works on lines" do
+    distance       = SameSame::CosineDistance.new
+    vector_builder = SameSame::DbscanTermFrequencyVectors.new
+
+    line_data.each do |key, group|
+      if group.size > 1
+        algo = SameSame::DbscanAlgorithm.new(
+          points: group,
+          eps: 0.3,
+          min_points: 2,
+          vector_calculator: vector_builder,
+          distance: distance)
+
+        clusters = algo.cluster
+        #expect(clusters.size).to be > 1
+        #SameSame::DendrogramPrinter.new.print_clusters( clusters.select {|c| c.name != "Noise"} )
+      end
+    end
+  end
+end

data/spec/same_same/jaquard_coefficient_spec.rb

@@ -0,0 +1,24 @@
+require 'same_same'
+
+describe SameSame::JaquardCoefficient do
+
+  describe '#similarity' do
+    it "raises ArgumentError if both sets are empty" do
+      expect {subject.similarity([],[])}.to raise_error(ArgumentError)
+    end
+
+    it "is 0 if either set is empty" do
+      expect(subject.similarity([],["yolo"])).to eq(0.0)
+      expect(subject.similarity(["yolo"],[])).to eq(0.0)
+    end
+
+    it "is 1.0 for identical sets" do
+      expect( subject.similarity(["yolo"], ["yolo"]) ).to eq(1.0)
+      expect( subject.similarity(["yolo", "oloy"], ["yolo", "oloy"]) ).to eq(1.0)
+    end
+
+    it "is 1/3 for 1 common element and 3 total" do
+      expect( subject.similarity(["yolo", "polo"], ["yolo", "oloy"]) ).to eq(1.0 / 3.0)
+    end
+  end
+end

data/spec/same_same/link_matrix_spec.rb

@@ -0,0 +1,29 @@
+require 'same_same'
+
+describe SameSame::LinkMatrix do
+  let( :datapoints ) {(0..19).to_a}
+  let( :similarity_matrix ) {
+    SameSame::SymmetricalMatrix.new(datapoints.size).tap do |m|
+      datapoints.each do |i|
+        m.set(i,i,1.0)
+      end
+      (0..datapoints.size-1).each do |i|
+        (0..i-1).each do |j|
+          m.set(i,j,(i+j)/40.0)
+        end
+      end
+    end
+  }
+  let( :th ) {0.2}
+
+  subject {SameSame::LinkMatrix.new(similarity_matrix: similarity_matrix, datapoints: datapoints, th: th)}
+
+  describe "#number_of_links_between_points" do
+    it "returns a number" do
+      # WARNING: I don't know if these numbers are right!!!!
+      expect( subject.number_of_links_between_points(0,1) ).to eq(12)
+      expect( subject.number_of_links_between_points(1,2) ).to eq(13)
+      expect( subject.number_of_links_between_points(19,19) ).to eq(20)
+    end
+  end
+end

data/spec/same_same/merge_goodness_measure_spec.rb

@@ -0,0 +1,34 @@
+require 'same_same'
+
+describe SameSame::MergeGoodnessMeasure do
+
+  context "0 proximity (all are neighbours)" do
+    subject {SameSame::MergeGoodnessMeasure.new(0.0)}
+
+    describe '#g' do
+      it "should do something I'm not quite sure of" do
+        # these 'specs' aren't specs - I'm not quite in a position
+        # to write anything specific here. Just really ensuring that
+        # the methods don't fail
+        expect( subject.g( 10, 10, 10 ) ).to eq(0.0016666666666666668)
+        expect( subject.g( 2, 10, 10 ) ).to  eq(0.0003333333333333333)
+      end
+    end
+  end
+
+  context "1.0 proximity (no neighbours)" do
+    subject {SameSame::MergeGoodnessMeasure.new(1.0)}
+
+    describe '#g' do
+      it "should always return infinity" do
+        # note to self: this makes sense. If no datapoints are
+        # considered neighbours the goodness for any cluster will
+        # infinite (where infinite means bad). Why isn't it called
+        # a BADNESS measure then?
+        expect( subject.g( 10, 10, 10 ) ).to eq(Float::INFINITY)
+        expect( subject.g( 2, 10, 10 ) ).to eq(Float::INFINITY)
+      end
+    end
+  end
+
+end

data/spec/same_same/rock_algorithm_spec.rb

@@ -0,0 +1,71 @@
+require 'csv'
+require 'same_same'
+require 'same_same/dendrogram_printer'
+
+describe SameSame::RockAlgorithm do
+  let(:dp1) {["book"]}
+  let(:dp2) {["water", "sun", "sand", "swim"]}
+  let(:dp3) {["water", "sun", "swim", "read"]}
+  let(:dp4) {["read", "sand"]}
+  let(:points) {[dp1, dp2, dp3, dp4].map {|data| SameSame::DataPoint.new(data.join("-"), data)}}
+  let(:digg_data) {
+    digg_data = CSV.read("spec/fixtures/digg_stories.csv", headers: true)
+    digg_data.map {|row|
+      SameSame::DataPoint.new( row["title"], 
+        %w(category topic description).map {|key|
+          row[key]
+        }.join(" ").downcase.split(/\s+/)
+      )
+    }
+  }
+
+  let(:line_data) {
+    csv = CSV.read("spec/fixtures/lines.csv", headers: true)
+
+    groups = csv.group_by {|csv| [csv['categories'], csv['price']].join("-")}
+
+    groups.map {|key, group|
+      [key, group.map {|row|
+        SameSame::DataPoint.new( [row["id"], row["name"]].map {|t| t.gsub(/\s+/, ' ')}.join(": "), 
+          %w(name price price price price categories).map {|key|
+            row[key]
+          }.join(" ").downcase.split(/\s+/) + [row["name"].downcase.gsub(/\s+/, ' ')]
+        )
+      }]
+    }
+  }
+
+  # it "works" do
+  #   k = 2
+  #   th = 0.2
+  #   algo = SameSame::RockAlgorithm.new(datapoints: points, k: k, th: th)
+  #   dnd = algo.cluster
+
+  #   SameSame::DendrogramPrinter.new.print(dnd)
+  # end
+
+
+
+  it "works on the digg data" do
+    k = 2
+    th = 0.2
+    algo = SameSame::RockAlgorithm.new(datapoints: digg_data, k: k, th: th)
+    dnd = algo.cluster
+
+    #SameSame::DendrogramPrinter.new.print_last(dnd)
+  end
+
+  it "works on lines" do
+    k = 4
+    th = 0.4
+    line_data.each do |key, group|
+      if group.size > 1
+        algo = SameSame::RockAlgorithm.new(datapoints: group, k: k, th: th)
+        dnd = algo.cluster
+        if dnd.non_singelton_leaves?
+          #SameSame::DendrogramPrinter.new.print_last(dnd)
+        end
+      end
+    end
+  end
+end

data/spec/same_same/similarity_matrix_spec.rb

@@ -0,0 +1,20 @@
+require 'same_same'
+
+describe SameSame::SimilarityMatrix do
+  let(:measure) {double("Measure")}
+  let(:datapoints) {[1,2,3]}
+
+  subject {SameSame::SimilarityMatrix.new( measure, datapoints )}
+  
+  describe "#[]" do
+    it "returns 1.0 without calculation for self similarity" do
+      expect( subject.lookup 1,1 ).to eq(1.0)
+    end
+
+    it "calculates simlarity for different datapoints" do
+      measure.should_receive( :similarity ).with( 2, 3 ).and_return 2.0
+      expect( subject.lookup 1,2 ).to eq(2.0)
+    end
+
+  end
+end

data/spec/same_same/symmetrical_matrix_spec.rb

@@ -0,0 +1,69 @@
+require 'same_same'
+
+describe SameSame::SymmetricalMatrix do
+
+  describe "#lookup" do
+    
+    context "with no lookup" do
+      subject {SameSame::SymmetricalMatrix.new(5)}
+
+      it "is nil for unspecified elements" do
+        expect( subject.lookup(1,2) ).to be_nil
+      end
+
+      it "returns a set value" do
+        subject.set(1,2,"value")
+        expect( subject.lookup(1,2) ).to eq("value")
+      end
+
+      it "returns the inverse of a set value" do
+        subject.set(1,2,"value")
+        expect( subject.lookup(2,1) ).to eq("value")
+      end
+
+      it "raises argument error for out of bounds parameters" do
+        expect { subject.lookup(5,1)  }.to raise_error( ArgumentError)
+        expect { subject.lookup(1,5)  }.to raise_error( ArgumentError)
+        expect { subject.lookup(-1,1) }.to raise_error( ArgumentError)
+        expect { subject.lookup(1,-1) }.to raise_error( ArgumentError)
+      end
+   
+      it "uses blocks to calulate missing values" do
+        expect( subject.lookup(1,2) {|x,y| "#{x}-#{y}"} ).to eq("1-2")
+      end
+
+      it "ignores blocks when value set" do
+        subject.set(1,2,"value")
+        expect( subject.lookup(1,2) {|x,y| "#{x}-#{y}"} ).to eq("value")
+      end
+
+    end
+
+    context "with a lookup proc" do
+      let( :proc ) {double("proc")}
+      subject {SameSame::SymmetricalMatrix.new(5, proc)}
+
+      it "is calculated for unspecified elements" do
+        proc.should_receive( :call ).with(1,2).and_return "1-2"
+        expect( subject.lookup(1,2) ).to eq("1-2")
+      end
+
+      it "is only calls the proc once" do
+        proc.should_receive( :call ).with(1,2).and_return "1-2"
+        expect( subject.lookup(1,2) ).to eq("1-2")
+        expect( subject.lookup(1,2) ).to eq("1-2")
+      end
+
+      it "is only calls the proc once if the inverse is looked up" do
+        proc.should_receive( :call ).with(1,2).and_return "1-2"
+        expect( subject.lookup(1,2) ).to eq("1-2")
+        expect( subject.lookup(2,1) ).to eq("1-2")
+      end
+
+      it "is prefers set values to the lookup" do
+        subject.set(1,2,"value")
+        expect( subject.lookup(1,2) ).to eq("value")
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,144 @@
+--- !ruby/object:Gem::Specification
+name: same_same
+version: !ruby/object:Gem::Version
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors:
+- Julian Russell
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  name: rspec
+  type: :development
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+- !ruby/object:Gem::Dependency
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  name: pry-debugger
+  type: :development
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+- !ruby/object:Gem::Dependency
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  name: colored
+  type: :development
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+description: Implementation of ROCK and DBSCAN clustering algorithms
+email:
+- julian@myfoodlink.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/dbscan_digg.rb
+- examples/dbscan_lines.rb
+- examples/rock_digg.rb
+- examples/rock_lines.rb
+- lib/same_same.rb
+- lib/same_same/cluster.rb
+- lib/same_same/cluster_similarity.rb
+- lib/same_same/cosine_distance.rb
+- lib/same_same/cosine_similarity.rb
+- lib/same_same/data_point.rb
+- lib/same_same/dbscan_algorithm.rb
+- lib/same_same/dbscan_clusters.rb
+- lib/same_same/dbscan_neighborhood.rb
+- lib/same_same/dbscan_numeric_vectors.rb
+- lib/same_same/dbscan_term_frequency_vectors.rb
+- lib/same_same/dendrogram.rb
+- lib/same_same/dendrogram_printer.rb
+- lib/same_same/jaquard_coefficient.rb
+- lib/same_same/link_matrix.rb
+- lib/same_same/merge_goodness_measure.rb
+- lib/same_same/rock_algorithm.rb
+- lib/same_same/rock_clusters.rb
+- lib/same_same/similarity_matrix.rb
+- lib/same_same/symmetrical_matrix.rb
+- lib/same_same/term_frequency_builder.rb
+- lib/same_same/version.rb
+- same_same.gemspec
+- spec/fixtures/digg_stories.csv
+- spec/fixtures/lines.csv
+- spec/same_same/dbscan_algorithm_spec.rb
+- spec/same_same/jaquard_coefficient_spec.rb
+- spec/same_same/link_matrix_spec.rb
+- spec/same_same/merge_goodness_measure_spec.rb
+- spec/same_same/rock_algorithm_spec.rb
+- spec/same_same/similarity_matrix_spec.rb
+- spec/same_same/symmetrical_matrix_spec.rb
+homepage: https://github.com/plusplus/same_same
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Implementation of ROCK and DBSCAN clustering algorithms
+test_files:
+- spec/fixtures/digg_stories.csv
+- spec/fixtures/lines.csv
+- spec/same_same/dbscan_algorithm_spec.rb
+- spec/same_same/jaquard_coefficient_spec.rb
+- spec/same_same/link_matrix_spec.rb
+- spec/same_same/merge_goodness_measure_spec.rb
+- spec/same_same/rock_algorithm_spec.rb
+- spec/same_same/similarity_matrix_spec.rb
+- spec/same_same/symmetrical_matrix_spec.rb