simhash2 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4649489bf0c74cadd1dc2387578fc76994d3bdf6
-  data.tar.gz: 3eb87d8c24c0b476278b7c7fc18a8555c1d3215e
+  metadata.gz: 8e8190a872a7ec498e0a500482e8814758de6cb4
+  data.tar.gz: b0ea69a076ab1a99584010b3c85b6c3d17d37eea
 SHA512:
-  metadata.gz: b4802c746e43f91b99b6cc6f17f7a2757666596b9b70104eae7b4581dbf63a878ac4741b10ba7686bc6d2b7749132c4bf2cc94bca8416614521dac80c1f5f8df
-  data.tar.gz: d5e1cec7950aef8d40f81518644554cfc6e23af590c8249b7adf1feef56c13db452548611e182f6449ed4182367a14c3ca5c6a288e34c40f32e63889b3b4b168
+  metadata.gz: 290b5f9daf27c8d4a138e26aabc062b4c8896fb6506bb41da13ea5c1e44095286e82b9b34780b189d22fdae1e4f4d34d7dc0ce6438b9c47539584b82fb23a541
+  data.tar.gz: 5b961fbf6271a50571d557f628ef111ca7f9683378869bc9443635cb678b0a06942424287e5e6103702074f604a8ac39a06d2a2120c15b3bbfc26ee77d1cc5ab

data/Gemfile

@@ -1,6 +1,6 @@
 source 'http://rubygems.org'
 
-# Specify your gem's dependencies in simhash2.gemspec
+# Specify your gem's dependencies in the .gemspec file
 gemspec
 
 gem 'coveralls', require: false

data/README.md

@@ -1,8 +1,37 @@
 [![Gem Version](https://badge.fury.io/rb/simhash2.svg)](https://badge.fury.io/rb/simhash2)
-[![Code Climate](https://codeclimate.com/github/armchairtheorist/simhash/badges/gpa.svg)](https://codeclimate.com/github/armchairtheorist/simhash)
-[![Build Status](https://travis-ci.org/armchairtheorist/simhash.svg?branch=master)](https://travis-ci.org/armchairtheorist/simhash)
-[![Coverage Status](https://coveralls.io/repos/github/armchairtheorist/simhash/badge.svg?branch=master)](https://coveralls.io/github/armchairtheorist/simhash?branch=master)
+[![Code Climate](https://codeclimate.com/github/armchairtheorist/simhash2/badges/gpa.svg)](https://codeclimate.com/github/armchairtheorist/simhash2)
+[![Build Status](https://travis-ci.org/armchairtheorist/simhash2.svg?branch=master)](https://travis-ci.org/armchairtheorist/simhash2)
+[![Coverage Status](https://coveralls.io/repos/github/armchairtheorist/simhash2/badge.svg?branch=master)](https://coveralls.io/github/armchairtheorist/simhash2?branch=master)
 
-# Simhash
+# Simhash2
+
+**Simhash2** is a rewrite of the [bookmate/simhash](https://github.com/bookmate/simhash) gem, which is an implementation of Moses Charikar's simhashes in Ruby. The key differences are that this gem doesn't monkey patch the `String` and `Integer`, and configuration is also slightly easier. The simhash values generated by this gem on a default configuration should be identical to what is generated by the Bookmate version.
+
+## Installation
+
+Install the gem from RubyGems:
+
+```bash
+gem install simhash2
+```
+
+If you use Bundler, just add it to your Gemfile and run `bundle install`
+
+```ruby
+gem 'simhash2'
+```
+
+I have only tested this gem on Ruby 2.4.1, but there shouldn't be any reason why it wouldn't work on earlier Ruby versions as well.
+
+## Usage
+
+```ruby
+str1 = "I am the king of the world!"
+str2 = "I am the queen of the world!"
+
+simhash1 = Simhash.generate(str1) # => 86798109229625320
+simhash2 = Simhash.generate(str2) # => 13921220612431195624
+
+Simhash.hamming_distance(simhash1, simhash2) # => 8
+```
 
-This is a rewrite of the [bookmate/simhash](https://github.com/bookmate/simhash) gem, which is an implementation of Moses Charikar's simhashes in Ruby.

data/lib/simhash2/version.rb

@@ -1,3 +1,3 @@
 module Simhash
-  VERSION = '0.0.2'.freeze
+  VERSION = '0.0.3'.freeze
 end

data/lib/simhash2.rb

@@ -1,68 +1,76 @@
-require 'simhash2/version'
-
-module Simhash
-  extend self
-
-  HASHBITS = 64
-
-  OPTIONS = {
-    min_token_length: 1,
-    unique: false,
-    stemming: false,
-    stop_words: []
-
-  }.freeze
-
-  def generate(str, options = {})
-    generate_from_tokens(str.split(/\s+/), options)
-  end
-
-  def generate_from_tokens(tokens, options = {})
-    filter_tokens(tokens, OPTIONS.merge(options))
-
-    v = [0] * HASHBITS
-
-    masks = v.dup
-    masks.each_with_index { |_e, i| masks[i] = (1 << i) }
-
-    hashes = tokens.map { |token| simple_string_hash(token, HASHBITS) }
-    hashes.each do |h|
-      HASHBITS.times do |i|
-        v[i] += (h & masks[i]).zero? ? -1 : +1
-      end
-    end
-
-    simhash = 0
-    HASHBITS.times { |i| simhash += 1 << i if v[i] >= 0 }
-
-    simhash
-  end
-
-  def hamming_distance(simhash1, simhash2)
-    (simhash1.to_i ^ simhash2.to_i).to_s(2).count('1')
-  end
-
-  private
-
-  def simple_string_hash(str, length)
-    return 0 if str == ''
-
-    x = str.bytes.first << 7
-    m = 1_000_003
-    mask = (1 << length) - 1
-    str.each_byte { |char| x = ((x * m) ^ char.to_i) & mask }
-
-    x ^= str.bytes.count
-    x = -2 if x == -1
-
-    x.to_i
-  end
-
-  def filter_tokens(tokens, options)
-    tokens.map! { |e| e.downcase.gsub(/\W+/, '') }
-    tokens.reject! { |e| e.nil? || e.length < options[:min_token_length] }
-    tokens.reject! { |e| options[:stop_words].include?(e) } unless options[:stop_words].nil? || options[:stop_words].empty?
-    tokens.map!(&:stem) if options[:stemming]
-    tokens.uniq! if options[:unique]
-  end
-end
+require 'simhash2/version'
+
+module Simhash
+  extend self
+
+  HASHBITS = 64
+
+  OPTIONS = {
+    min_token_length: 1,
+    unique: false,
+    stemming: false,
+    stop_words: []
+  }.freeze
+
+  def similarity(string1, string2, options = {})
+    return hash_similarity(generate(string1, options), generate(string2, options))
+  end
+
+  def generate(str, options = {})
+    generate_from_tokens(str.split(/\s+/), options)
+  end
+
+  def generate_from_tokens(tokens, options = {})
+    filter_tokens(tokens, OPTIONS.merge(options))
+
+    v = [0] * HASHBITS
+
+    masks = v.dup
+    masks.each_with_index { |_e, i| masks[i] = (1 << i) }
+
+    hashes = tokens.map { |token| simple_string_hash(token, HASHBITS) }
+    hashes.each do |h|
+      HASHBITS.times do |i|
+        v[i] += (h & masks[i]).zero? ? -1 : +1
+      end
+    end
+
+    simhash = 0
+    HASHBITS.times { |i| simhash += 1 << i if v[i] >= 0 }
+
+    simhash
+  end
+
+  def hamming_distance(simhash1, simhash2)
+    (simhash1.to_i ^ simhash2.to_i).to_s(2).count('1')
+  end
+
+  def hash_similarity(left, right)
+    return (1.0 - (hamming_distance(left, right).to_f / HASHBITS))
+  end
+
+  private
+
+  def simple_string_hash(str, length)
+    return 0 if str == ''
+
+    x = str.bytes.first << 7
+    m = 1_000_003
+    mask = (1 << length) - 1
+    str.each_byte { |char| x = ((x * m) ^ char.to_i) & mask }
+
+    x ^= str.bytes.count
+    x = -2 if x == -1
+
+    x.to_i
+  end
+
+  def filter_tokens(tokens, options)
+    tokens.map! { |e| e.downcase.gsub(/\W+/, '') }
+    tokens.reject! { |e| e.nil? || e.length < options[:min_token_length] }
+    tokens.reject! { |e| options[:stop_words].include?(e) } unless options[:stop_words].nil? || options[:stop_words].empty?
+    tokens.map!(&:stem) if options[:stemming]
+    tokens.uniq! if options[:unique]
+  end
+
+end

data/simhash2.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Jonathan Wong']
   spec.email         = ['jonathan@armchairtheorist.com']
   spec.summary       = 'A rewrite of the \'simhash\' gem, which is an implementation of Moses Charikar\'s simhashes in Ruby.'
-  spec.homepage      = 'http://github.com/armchairtheorist/simhash'
+  spec.homepage      = 'https://github.com/armchairtheorist/simhash2'
   spec.license       = 'MIT'
 
   spec.files         = `git ls-files`.split("\n")

data/spec/simhash_spec.rb

@@ -1,51 +1,60 @@
-require 'spec_helper'
-require 'fast-stemmer'
-
-describe Simhash do
-  it 'should generate the same simhash for the same string, and a different simhash for a different string' do
-    str1 = 'I like going to the beach'
-    str2 = 'I like going to the beach'
-    str3 = 'I like going to the mall'
-
-    expect(Simhash.generate(str1)).to eq Simhash.generate(str2)
-    expect(Simhash.generate(str1)).not_to eq Simhash.generate(str3)
-  end
-
-  it 'should strip punctuation and capitalization properly' do
-    str1 = "Hello, nurse!  How's it going today...   my man?"
-    str2 = 'hello nurse hows it going today my man'
-
-    expect(Simhash.generate(str1, unique: true)).to eq Simhash.generate(str2, unique: true)
-  end
-
-  it "should respect the 'unique' option" do
-    str1 = 'apple pear'
-    str2 = 'apple apple apple pear'
-
-    expect(Simhash.generate(str1, unique: true)).to eq Simhash.generate(str2, unique: true)
-    expect(Simhash.generate(str1, unique: false)).not_to eq Simhash.generate(str2, unique: false)
-  end
-
-  it "should respect the 'stop_words' option" do
-    str1 = 'I like the man on the moon.'
-    str2 = 'like man moon'
-    stop_words = %w[i the on]
-
-    expect(Simhash.generate(str1, stop_words: stop_words)).to eq Simhash.generate(str2, stop_words: stop_words)
-    expect(Simhash.generate(str1)).not_to eq Simhash.generate(str2)
-  end
-
-  it "should respect the 'stemming' option" do
-    str1 = 'My crazy cars have crazy minds!'
-    str2 = 'My crazi car have crazi mind!'
-
-    expect(Simhash.generate(str1, stemming: true)).to eq Simhash.generate(str2, stemming: true)
-    expect(Simhash.generate(str1, stemming: false)).not_to eq Simhash.generate(str2, stemming: false)
-  end
-
-  it 'should calculate hamming distances correctly' do
-    expect(Simhash.hamming_distance(2, 2)).to eq 0
-    expect(Simhash.hamming_distance(2, 3)).to eq 1
-    expect(Simhash.hamming_distance(255, 197)).to eq 4
-  end
-end
+require 'simhash2'
+require 'fast-stemmer'
+
+describe Simhash do
+  it 'should generate the same simhash for the same string, and a different simhash for a different string' do
+    str1 = 'I like going to the beach'
+    str2 = 'I like going to the beach'
+    str3 = 'I like going to the mall'
+
+    expect(Simhash.generate(str1)).to eq Simhash.generate(str2)
+    expect(Simhash.generate(str1)).not_to eq Simhash.generate(str3)
+  end
+
+  it 'should calculate the same similarity for the same string, and a lower similarity for a different string' do
+    str1 = 'I like going to the beach'
+    str2 = 'I like going to the beach'
+    str3 = 'I like going to the mall'
+
+    expect(Simhash.similarity(str1, str2)).to eq 1.0
+    expect(Simhash.similarity(str1, str3)).to be < 1.0
+  end
+
+  it 'should strip punctuation and capitalization properly' do
+    str1 = "Hello, nurse!  How's it going today...   my man?"
+    str2 = 'hello nurse hows it going today my man'
+
+    expect(Simhash.generate(str1, unique: true)).to eq Simhash.generate(str2, unique: true)
+  end
+
+  it "should respect the 'unique' option" do
+    str1 = 'apple pear'
+    str2 = 'apple apple apple pear'
+
+    expect(Simhash.generate(str1, unique: true)).to eq Simhash.generate(str2, unique: true)
+    expect(Simhash.generate(str1, unique: false)).not_to eq Simhash.generate(str2, unique: false)
+  end
+
+  it "should respect the 'stop_words' option" do
+    str1 = 'I like the man on the moon.'
+    str2 = 'like man moon'
+    stop_words = %w[i the on]
+
+    expect(Simhash.generate(str1, stop_words: stop_words)).to eq Simhash.generate(str2, stop_words: stop_words)
+    expect(Simhash.generate(str1)).not_to eq Simhash.generate(str2)
+  end
+
+  it "should respect the 'stemming' option" do
+    str1 = 'My crazy cars have crazy minds!'
+    str2 = 'My crazi car have crazi mind!'
+
+    expect(Simhash.generate(str1, stemming: true)).to eq Simhash.generate(str2, stemming: true)
+    expect(Simhash.generate(str1, stemming: false)).not_to eq Simhash.generate(str2, stemming: false)
+  end
+
+  it 'should calculate hamming distances correctly' do
+    expect(Simhash.hamming_distance(2, 2)).to eq 0
+    expect(Simhash.hamming_distance(2, 3)).to eq 1
+    expect(Simhash.hamming_distance(255, 197)).to eq 4
+  end
+end

data/spec/spec_helper.rb

@@ -1,8 +1,6 @@
 require 'coveralls'
 Coveralls.wear!
 
-require 'simhash'
-
 # This file was generated by the `rspec --init` command. Conventionally, all
 # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
 # The generated `.rspec` file contains `--require spec_helper` which will cause

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: simhash2
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Jonathan Wong
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-12 00:00:00.000000000 Z
+date: 2017-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -59,7 +59,7 @@ files:
 - simhash2.gemspec
 - spec/simhash_spec.rb
 - spec/spec_helper.rb
-homepage: http://github.com/armchairtheorist/simhash
+homepage: https://github.com/armchairtheorist/simhash2
 licenses:
 - MIT
 metadata: {}