rp-simhash 1.1.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 46c04f99125260cf8c37455d86de8e4eda2b0d30
4
+ data.tar.gz: 13b97e76cddcfed9d0494d3728b51688e0f87e6e
5
+ SHA512:
6
+ metadata.gz: a3b8aa2473b65259fa4a20385ffd319589629a135080cc936e0de7d243cadac4d90dde89f7bb737891d6b9582945b39ca4e58015a35a39135b2723a3062e457d
7
+ data.tar.gz: f238ba5da592a28dd23d90017a9096a76ddd0c08e21ef318b4d9dc80942d76b009578410e87cce73c06fe37b39e7d3d036a2fc65ebc638bd7483b3238b11518a
data/LICENSE ADDED
@@ -0,0 +1,28 @@
1
+
2
+ LICENSE
3
+
4
+ The MIT License
5
+
6
+ Copyright (c) 2002 Charikar, Simhash algorythm
7
+ Copyright (c) 2009 Andre Hagenbruch, Python implementation
8
+ Copyright (c) 2010 Bookmate.ru
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy
11
+ of this software and associated documentation files (the "Software"), to deal
12
+ in the Software without restriction, including without limitation the rights
13
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14
+ copies of the Software, and to permit persons to whom the Software is
15
+ furnished to do so, subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in
18
+ all copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26
+ THE SOFTWARE.
27
+
28
+
@@ -0,0 +1,61 @@
1
+ ==Absctract
2
+
3
+ This is implementation of {Moses Charikar's simhashes}[http://portal.acm.org/citation.cfm?id=509965] in Ruby.
4
+
5
+ ==Usage
6
+
7
+ When you have a string and want to calculate it's simhash, you should
8
+
9
+ my_string.simhash
10
+
11
+ By default it will generate 64-bit integer - that is simhash for this string
12
+
13
+ It's always better to tokenize string before simhashing. It's as simple as
14
+
15
+ my_string.simhash(:split_by => / /)
16
+
17
+ This will generate 64-bit integer based, but will split string into words before.
18
+ It's handy when you need to calculate similarity of strings based on word usage.
19
+ You can split string as you like: by letters/sentences/specific letter-combinations, etc.
20
+
21
+ my_string.simhash(:split_by => /./, :hashbits => 512)
22
+
23
+ Sometimes you might need longer simhash (finding similarity for very long strings is a good example).
24
+ You can set length of result hash by passing hashbits parameter. This example will return 512-bit simhash
25
+ for your string splitted by sentences.
26
+
27
+ ==Advanced usage
28
+
29
+ It's useful to clean your string before simhashing. But it's useful not to clean, too.
30
+
31
+ Here are examples:
32
+
33
+ my_string.simhash(:stop_words => true) # here we clean
34
+
35
+ This will find stop-words in your string and remove them before simhashing. Stop-words are "the", "not", "about", etc.
36
+ Currently we remove only Russian and English stop-words.
37
+
38
+ my_string.simhash(:preserve_punctuation => true) # here we not
39
+
40
+ This will not remove punctuation before simhashing. Yes, we remove all dots, commas, etc. after splitting string to words by default.
41
+ Because different punctiation does not mean difference in general. If you not agree you can turn this default off.
42
+
43
+ ==Installation
44
+
45
+ As usual:
46
+
47
+ gem install simhash
48
+
49
+ But if you have {GNU MP library}[http://gmplib.org/], simhash will work faster! To check out which version is used, type:
50
+
51
+ Simhash::DEFAULT_STRING_HASH_METHOD
52
+
53
+ It should return symbol. If symbol ends with "rb", your simhash is slow. If you want make it faster, install GNU MP.
54
+
55
+ To build C extensions:
56
+
57
+ cd ext/string_hashing
58
+ ARCHFLAGS='-arch x84_64' ruby extconf.rb
59
+ make
60
+
61
+ You might be able to omit `ARCHFLAGS` or use a different version.
@@ -0,0 +1,46 @@
1
+ require 'rake'
2
+ require 'rake/testtask'
3
+ require 'rake/rdoctask'
4
+
5
+ $LOAD_PATH << File.join(File.dirname(__FILE__), 'lib')
6
+ require 'simhash'
7
+
8
+ desc 'Default: run unit tests.'
9
+ task :default => [:test]
10
+
11
+ desc 'Test the simhash gem'
12
+ Rake::TestTask.new(:test) do |t|
13
+ t.libs << '.'
14
+ t.pattern = 'test/**/*_test.rb'
15
+ t.verbose = true
16
+ end
17
+
18
+ desc 'Start an IRB session with all necessary files required.'
19
+ task :shell do |t|
20
+ chdir File.dirname(__FILE__)
21
+ exec 'irb -I lib/ -I lib/simhash -I lib/string -I lib/integer -r rubygems'
22
+ end
23
+
24
+ desc 'Build the gemspec.'
25
+ task :gemspec do |t|
26
+ exec 'gem build simhash.gemspec'
27
+ end
28
+
29
+ desc "Print a list of the files to be put into the gem"
30
+ task :manifest do
31
+ spec.files.each do |file|
32
+ puts file
33
+ end
34
+ end
35
+
36
+ desc "Generate a gemspec file for GitHub"
37
+ task :gemspec do
38
+ File.open("#{spec.name}.gemspec", 'w') do |f|
39
+ f.write spec.to_ruby
40
+ end
41
+ end
42
+
43
+ desc "Build the gem into the current directory"
44
+ task :gem => :gemspec do
45
+ `gem build #{spec.name}.gemspec`
46
+ end
@@ -0,0 +1,48 @@
1
+ require 'mkmf'
2
+
3
+ extension_name = 'string_hashing'
4
+ # should link against the libgmp library
5
+ $LDFLAGS << ' -lgmp'
6
+
7
+ # Sort out the universal vs. single-archicture build problems on MacOS X
8
+ if RUBY_PLATFORM.include?( 'darwin' )
9
+ puts "MacOS X build: fixing architecture flags:"
10
+
11
+ commonflags = nil
12
+ if ENV['ARCHFLAGS']
13
+ puts " using the value in ARCHFLAGS environment variable (%p)." % [ ENV['ARCHFLAGS'] ]
14
+ commonflags = ENV['ARCHFLAGS']
15
+ else
16
+ $stderr.puts %{
17
+ =========== WARNING ===========
18
+
19
+ You are building this extension on OS X without setting the
20
+ ARCHFLAGS environment variable.
21
+
22
+ If you are seeing this message, that means that the
23
+ build will probably fail.
24
+
25
+ ===================================
26
+ }.gsub( /^\t+/, ' ' )
27
+ end
28
+
29
+ if commonflags
30
+ $CFLAGS.gsub!( /-arch\s+\S+ /, '' )
31
+ $LDFLAGS.gsub!( /-arch\s+\S+ /, '' )
32
+ CONFIG['LDSHARED'].gsub!( /-arch\s+\S+ /, '' )
33
+
34
+ $CFLAGS << ' ' << commonflags
35
+ $LDFLAGS << ' ' << commonflags
36
+ CONFIG['LDSHARED'] << ' ' << commonflags
37
+ end
38
+ end
39
+
40
+ if (find_header("gmp.h") rescue false)
41
+ $stderr.puts "Configuring extensions"
42
+ dir_config(extension_name)
43
+ create_makefile(extension_name)
44
+ else
45
+ $stderr.puts "Skipping building of C extension"
46
+ # creating foo Makefile to avoid building stuff
47
+ File.open(File.join(File.dirname(__FILE__), "Makefile"), "w"){|f| f.write("all: \ninstall: \n")}
48
+ end
@@ -0,0 +1,64 @@
1
+ #include "ruby.h"
2
+ #include <gmp.h>
3
+ #include <stdio.h>
4
+
5
+ VALUE StringHashing = Qnil;
6
+
7
+ void Init_string_hashing();
8
+
9
+ VALUE method_hash_vl(VALUE self, VALUE bitlength);
10
+
11
+ void Init_string_hashing() {
12
+ rb_define_method(rb_cString, "hash_vl", method_hash_vl, 1);
13
+ }
14
+
15
+ VALUE method_hash_vl(VALUE self, VALUE bitlength) {
16
+ int bl = NUM2INT(bitlength);
17
+
18
+ // for hard typecasting
19
+ unsigned char one_char;
20
+ char* result;
21
+ result = malloc(bl*sizeof(char));
22
+ unsigned long long len = RSTRING_LEN(self);
23
+ char *string = RSTRING_PTR(self);
24
+
25
+ if(len == 0){ return 0; }
26
+
27
+ mpz_t x, mask, long_len;
28
+ mpz_init_set_ui (long_len, len);
29
+ one_char = RSTRING_PTR(self)[0];
30
+ mpz_init_set_ui (x, one_char << 7);
31
+ int m = 1000003;
32
+
33
+ // generating mask of length bitlength filled with 1
34
+ mpz_init (mask);
35
+ mpz_ui_pow_ui(mask, 2, bl);
36
+ mpz_sub_ui (mask, mask, 1);
37
+
38
+ mpz_t computations, byte;
39
+ mpz_init(computations);
40
+ mpz_init2 (byte, 8);
41
+
42
+ int i = 0;
43
+ for(i; i < len; i++) {
44
+ one_char = string[i];
45
+ mpz_set_ui(byte, one_char);
46
+ mpz_mul_ui(computations, x, m);
47
+ mpz_xor(computations, computations, byte);
48
+ mpz_and (x, mask, computations);
49
+ }
50
+
51
+ mpz_xor(x, x, long_len);
52
+ //gmp_printf ("C xored x is %Zd\n", x);
53
+ mpz_get_str (result, 10, x);
54
+ VALUE res = rb_str_new2(result);
55
+
56
+ mpz_clear(x);
57
+ mpz_clear(byte);
58
+ mpz_clear(computations);
59
+ mpz_clear(mask);
60
+ mpz_clear(long_len);
61
+ free(result);
62
+
63
+ return res;
64
+ }
@@ -0,0 +1,16 @@
1
+ class Integer
2
+ # Hamming distance – number of different bits in same positions
3
+ # H(1001, 1110) = 3
4
+ # H(1001, 1000) = 1
5
+ def hamming_distance_to(integer)
6
+ total = 0
7
+ difference = self ^ integer
8
+
9
+ while difference > 0 do
10
+ total += 1 if (difference & 1).nonzero?
11
+ difference >>= 1
12
+ end
13
+
14
+ total
15
+ end
16
+ end
@@ -0,0 +1,54 @@
1
+ # encoding: utf-8
2
+
3
+ require 'active_support/core_ext/string/multibyte'
4
+ require 'unicode'
5
+
6
+ require 'string_hasher'
7
+ require 'integer'
8
+ require 'simhash/stopwords'
9
+
10
+ begin
11
+ require_relative '../ext/string_hashing/string_hashing'
12
+ rescue LoadError
13
+ end
14
+
15
+ module Simhash
16
+ # @param hashbits The number of bits in the resulting hash
17
+ # @param hasher An object which returns a numeric hash when #hash is called with a token
18
+ #
19
+ def self.hash(tokens, options={})
20
+ hashbits = options[:hashbits] || 64
21
+ hasher = options[:hasher] || StringHasher.new(hashbits)
22
+
23
+ v = [0] * hashbits
24
+ masks = v.dup
25
+ masks.each_with_index {|e, i| masks[i] = (1 << i)}
26
+
27
+ self.each_token(tokens, options) do |token|
28
+ hashed_token = hasher.hash(token).to_i
29
+ hashbits.times do |i|
30
+ v[i] += (hashed_token & masks[i]).zero? ? -1 : +1
31
+ end
32
+ end
33
+
34
+ fingerprint = 0
35
+
36
+ hashbits.times { |i| fingerprint += 1 << i if v[i] >= 0 }
37
+
38
+ fingerprint
39
+ end
40
+
41
+ def self.each_token(tokens, options={})
42
+ token_min_size = options[:token_min_size].to_i
43
+
44
+ tokens.each do |token|
45
+ next if token.size.zero? || token.mb_chars.size < token_min_size
46
+
47
+ yield token
48
+ end
49
+ end
50
+
51
+ def self.hm
52
+ @@string_hash_method
53
+ end
54
+ end
@@ -0,0 +1,8 @@
1
+ require File.join(File.dirname(__FILE__), "stopwords", "en")
2
+ require File.join(File.dirname(__FILE__), "stopwords", "ru")
3
+
4
+ module Simhash
5
+ module Stopwords
6
+ ALL = RU + EN
7
+ end
8
+ end
@@ -0,0 +1,5 @@
1
+ module Simhash
2
+ module Stopwords
3
+ EN = " a able about above abst accordance according accordingly across act actually added adj adopted affected affecting affects after afterwards again against ah all almost alone along already also although always am among amongst an and announce another any anybody anyhow anymore anyone anything anyway anyways anywhere apparently approximately are aren arent arise around as aside ask asking at auth available away awfully b back be became because become becomes becoming been before beforehand begin beginning beginnings begins behind being believe below beside besides between beyond biol both brief briefly but by c ca came can cannot can't cause causes certain certainly co com come comes contain containing contains could couldnt d date did didn't different do does doesn't doing done don't down downwards due during e each ed edu effect eg eight eighty either else elsewhere end ending enough especially et et-al etc even ever every everybody everyone everything everywhere ex except f far few ff fifth first five fix followed following follows for former formerly forth found four from further furthermore g gave get gets getting give given gives giving go goes gone got gotten h had happens hardly has hasn't have haven't having he hed hence her here hereafter hereby herein heres hereupon hers herself hes hi hid him himself his hither home how howbeit however hundred i id ie if i'll im immediate immediately importance important in inc indeed index information instead into invention inward is isn't it itd it'll its itself i've j just k keep keeps kept keys kg km know known knows l largely last lately later latter latterly least less lest let lets like liked likely line little 'll look looking looks ltd m made mainly make makes many may maybe me mean means meantime meanwhile merely mg might million miss ml more moreover most mostly mr mrs much mug must my myself n na name namely nay nd near nearly necessarily necessary need needs neither never nevertheless new next nine ninety no nobody non none nonetheless noone nor normally nos not noted nothing now nowhere o obtain obtained obviously of off often oh ok okay old omitted on once one ones only onto or ord other others otherwise ought our ours ourselves out outside over overall owing own p page pages part particular particularly past per perhaps placed please plus poorly possible possibly potentially pp predominantly present previously primarily probably promptly proud provides put q que quickly quite qv r ran rather rd re readily really recent recently ref refs regarding regardless regards related relatively research respectively resulted resulting results right run s said same saw say saying says sec section see seeing seem seemed seeming seems seen self selves sent seven several shall she shed she'll shes should shouldn't show showed shown showns shows significant significantly similar similarly since six slightly so some somebody somehow someone somethan something sometime sometimes somewhat somewhere soon sorry specifically specified specify specifying state states still stop strongly sub substantially successfully such sufficiently suggest sup sure t take taken taking tell tends th than thank thanks thanx that that'll thats that've the their theirs them themselves then thence there thereafter thereby thered therefore therein there'll thereof therere theres thereto thereupon there've these they theyd they'll theyre they've think this those thou though thoughh thousand throug through throughout thru thus til tip to together too took toward towards tried tries truly try trying ts twice two u un under unfortunately unless unlike unlikely until unto up upon ups us use used useful usefully usefulness uses using usually v value various 've very via viz vol vols vs w want wants was wasn't way we wed welcome we'll went were weren't we've what whatever what'll whats when whence whenever where whereafter whereas whereby wherein wheres whereupon wherever whether which while whim whither who whod whoever whole who'll whom whomever whos whose why widely willing wish with within without won't words world would wouldn't www x y yes yet you youd you'll your youre yours yourself yourselves you've z zero "
4
+ end
5
+ end
@@ -0,0 +1,6 @@
1
+ # encoding: utf-8
2
+ module Simhash
3
+ module Stopwords
4
+ RU = " а е и ж м о на не ни об но он мне мои мож она они оно мной много многочисленное многочисленная многочисленные многочисленный мною мой мог могут можно может можхо мор моя моё мочь над нее оба нам нем нами ними мимо немного одной одного менее однажды однако меня нему меньше ней наверху него ниже мало надо один одиннадцать одиннадцатый назад наиболее недавно миллионов недалеко между низко меля нельзя нибудь непрерывно наконец никогда никуда нас наш нет нею неё них мира наша наше наши ничего начала нередко несколько обычно опять около мы ну нх от отовсюду особенно нужно очень отсюда в во вон вниз внизу вокруг вот восемнадцать восемнадцатый восемь восьмой вверх вам вами важное важная важные важный вдали везде ведь вас ваш ваша ваше ваши впрочем весь вдруг вы все второй всем всеми времени время всему всего всегда всех всею всю вся всё всюду г год говорил говорит года году где да ее за из ли же им до по ими под иногда довольно именно долго позже более должно пожалуйста значит иметь больше пока ему имя пор пора потом потому после почему почти посреди ей два две двенадцать двенадцатый двадцать двадцатый двух его дел или без день занят занята занято заняты действительно давно девятнадцать девятнадцатый девять девятый даже алло жизнь далеко близко здесь дальше для лет зато даром первый перед затем зачем лишь десять десятый ею её их бы еще при был про процентов против просто бывает бывь если люди была были было будем будет будете будешь прекрасно буду будь будто будут ещё пятнадцать пятнадцатый друго другое другой другие другая других есть пять быть лучше пятый к ком конечно кому кого когда которой которого которая которые который которых кем каждое каждая каждые каждый кажется как какой какая кто кроме куда кругом с т у я та те уж со то том снова тому совсем того тогда тоже собой тобой собою тобою сначала только уметь тот тою хорошо хотеть хочешь хоть хотя свое свои твой своей своего своих свою твоя твоё раз уже сам там тем чем сама сами теми само рано самом самому самой самого семнадцать семнадцатый самим самими самих саму семь чему раньше сейчас чего сегодня себе тебе сеаой человек разве теперь себя тебя седьмой спасибо слишком так такое такой такие также такая сих тех чаще четвертый через часто шестой шестнадцать шестнадцатый шесть четыре четырнадцать четырнадцатый сколько сказал сказала сказать ту ты три эта эти что это чтоб этом этому этой этого чтобы этот стал туда этим этими рядом тринадцать тринадцатый этих третий тут эту суть чуть тысяч "
5
+ end
6
+ end
@@ -0,0 +1,3 @@
1
+ module Simhash
2
+ VERSION = "1.1.1"
3
+ end
@@ -0,0 +1,32 @@
1
+ class StringHasher
2
+ attr_reader :length
3
+
4
+ def initialize(length)
5
+ @length = length
6
+ end
7
+
8
+ def hash(string)
9
+ if String.public_instance_methods.include?(:hash_vl)
10
+ c_hash(string)
11
+ else
12
+ ruby_hash(string)
13
+ end
14
+ end
15
+
16
+ def c_hash(string)
17
+ string.hash_vl(length)
18
+ end
19
+
20
+ def ruby_hash(string)
21
+ return 0 if string == ""
22
+
23
+ x = string.bytes.first << 7
24
+ m = 1000003
25
+ mask = (1<<length) - 1
26
+ string.each_byte{ |char| x = ((x * m) ^ char.to_i) & mask }
27
+
28
+ x ^= string.bytes.count
29
+ x = -2 if x == -1
30
+ x.to_s
31
+ end
32
+ end
metadata ADDED
@@ -0,0 +1,102 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: rp-simhash
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.1.1
5
+ platform: ruby
6
+ authors:
7
+ - Alex Gusev
8
+ - Ryan Michael
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2014-09-12 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: unicode
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - ">="
19
+ - !ruby/object:Gem::Version
20
+ version: 0.3.1
21
+ type: :runtime
22
+ prerelease: false
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - ">="
26
+ - !ruby/object:Gem::Version
27
+ version: 0.3.1
28
+ - !ruby/object:Gem::Dependency
29
+ name: activesupport
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - ">="
33
+ - !ruby/object:Gem::Version
34
+ version: '0'
35
+ type: :runtime
36
+ prerelease: false
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - ">="
40
+ - !ruby/object:Gem::Version
41
+ version: '0'
42
+ - !ruby/object:Gem::Dependency
43
+ name: pry
44
+ requirement: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - ">="
47
+ - !ruby/object:Gem::Version
48
+ version: '0'
49
+ type: :development
50
+ prerelease: false
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - ">="
54
+ - !ruby/object:Gem::Version
55
+ version: '0'
56
+ description: Implementation of Charikar simhashes in Ruby
57
+ email:
58
+ - alex.gusev@bookmate.ru
59
+ - kerinin@gmail.com
60
+ executables: []
61
+ extensions:
62
+ - ext/string_hashing/extconf.rb
63
+ extra_rdoc_files: []
64
+ files:
65
+ - LICENSE
66
+ - README.rdoc
67
+ - Rakefile
68
+ - ext/string_hashing/extconf.rb
69
+ - ext/string_hashing/string_hashing.c
70
+ - lib/integer.rb
71
+ - lib/simhash.rb
72
+ - lib/simhash/stopwords.rb
73
+ - lib/simhash/stopwords/en.rb
74
+ - lib/simhash/stopwords/ru.rb
75
+ - lib/simhash/version.rb
76
+ - lib/string_hasher.rb
77
+ homepage: http://github.com/ReturnPath/simhash
78
+ licenses: []
79
+ metadata: {}
80
+ post_install_message:
81
+ rdoc_options: []
82
+ require_paths:
83
+ - lib
84
+ required_ruby_version: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ required_rubygems_version: !ruby/object:Gem::Requirement
90
+ requirements:
91
+ - - ">="
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ requirements: []
95
+ rubyforge_project:
96
+ rubygems_version: 2.4.1
97
+ signing_key:
98
+ specification_version: 3
99
+ summary: 'Gives you possbility to convert string into simhashes to futher use: finding
100
+ near-duplicates, similar strings, etc.'
101
+ test_files: []
102
+ has_rdoc: