nilsimsa 1.0.1

data/README

@@ -0,0 +1,18 @@
+nilsimsa
+--------
+Nilsimsa is a distance based hash, which is the opposite of more familiar
+hashes like MD5.  Instead of small changes making a large difference in
+the resulting hash (to avoid collisions), distance based hashes cause
+similar values to have similar output.  This is good for detecting near
+similar documents without having to store the original text.
+
+Standard usage is as follows:
+
+  require 'nilsimsa'
+
+  n1 = Nilsimsa::new
+  text1 = "The quick brown fox"
+  n1.update(text1)
+  puts "Text '#{text1}': #{n1.hexdigest}"
+
+

data/bin/nilsimsa

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'nilsimsa'
+
+if ARGV.size > 0 then
+  ARGV.each do |filename|
+    if FileTest::exists?(filename) then
+      n = Nilsimsa::new
+      n.file(filename)
+      puts n.hexdigest+" #{filename}"
+    else
+      puts "error: can't find '#{filename}'"
+    end
+  end
+else
+  puts "Specify a file to hash"
+end
+

data/examples/simple.rb

@@ -0,0 +1,35 @@
+require 'nilsimsa'
+
+# Levenshtein implementation from
+# http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/Levenshtein_distance
+# Used under the GNU Free Documentation license
+class String
+  def levenshtein(other)
+    a, b = self.unpack('U*'), other.unpack('U*')
+    n, m = a.length, b.length
+    a, b, n, m = b, a, m, n if n > m
+    current = [*0..n]
+    1.upto(m) do |i|
+      previous, current = current, [i]+[0]*n
+      1.upto(n) do |j|
+        add, delete = previous[j]+1, current[j-1]+1
+        change = previous[j-1]
+        change += 1 if a[j-1] != b[i-1]
+        current[j] = [add, delete, change].min
+      end
+    end
+    current[n]
+  end
+end
+
+n1 = Nilsimsa::new
+text1 = "The quick brown fox"
+n1.update(text1)
+puts "'#{text1}':\n  #{n1.hexdigest}"
+
+n2 = Nilsimsa::new
+text2 = "The quick red fox"
+n2.update(text2)
+puts "'#{text2}':\n  #{n2.hexdigest}"
+
+puts "Distance: #{n1.hexdigest.levenshtein(n2.hexdigest)}"

data/ext/extconf.rb

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'mkmf'
+
+create_makefile( 'nilsimsa_native' )

data/ext/nilsimsa.c

@@ -0,0 +1,312 @@
+/*
+ * Source: Digest-Nilsimsa-0.06; _nilsimsa.c, nilsimsa.h
+ * Changes: 2005-04-14 Stephen Lewis <slewis@orcon.net.nz>
+ * - stripped superfluous code
+ * - some cleanups, reformatting
+ * - refactored to provide more appropriate ruby interface
+ *
+ * NOTE - I haven't gotten around to fixing the previous comment
+ * headers below this
+ */
+
+/*
+ * chad's modifications for perl xs - Digest::Nilsimsa
+ *
+ * main() - removed (too many warnings)
+ * accbuf - added, practically identical to accfile()
+ * dprint() - added  (prints debug msgs to debug.txt)
+ *
+ * $Id: _nilsimsa.c,v 1.1 2002/05/20 22:29:07 chad Exp $
+ */
+
+/***************************************************************************
+ * main.c  -  nilsimsa
+ * -------------------
+ * begin                : Fri Mar 16 01:41:08 EST 2001
+ * copyright            : (C) 2001-2002 by cmeclax
+ * email                : cmeclax@ixazon.dynip.com
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+/* NOTE - this should really use the support code from Digest, but
+ * would require a ruby source tree to build in that case ( the
+ * required headers don't seem to be generally installed :/ )
+ */
+
+#include <ruby.h>
+#include <assert.h>
+
+#define NSR_CODE_LEN 32
+#define RB_NSR_IVAR "cdata" /* not prefixing @ makes innaccessible from ruby */
+
+#define tran3(a,b,c,n) (((tran[((a)+(n))&255]^tran[(b)]*((n)+(n)+1))+tran[(c)^tran[n]])&255)
+
+
+struct nsrecord {
+  int acc[256];  /* counts each trigram's hash */
+  char code[NSR_CODE_LEN]; /* the nilsimsa code as a bit vector */
+
+  unsigned int chcount; /* number of characters processed so far */
+  int lastch[4]; /* last 4 characters processed */
+};
+
+void nsr_init( struct nsrecord *a );
+void nsr_digest( struct nsrecord *a );
+void nsr_update(struct nsrecord *, char *buf,unsigned long len);
+int nilsimsa(char *a,char *b);
+/*void nsr_free( struct nsrecord *r );*/
+static void filltran(void);
+static void fillpopcount(void);
+
+static struct nsrecord *get_nsr( VALUE obj );
+VALUE rbns_init(int argc, VALUE *argv, VALUE self);
+VALUE rbns_update(VALUE self, VALUE data);
+VALUE rbns_nilsimsa(VALUE self, VALUE other);
+VALUE rbns_digest(VALUE self);
+void Init_nilsimsa_native(void);
+
+unsigned char tran[256], popcount[256];
+
+/* formerly clear() */
+void
+nsr_init( struct nsrecord *a ) {
+  assert( a );
+
+  memset(a->acc,0,sizeof(a->acc));
+  memset(a->code,0,sizeof(a->code));
+
+  a->chcount = 0;
+  {
+    int i;
+    for (i=0; i<4; i++) {
+      a->lastch[i] = -1;
+    }
+  }
+}
+
+static void
+filltran(void) {
+  int i,j,k;
+  for (i=j=0;i<256;i++) {
+    j=(j*53+1)&255;
+    j+=j;
+    if (j>255) {
+      j-=255;
+    }
+    for (k=0;k<i;k++) {
+      if (j==tran[k]) {
+        j=(j+1)&255;
+        k=0;
+      }
+    }
+    tran[i]=j;
+  }
+}
+
+static void
+fillpopcount(void) {
+  int i,j;
+  memset(popcount,0,sizeof(popcount));
+
+  for (i=0;i<256;i++) {
+    for (j=0;j<8;j++) {
+      popcount[i]+=1&(i>>j);
+    }
+  }
+}
+
+
+/* formerly accfile() */
+void
+nsr_update(struct nsrecord *a, char *buf, unsigned long len) {
+  unsigned int idx;
+  int *lastch=a->lastch; // convenience
+
+  assert( a );
+
+  for(idx=0; idx<len; idx++) {
+    unsigned char ch = (unsigned char) buf[idx];
+    a->chcount++;
+    if (lastch[1]>=0)
+      a->acc[tran3(ch,lastch[0],lastch[1],0)]++;
+    if (lastch[2]>=0) {
+      a->acc[tran3(ch,lastch[0],lastch[2],1)]++;
+      a->acc[tran3(ch,lastch[1],lastch[2],2)]++;
+    }
+    if (lastch[3]>=0) {
+      a->acc[tran3(ch,lastch[0],lastch[3],3)]++;
+      a->acc[tran3(ch,lastch[1],lastch[3],4)]++;
+      a->acc[tran3(ch,lastch[2],lastch[3],5)]++;
+      a->acc[tran3(lastch[3],lastch[0],ch,6)]++;
+      a->acc[tran3(lastch[3],lastch[2],ch,7)]++;
+    }
+    lastch[3]=lastch[2];
+    lastch[2]=lastch[1];
+    lastch[1]=lastch[0];
+    lastch[0]=ch;
+  }
+}
+
+/* formerly makecode() */
+void
+nsr_digest(struct nsrecord *a) {
+  int i;
+  int total=0;     /* total number of trigrams counted */
+  int threshold=0; /* mean of all numbers in acc */
+
+  assert( a );
+
+  switch (a->chcount) {
+    case 0:
+    case 1:
+    case 2:
+      break;
+    case 3:
+      total = 1;
+      break;
+    case 4:
+      total = 4;
+      break;
+    default:
+      total = (8 * a->chcount) - 28;
+      break;
+  }
+
+  threshold=total/256; /* round down because criterion is >threshold */
+
+  memset(a->code,0,sizeof(a->code));
+  for (i=0;i<256;i++) {
+    a->code[i>>3]+=((a->acc[i]>threshold)<<(i&7));
+  }
+}
+
+/* NOTE - assumes both of length 32 */
+int
+nilsimsa(char *a,char *b) {
+  int i,bits=0;
+  assert( a );
+  assert( b );
+
+  for (i=0;i<NSR_CODE_LEN;i++) {
+    bits+=popcount[255&(a[i]^b[i])];
+  }
+
+  return 128-bits;
+}
+
+/*
+   void
+   nsr_free( struct nsrecord *r ) {
+   if (r) {
+   free( r );
+   }
+   }
+   */
+
+
+/*
+ *
+ * begin ruby wrapper functions 
+ * 
+ */
+
+static struct nsrecord *
+get_nsr( VALUE obj ) {
+  VALUE wrapped;
+  struct nsrecord *ret;
+
+  if (!RTEST( rb_funcall( obj, rb_intern( "kind_of?" ), 1, 
+          rb_eval_string("Nilsimsa")))) {
+    /* FIXME should raise exception */
+    return NULL;
+  }
+
+  wrapped = rb_iv_get( obj, RB_NSR_IVAR );
+  if (Qnil == wrapped) {
+    return NULL;
+  }
+
+  Data_Get_Struct( wrapped, struct nsrecord, ret );
+  return ret;
+}
+
+VALUE
+rbns_init(int argc, VALUE *argv, VALUE self) {
+  VALUE wrapped_nsr;
+  struct nsrecord *r;
+  wrapped_nsr = Data_Make_Struct( rb_cObject, struct nsrecord,
+      NULL, -1, r );
+  rb_iv_set( self, RB_NSR_IVAR, wrapped_nsr );
+  nsr_init( r );
+
+  return rb_funcall2( self, rb_intern( "old_initialize" ), argc, argv );
+}
+
+VALUE
+rbns_update(VALUE self, VALUE data) {
+  struct nsrecord *r;
+  char *chdata;
+  long chdata_len;
+  r = get_nsr( self );
+
+  Check_Type( data, T_STRING );
+  chdata = rb_str2cstr( data, &chdata_len );
+  nsr_update( r, chdata, chdata_len );
+  return data;
+}
+
+VALUE
+rbns_nilsimsa(VALUE self, VALUE other) {
+  long len;
+  char *d1;
+  char *d2;
+
+  d1 = rb_str2cstr( rb_funcall( self, rb_intern( "digest" ), 0 ), &len );
+  if (len < NSR_CODE_LEN) {
+    return Qnil;
+  }
+
+  Check_Type( other, T_STRING );
+  d2 = rb_str2cstr( other, &len );
+  if (len < NSR_CODE_LEN) {
+    return Qnil;
+  }
+
+  return INT2NUM( nilsimsa( d1, d2 ) );
+}
+
+VALUE
+rbns_digest(VALUE self) {
+  struct nsrecord *r=get_nsr( self );
+
+  nsr_digest( r );
+
+  /* reverse a newly created string of the digest */
+  return rb_funcall( rb_str_new( r->code, NSR_CODE_LEN ), 
+      rb_intern( "reverse"), 0 );
+}
+
+void
+Init_nilsimsa_native(void) {
+  VALUE rb_cNilsimsa;
+  /* initialize invariant data */
+  filltran();
+  fillpopcount();
+
+  /* this grafts itself over the top of an existing Nilsimsa class */
+  rb_cNilsimsa = rb_eval_string( "Nilsimsa" );
+  /* we'll call old_initialize from our new initialize */
+  rb_define_alias( rb_cNilsimsa, "old_initialize", "initialize" );
+  rb_define_method( rb_cNilsimsa, "initialize", rbns_init, -1 );
+  rb_define_method( rb_cNilsimsa, "update", rbns_update, 1 );
+  rb_define_method( rb_cNilsimsa, "nilsimsa", rbns_nilsimsa, 1 );
+  rb_define_method( rb_cNilsimsa, "digest", rbns_digest, 0 );
+}

data/gemspec.rb

@@ -0,0 +1,33 @@
+require 'rake'
+require 'mkmf'
+
+SPEC = Gem::Specification.new do |spec|
+  # Descriptive and source information for this gem.
+  spec.name = "nilsimsa"
+  spec.version = "1.0.1"
+  spec.summary = "Computes Nilsimsa values.  Nilsimsa is a distance based hash"
+  spec.author = "Jonathan Wilkins"
+  spec.email = "jwilkins[at]nospam[dot]bitland[dot]net"
+  spec.has_rdoc = true
+  spec.extra_rdoc_files = ["README"]
+  spec.require_path = "."
+  spec.autorequire = "nilsimsa.rb"
+
+  unfiltered_files = FileList['*', 'examples/*', 'bin/*', 'ext/*']
+  spec.files = unfiltered_files.delete_if do |filename|
+    filename.include?(".gem") || filename.include?("Makefile") ||
+    filename.include?(".so")  || filename.include?(".o")
+  end
+  spec.executables = ['nilsimsa']
+
+  # optional native component
+  if cc_command
+    spec.extensions << 'ext/extconf.rb'
+  end
+
+  puts "Building gem w/ "
+  spec.files.each do |f|
+    puts "- #{f}"
+  end
+
+end

data/nilsimsa.rb

@@ -0,0 +1,177 @@
+# Nilsimsa hash (build 20050414)
+# Ruby port (C) 2005 Martin Pirker
+# released under GNU GPL V2 license
+#
+# inspired by Digest::Nilsimsa-0.06 from Perl CPAN and
+# the original C nilsimsa-0.2.4 implementation by cmeclax
+# http://ixazon.dynip.com/~cmeclax/nilsimsa.html
+
+class Nilsimsa
+
+  TRAN =
+  "\x02\xD6\x9E\x6F\xF9\x1D\x04\xAB\xD0\x22\x16\x1F\xD8\x73\xA1\xAC" <<
+  "\x3B\x70\x62\x96\x1E\x6E\x8F\x39\x9D\x05\x14\x4A\xA6\xBE\xAE\x0E" <<
+  "\xCF\xB9\x9C\x9A\xC7\x68\x13\xE1\x2D\xA4\xEB\x51\x8D\x64\x6B\x50" <<
+  "\x23\x80\x03\x41\xEC\xBB\x71\xCC\x7A\x86\x7F\x98\xF2\x36\x5E\xEE" <<
+  "\x8E\xCE\x4F\xB8\x32\xB6\x5F\x59\xDC\x1B\x31\x4C\x7B\xF0\x63\x01" <<
+  "\x6C\xBA\x07\xE8\x12\x77\x49\x3C\xDA\x46\xFE\x2F\x79\x1C\x9B\x30" <<
+  "\xE3\x00\x06\x7E\x2E\x0F\x38\x33\x21\xAD\xA5\x54\xCA\xA7\x29\xFC" <<
+  "\x5A\x47\x69\x7D\xC5\x95\xB5\xF4\x0B\x90\xA3\x81\x6D\x25\x55\x35" <<
+  "\xF5\x75\x74\x0A\x26\xBF\x19\x5C\x1A\xC6\xFF\x99\x5D\x84\xAA\x66" <<
+  "\x3E\xAF\x78\xB3\x20\x43\xC1\xED\x24\xEA\xE6\x3F\x18\xF3\xA0\x42" <<
+  "\x57\x08\x53\x60\xC3\xC0\x83\x40\x82\xD7\x09\xBD\x44\x2A\x67\xA8" <<
+  "\x93\xE0\xC2\x56\x9F\xD9\xDD\x85\x15\xB4\x8A\x27\x28\x92\x76\xDE" <<
+  "\xEF\xF8\xB2\xB7\xC9\x3D\x45\x94\x4B\x11\x0D\x65\xD5\x34\x8B\x91" <<
+  "\x0C\xFA\x87\xE9\x7C\x5B\xB1\x4D\xE5\xD4\xCB\x10\xA2\x17\x89\xBC" <<
+  "\xDB\xB0\xE2\x97\x88\x52\xF7\x48\xD3\x61\x2C\x3A\x2B\xD1\x8C\xFB" <<
+  "\xF1\xCD\xE4\x6A\xE7\xA9\xFD\xC4\x37\xC8\xD2\xF6\xDF\x58\x72\x4E"
+
+  POPC =
+  "\x00\x01\x01\x02\x01\x02\x02\x03\x01\x02\x02\x03\x02\x03\x03\x04" <<
+  "\x01\x02\x02\x03\x02\x03\x03\x04\x02\x03\x03\x04\x03\x04\x04\x05" <<
+  "\x01\x02\x02\x03\x02\x03\x03\x04\x02\x03\x03\x04\x03\x04\x04\x05" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x01\x02\x02\x03\x02\x03\x03\x04\x02\x03\x03\x04\x03\x04\x04\x05" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x03\x04\x04\x05\x04\x05\x05\x06\x04\x05\x05\x06\x05\x06\x06\x07" <<
+  "\x01\x02\x02\x03\x02\x03\x03\x04\x02\x03\x03\x04\x03\x04\x04\x05" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x03\x04\x04\x05\x04\x05\x05\x06\x04\x05\x05\x06\x05\x06\x06\x07" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x03\x04\x04\x05\x04\x05\x05\x06\x04\x05\x05\x06\x05\x06\x06\x07" <<
+  "\x03\x04\x04\x05\x04\x05\x05\x06\x04\x05\x05\x06\x05\x06\x06\x07" <<
+  "\x04\x05\x05\x06\x05\x06\x06\x07\x05\x06\x06\x07\x06\x07\x07\x08"
+
+  def initialize(*data)
+    @threshold=0; @count=0
+    @acc =Array::new(256,0)
+    @lastch0=@lastch1=@lastch2=@lastch3= -1
+
+    data.each do |d| update(d) end  if data && (data.size>0)
+  end
+
+  def tran3(a,b,c,n)
+    (((TRAN[(a+n)&255]^TRAN[b]*(n+n+1))+TRAN[(c)^TRAN[n]])&255)
+  end
+
+  def update(data)
+    data.each_byte do |ch|
+      @count +=1
+      if @lastch1>-1 then
+        @acc[tran3(ch,@lastch0,@lastch1,0)] +=1
+      end
+      if @lastch2>-1 then
+        @acc[tran3(ch,@lastch0,@lastch2,1)] +=1
+        @acc[tran3(ch,@lastch1,@lastch2,2)] +=1
+      end
+      if @lastch3>-1 then
+        @acc[tran3(ch,@lastch0,@lastch3,3)] +=1
+        @acc[tran3(ch,@lastch1,@lastch3,4)] +=1
+        @acc[tran3(ch,@lastch2,@lastch3,5)] +=1
+        @acc[tran3(@lastch3,@lastch0,ch,6)] +=1
+        @acc[tran3(@lastch3,@lastch2,ch,7)] +=1
+      end
+      @lastch3=@lastch2
+      @lastch2=@lastch1
+      @lastch1=@lastch0
+      @lastch0=ch
+    end
+  end
+
+  def digest
+    @total=0;
+    case @count
+      when 0..2:
+      when 3   : @total +=1
+      when 4   : @total +=4
+      else     
+        @total +=(8*@count)-28    
+    end
+    @threshold=@total/256	
+
+    @code=String::new(
+           "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" <<
+           "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")
+    (0..255).each do |i|
+      @code[i>>3]+=( ((@acc[i]>@threshold)?(1):(0))<<(i&7) )
+    end
+
+    @code[0..31].reverse
+  end
+
+  def hexdigest
+    digest.unpack("H*")[0]
+  end
+
+  def to_s
+    hexdigest
+  end
+
+  def <<(whatever)
+    update(whatever)
+  end
+
+  def ==(otherdigest)
+    digest == otherdigest
+  end
+
+  def file(thisone)
+    File.open(thisone,"rb") do |f|
+       until f.eof? do update(f.read(10480)) end
+    end
+  end
+
+  def nilsimsa(otherdigest)
+    bits=0; myd=digest
+    (0..31).each do |i|
+      bits += POPC[255&myd[i]^otherdigest[i]]
+    end
+    (128-bits)
+  end
+  
+end
+
+def selftest
+  n1 = Nilsimsa::new;
+  n1.update("abcdefgh")
+  puts "abcdefgh:  #{n1.hexdigest=='14c8118000000000030800000004042004189020001308014088003280000078'}"
+  n2 = Nilsimsa::new("abcd","efgh")
+  puts "abcd efgh: #{n2.hexdigest=='14c8118000000000030800000004042004189020001308014088003280000078'}"
+  puts "digest:    #{n1 == n2.digest}"
+  n1.update("ijk")
+  puts "ijk:       #{n1.hexdigest=='14c811840010000c0328200108040630041890200217582d4098103280000078'}"
+  puts "nilsimsa:  #{n1.nilsimsa(n2.digest)==109}"
+  puts
+end
+
+if __FILE__ == $0 then
+  if ARGV.size>0 then
+    begin                               # load C core - if available
+      require 'nilsimsa_native'
+    rescue LoadError => e
+      # ignore lack of native module
+    end
+
+    ARGV.each do |filename|
+      if FileTest::exists?(filename) then
+        n = Nilsimsa::new
+        n.file(filename)
+        puts n.hexdigest+" #{filename}"
+      else
+        puts "error: can't find '#{filename}'"
+      end
+    end
+  else
+    puts 'Running selftest using native ruby version'
+    selftest
+    begin                               # load C core - if available
+      require './nilsimsa_native'
+      puts 'Running selftest using compiled nilsimsa in current dir'
+      selftest
+    rescue LoadError => e
+      puts "Couldnt run selftest with compiled nilsimsa"
+    end
+  end
+end

metadata

@@ -0,0 +1,55 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.4
+specification_version: 1
+name: nilsimsa
+version: !ruby/object:Gem::Version 
+  version: 1.0.1
+date: 2007-11-15 00:00:00 -08:00
+summary: Computes Nilsimsa values.  Nilsimsa is a distance based hash
+require_paths: 
+- .
+email: jwilkins[at]nospam[dot]bitland[dot]net
+homepage: 
+rubyforge_project: 
+description: 
+autorequire: nilsimsa.rb
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Jonathan Wilkins
+files: 
+- README
+- examples
+- ext
+- nilsimsa.rb
+- bin
+- gemspec.rb
+- examples/simple.rb
+- bin/nilsimsa
+- ext/nilsimsa.c
+- ext/extconf.rb
+test_files: []
+
+rdoc_options: []
+
+extra_rdoc_files: 
+- README
+executables: 
+- nilsimsa
+extensions: 
+- ext/extconf.rb
+requirements: []
+
+dependencies: []
+