Nilsimsa 1.0.0

data/extconf.rb

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'mkmf'
+
+create_makefile( 'nilsimsa_native' )

data/nilsimsa.c

@@ -0,0 +1,321 @@
+/*
+  Source: Digest-Nilsimsa-0.06; _nilsimsa.c, nilsimsa.h
+  Changes: 2005-04-14 Stephen Lewis <slewis@orcon.net.nz>
+  - stripped superfluous code
+  - some cleanups, reformatting
+  - refactored to provide more appropriate ruby interface
+
+  NOTE - I haven't gotten around to fixing the previous comment
+  headers below this
+*/
+
+/**
+ ** chad's modifications for perl xs - Digest::Nilsimsa
+ **
+ ** main() - removed (too many warnings)
+ ** accbuf - added, practically identical to accfile()
+ ** dprint() - added  (prints debug msgs to debug.txt)
+ **
+ ** $Id: _nilsimsa.c,v 1.1 2002/05/20 22:29:07 chad Exp $
+ **/
+/***************************************************************************
+                             main.c  -  nilsimsa
+                             -------------------
+    begin                : Fri Mar 16 01:41:08 EST 2001
+    copyright            : (C) 2001-2002 by cmeclax
+    email                : cmeclax@ixazon.dynip.com
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+
+
+/* NOTE - this should really use the support code from Digest, but
+   would require a ruby source tree to build in that case ( the
+   required headers don't seem to be generally installed :/ )
+ */
+
+#include <ruby.h>
+#include <assert.h>
+
+#define NSR_CODE_LEN 32
+#define RB_NSR_IVAR "cdata" /* not prefixing @ makes innaccessible from ruby */
+
+#define tran3(a,b,c,n) (((tran[((a)+(n))&255]^tran[(b)]*((n)+(n)+1))+tran[(c)^tran[n]])&255)
+
+
+struct nsrecord {
+    int acc[256];  /* counts each trigram's hash */
+    char code[NSR_CODE_LEN]; /* the nilsimsa code as a bit vector */
+
+    unsigned int chcount; /* number of characters processed so far */
+    int lastch[4]; /* last 4 characters processed */
+};
+
+
+void nsr_init( struct nsrecord *a );
+void nsr_digest( struct nsrecord *a );
+void nsr_update(struct nsrecord *, char *buf,unsigned long len);
+int nilsimsa(char *a,char *b);
+/*void nsr_free( struct nsrecord *r );*/
+static void filltran(void);
+static void fillpopcount(void);
+
+
+static struct nsrecord *get_nsr( VALUE obj );
+VALUE rbns_init(int argc, VALUE *argv, VALUE self);
+VALUE rbns_update(VALUE self, VALUE data);
+VALUE rbns_nilsimsa(VALUE self, VALUE other);
+VALUE rbns_digest(VALUE self);
+void Init_nilsimsa_native(void);
+
+
+
+unsigned char tran[256], popcount[256];
+
+
+
+
+
+/* formerly clear() */
+void
+nsr_init( struct nsrecord *a ) {
+    assert( a );
+
+    memset(a->acc,0,sizeof(a->acc));
+    memset(a->code,0,sizeof(a->code));
+
+    a->chcount = 0;
+    {
+	int i;
+	for (i=0; i<4; i++) {
+	    a->lastch[i] = -1;
+	}
+    }
+}
+
+static void
+filltran(void) {
+    int i,j,k;
+    for (i=j=0;i<256;i++) {
+	j=(j*53+1)&255;
+	j+=j;
+	if (j>255) {
+	    j-=255;
+	}
+	for (k=0;k<i;k++) {
+          if (j==tran[k]) {
+	      j=(j+1)&255;
+              k=0;
+	  }
+	}
+	tran[i]=j;
+    }
+}
+
+static void
+fillpopcount(void) {
+    int i,j;
+    memset(popcount,0,sizeof(popcount));
+
+    for (i=0;i<256;i++) {
+	for (j=0;j<8;j++) {
+	    popcount[i]+=1&(i>>j);
+	}
+    }
+}
+
+
+/* formerly accfile() */
+void
+nsr_update(struct nsrecord *a, char *buf, unsigned long len) {
+    unsigned int idx;
+    int *lastch=a->lastch; // convenience
+
+    assert( a );
+    
+    for(idx=0; idx<len; idx++) {
+	unsigned char ch = (unsigned char) buf[idx];
+	a->chcount++;
+	if (lastch[1]>=0)
+	    a->acc[tran3(ch,lastch[0],lastch[1],0)]++;
+	if (lastch[2]>=0) {
+	    a->acc[tran3(ch,lastch[0],lastch[2],1)]++;
+	    a->acc[tran3(ch,lastch[1],lastch[2],2)]++;
+	}
+	if (lastch[3]>=0) {
+	    a->acc[tran3(ch,lastch[0],lastch[3],3)]++;
+	    a->acc[tran3(ch,lastch[1],lastch[3],4)]++;
+	    a->acc[tran3(ch,lastch[2],lastch[3],5)]++;
+	    a->acc[tran3(lastch[3],lastch[0],ch,6)]++;
+	    a->acc[tran3(lastch[3],lastch[2],ch,7)]++;
+	}
+	lastch[3]=lastch[2];
+	lastch[2]=lastch[1];
+	lastch[1]=lastch[0];
+	lastch[0]=ch;
+    }
+}
+
+/* formerly makecode() */
+void
+nsr_digest(struct nsrecord *a) {
+    int i;
+    int total=0;     /* total number of trigrams counted */
+    int threshold=0; /* mean of all numbers in acc */
+
+    assert( a );
+
+    switch (a->chcount) {
+    case 0:
+    case 1:
+    case 2:
+	break;
+    case 3:
+	total = 1;
+	break;
+    case 4:
+	total = 4;
+	break;
+    default:
+	total = (8 * a->chcount) - 28;
+	break;
+    }
+
+    threshold=total/256; /* round down because criterion is >threshold */
+
+    memset(a->code,0,sizeof(a->code));
+    for (i=0;i<256;i++) {
+	a->code[i>>3]+=((a->acc[i]>threshold)<<(i&7));
+    }
+}
+
+/* NOTE - assumes both of length 32 */
+int
+nilsimsa(char *a,char *b) {
+    int i,bits=0;
+    assert( a );
+    assert( b );
+
+    for (i=0;i<NSR_CODE_LEN;i++) {
+	bits+=popcount[255&(a[i]^b[i])];
+    }
+
+    return 128-bits;
+}
+
+/*
+void
+nsr_free( struct nsrecord *r ) {
+    if (r) {
+	free( r );
+    }
+}
+*/
+
+
+/*
+ *
+ * begin ruby wrapper functions 
+ * 
+ */
+
+static struct nsrecord *
+get_nsr( VALUE obj ) {
+    VALUE wrapped;
+    struct nsrecord *ret;
+    
+    if (!RTEST( rb_funcall( obj, rb_intern( "kind_of?" ), 1, 
+			    rb_eval_string("Nilsimsa")))) {
+	/* FIXME should raise exception */
+	return NULL;
+    }
+
+    wrapped = rb_iv_get( obj, RB_NSR_IVAR );
+    if (Qnil == wrapped) {
+	return NULL;
+    }
+
+    Data_Get_Struct( wrapped, struct nsrecord, ret );
+    return ret;
+}
+
+VALUE
+rbns_init(int argc, VALUE *argv, VALUE self) {
+    VALUE wrapped_nsr;
+    struct nsrecord *r;
+    wrapped_nsr = Data_Make_Struct( rb_cObject, struct nsrecord,
+				    NULL, -1, r );
+    rb_iv_set( self, RB_NSR_IVAR, wrapped_nsr );
+    nsr_init( r );
+
+    return rb_funcall2( self, rb_intern( "old_initialize" ), argc, argv );
+}
+
+VALUE
+rbns_update(VALUE self, VALUE data) {
+    struct nsrecord *r;
+    char *chdata;
+    long chdata_len;
+    r = get_nsr( self );
+
+    Check_Type( data, T_STRING );
+    chdata = rb_str2cstr( data, &chdata_len );
+    nsr_update( r, chdata, chdata_len );
+    return data;
+}
+
+VALUE
+rbns_nilsimsa(VALUE self, VALUE other) {
+    long len;
+    char *d1;
+    char *d2;
+
+    d1 = rb_str2cstr( rb_funcall( self, rb_intern( "digest" ), 0 ), &len );
+    if (len < NSR_CODE_LEN) {
+	return Qnil;
+    }
+
+    Check_Type( other, T_STRING );
+    d2 = rb_str2cstr( other, &len );
+    if (len < NSR_CODE_LEN) {
+	return Qnil;
+    }
+    
+    return INT2NUM( nilsimsa( d1, d2 ) );
+}
+
+VALUE
+rbns_digest(VALUE self) {
+    struct nsrecord *r=get_nsr( self );
+
+    nsr_digest( r );
+
+    /* reverse a newly created string of the digest */
+    return rb_funcall( rb_str_new( r->code, NSR_CODE_LEN ), 
+		       rb_intern( "reverse"), 0 );
+}
+
+void
+Init_nilsimsa_native(void) {
+    VALUE rb_cNilsimsa;
+    /* initialize invariant data */
+    filltran();
+    fillpopcount();
+
+    /* this grafts itself over the top of an existing Nilsimsa class */
+    rb_cNilsimsa = rb_eval_string( "Nilsimsa" );
+    /* we'll call old_initialize from our new initialize */
+    rb_define_alias( rb_cNilsimsa, "old_initialize", "initialize" );
+    rb_define_method( rb_cNilsimsa, "initialize", rbns_init, -1 );
+    rb_define_method( rb_cNilsimsa, "update", rbns_update, 1 );
+    rb_define_method( rb_cNilsimsa, "nilsimsa", rbns_nilsimsa, 1 );
+    rb_define_method( rb_cNilsimsa, "digest", rbns_digest, 0 );
+}

data/nilsimsa.rb

@@ -0,0 +1,166 @@
+# Nilsimsa hash (build 20050414)
+# Ruby port (C) 2005 Martin Pirker
+# released under GNU GPL V2 license
+#
+# inspired by Digest::Nilsimsa-0.06 from Perl CPAN and
+# the original C nilsimsa-0.2.4 implementation by cmeclax
+# http://ixazon.dynip.com/~cmeclax/nilsimsa.html
+
+
+class Nilsimsa
+
+  TRAN =
+  "\x02\xD6\x9E\x6F\xF9\x1D\x04\xAB\xD0\x22\x16\x1F\xD8\x73\xA1\xAC" <<
+  "\x3B\x70\x62\x96\x1E\x6E\x8F\x39\x9D\x05\x14\x4A\xA6\xBE\xAE\x0E" <<
+  "\xCF\xB9\x9C\x9A\xC7\x68\x13\xE1\x2D\xA4\xEB\x51\x8D\x64\x6B\x50" <<
+  "\x23\x80\x03\x41\xEC\xBB\x71\xCC\x7A\x86\x7F\x98\xF2\x36\x5E\xEE" <<
+  "\x8E\xCE\x4F\xB8\x32\xB6\x5F\x59\xDC\x1B\x31\x4C\x7B\xF0\x63\x01" <<
+  "\x6C\xBA\x07\xE8\x12\x77\x49\x3C\xDA\x46\xFE\x2F\x79\x1C\x9B\x30" <<
+  "\xE3\x00\x06\x7E\x2E\x0F\x38\x33\x21\xAD\xA5\x54\xCA\xA7\x29\xFC" <<
+  "\x5A\x47\x69\x7D\xC5\x95\xB5\xF4\x0B\x90\xA3\x81\x6D\x25\x55\x35" <<
+  "\xF5\x75\x74\x0A\x26\xBF\x19\x5C\x1A\xC6\xFF\x99\x5D\x84\xAA\x66" <<
+  "\x3E\xAF\x78\xB3\x20\x43\xC1\xED\x24\xEA\xE6\x3F\x18\xF3\xA0\x42" <<
+  "\x57\x08\x53\x60\xC3\xC0\x83\x40\x82\xD7\x09\xBD\x44\x2A\x67\xA8" <<
+  "\x93\xE0\xC2\x56\x9F\xD9\xDD\x85\x15\xB4\x8A\x27\x28\x92\x76\xDE" <<
+  "\xEF\xF8\xB2\xB7\xC9\x3D\x45\x94\x4B\x11\x0D\x65\xD5\x34\x8B\x91" <<
+  "\x0C\xFA\x87\xE9\x7C\x5B\xB1\x4D\xE5\xD4\xCB\x10\xA2\x17\x89\xBC" <<
+  "\xDB\xB0\xE2\x97\x88\x52\xF7\x48\xD3\x61\x2C\x3A\x2B\xD1\x8C\xFB" <<
+  "\xF1\xCD\xE4\x6A\xE7\xA9\xFD\xC4\x37\xC8\xD2\xF6\xDF\x58\x72\x4E"
+
+  POPC =
+  "\x00\x01\x01\x02\x01\x02\x02\x03\x01\x02\x02\x03\x02\x03\x03\x04" <<
+  "\x01\x02\x02\x03\x02\x03\x03\x04\x02\x03\x03\x04\x03\x04\x04\x05" <<
+  "\x01\x02\x02\x03\x02\x03\x03\x04\x02\x03\x03\x04\x03\x04\x04\x05" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x01\x02\x02\x03\x02\x03\x03\x04\x02\x03\x03\x04\x03\x04\x04\x05" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x03\x04\x04\x05\x04\x05\x05\x06\x04\x05\x05\x06\x05\x06\x06\x07" <<
+  "\x01\x02\x02\x03\x02\x03\x03\x04\x02\x03\x03\x04\x03\x04\x04\x05" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x03\x04\x04\x05\x04\x05\x05\x06\x04\x05\x05\x06\x05\x06\x06\x07" <<
+  "\x02\x03\x03\x04\x03\x04\x04\x05\x03\x04\x04\x05\x04\x05\x05\x06" <<
+  "\x03\x04\x04\x05\x04\x05\x05\x06\x04\x05\x05\x06\x05\x06\x06\x07" <<
+  "\x03\x04\x04\x05\x04\x05\x05\x06\x04\x05\x05\x06\x05\x06\x06\x07" <<
+  "\x04\x05\x05\x06\x05\x06\x06\x07\x05\x06\x06\x07\x06\x07\x07\x08"
+
+  def initialize(*data)
+    @threshold=0; @count=0
+    @acc =Array::new(256,0)
+    @lastch0=@lastch1=@lastch2=@lastch3= -1
+    
+    data.each do |d| update(d) end  if data && (data.size>0)
+  end
+  
+    def tran3(a,b,c,n)
+      (((TRAN[(a+n)&255]^TRAN[b]*(n+n+1))+TRAN[(c)^TRAN[n]])&255)
+    end
+
+  def update(data)
+    data.each_byte do |ch|
+      @count +=1
+      if @lastch1>-1 then
+        @acc[tran3(ch,@lastch0,@lastch1,0)] +=1
+      end
+      if @lastch2>-1 then
+        @acc[tran3(ch,@lastch0,@lastch2,1)] +=1
+        @acc[tran3(ch,@lastch1,@lastch2,2)] +=1
+      end
+      if @lastch3>-1 then
+        @acc[tran3(ch,@lastch0,@lastch3,3)] +=1
+        @acc[tran3(ch,@lastch1,@lastch3,4)] +=1
+        @acc[tran3(ch,@lastch2,@lastch3,5)] +=1
+        @acc[tran3(@lastch3,@lastch0,ch,6)] +=1
+        @acc[tran3(@lastch3,@lastch2,ch,7)] +=1
+      end
+      @lastch3=@lastch2
+      @lastch2=@lastch1
+      @lastch1=@lastch0
+      @lastch0=ch
+    end
+  end
+
+  def digest
+    @total=0;
+    case @count
+      when 0..2:
+      when 3   : @total +=1
+      when 4   : @total +=4
+      else     
+        @total +=(8*@count)-28    
+    end
+    @threshold=@total/256	
+
+    @code=String::new("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" <<
+                      "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")
+    (0..255).each do |i|
+      @code[i>>3]+=( ((@acc[i]>@threshold)?(1):(0))<<(i&7) )
+    end
+
+    @code[0..31].reverse
+  end
+  
+  def hexdigest
+    digest.unpack("H*")[0]
+  end
+  
+  def to_s
+    hexdigest
+  end
+  
+  def <<(whatever)
+    update(whatever)
+  end
+  
+  def ==(otherdigest)
+    digest == otherdigest
+  end
+  
+  def file(thisone)
+    File.open(thisone,"rb") do |f|
+       until f.eof? do update(f.read(10480)) end
+    end
+  end
+  
+  def nilsimsa(otherdigest)
+    bits=0; myd=digest
+    (0..31).each do |i|
+      bits += POPC[255&myd[i]^otherdigest[i]]
+    end
+    (128-bits)
+  end
+  
+end
+
+begin                                    # load C core - if available
+    require 'nilsimsa_native'
+    puts 'using compiled nilsimsa'
+rescue LoadError => e
+    # ignore lack of native module
+    puts 'using interpreted nilsimsa'
+end
+
+if __FILE__ == $0 then
+  if ARGV.size>0 then
+    ARGV.each do |filename|
+      if FileTest::exists?(filename) then
+        n = Nilsimsa::new
+        n.file(filename)
+        puts n.hexdigest+" #{filename}"
+      else
+        puts "error: can't find '#{filename}'"
+      end
+    end
+  else
+    puts "running selftest..."
+    n1 = Nilsimsa::new;  n1.update("abcdefgh")
+    puts "abcdefgh:  #{n1.hexdigest=='14c8118000000000030800000004042004189020001308014088003280000078'}"
+    n2 = Nilsimsa::new(            "abcd","efgh")
+    puts "abcd efgh: #{n2.hexdigest=='14c8118000000000030800000004042004189020001308014088003280000078'}"
+    puts "digest:    #{n1 == n2.digest}"
+    n1.update("ijk")
+    puts "ijk:       #{n1.hexdigest=='14c811840010000c0328200108040630041890200217582d4098103280000078'}"    
+    puts "nilsimsa:  #{n1.nilsimsa(n2.digest)==109}"
+  end
+end

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.4
+specification_version: 1
+name: Nilsimsa
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+date: 2007-10-29 00:00:00 -07:00
+summary: Computes Nilsimsa values.  Nilsimsa is a distance based hash
+require_paths: 
+- .
+email: jwilkins[at]nospam[dot]bitland[dot]net
+homepage: 
+rubyforge_project: 
+description: 
+autorequire: nilsimsa.rb
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Jonathan Wilkins
+files: 
+- nilsimsa.c
+- nilsimsa.rb
+- extconf.rb
+test_files: []
+
+rdoc_options: []
+
+extra_rdoc_files: []
+
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: []
+