jwilkins-spamsum 0.1.1 → 0.1.2

data/ext/spamsum.c

@@ -29,6 +29,7 @@
 
 /* the output is a string of length 64 in base64 */
 #define SPAMSUM_LENGTH 64
+#define SPAMSUM_HEX_LENGTH 128
 
 #define MIN_BLOCKSIZE 3
 #define HASH_PRIME 0x01000193
@@ -226,6 +227,131 @@ again:
 	return ret;
 }
 
+/*
+  take a message of length 'length' and return a string representing a hash of that message,
+  prefixed by the selected blocksize
+*/
+char *spamsum_hex(const uchar *in, u32 length, u32 flags, u32 bsize)
+{
+	char *ret, *p;
+	u32 total_chars;
+	u32 h, h2, h3;
+	u32 j, n, i, k;
+	u32 block_size;
+	uchar ret2[SPAMSUM_HEX_LENGTH/2 + 1];
+
+	/* if we are ignoring email headers then skip past them now */
+	if (flags & FLAG_IGNORE_HEADERS) {
+		const uchar *s = strstr(in, "\n\n");
+		if (s) {
+			length -= (s+2 - in);
+			in = s+2;
+		}
+	}
+
+	if (flags & FLAG_IGNORE_WHITESPACE) {
+		/* count the non-ignored chars */
+		for (n=0, i=0; i<length; i++) {
+			if (isspace(in[i])) continue;
+			n++;
+		}
+		total_chars = n;
+	} else {
+		total_chars = length;
+	}
+
+	if (bsize == 0) {
+	/* guess a reasonable block size */
+		block_size = MIN_BLOCKSIZE;
+		while (block_size * SPAMSUM_HEX_LENGTH < total_chars) {
+			block_size = block_size * 2;
+		}
+	} else {
+		block_size = bsize;
+	}
+
+	ret = malloc(SPAMSUM_HEX_LENGTH + SPAMSUM_HEX_LENGTH/2 + 20);
+	if (!ret) return NULL;
+
+again:
+	/* the first part of the spamsum signature is the blocksize */
+	snprintf(ret, 12, "%u:", block_size);
+	p = ret + strlen(ret);
+
+	memset(p, 0, SPAMSUM_HEX_LENGTH+1);
+	memset(ret2, 0, sizeof(ret2));
+
+	k = j = 0;
+	h3 = h2 = HASH_INIT;
+	h = roll_reset();
+
+	for (i=0; i<length; i++) {
+		if ((flags & FLAG_IGNORE_WHITESPACE) &&
+		    isspace(in[i])) continue;
+
+		/*
+		   at each character we update the rolling hash and
+		   the normal hash. When the rolling hash hits the
+		   reset value then we emit the normal hash as a
+		   element of the signature and reset both hashes
+		*/
+		h = roll_hash(in[i]);
+		h2 = sum_hash(in[i], h2);
+		h3 = sum_hash(in[i], h3);
+
+		if (h % block_size == (block_size-1)) {
+			/* we have hit a reset point. We now emit a
+			   hash which is based on all chacaters in the
+			   piece of the message between the last reset
+			   point and this one */
+			snprintf(&p[j], 2, "%02x", h2);
+			if (j < SPAMSUM_HEX_LENGTH-1) {
+				/* we can have a problem with the tail
+				   overflowing. The easiest way to
+				   cope with this is to only reset the
+				   second hash if we have room for
+				   more characters in our
+				   signature. This has the effect of
+				   combining the last few pieces of
+				   the message into a single piece */
+				h2 = HASH_INIT;
+				j++;
+			}
+		}
+
+		/* this produces a second signature with a block size
+		   of block_size*2. By producing dual signatures in
+		   this way the effect of small changes in the message
+		   size near a block size boundary is greatly reduced. */
+		if (h % (block_size*2) == ((block_size*2)-1)) {
+			snprintf(&ret2[k], 2, "%02x", h3);
+			if (k < SPAMSUM_HEX_LENGTH/2-1) {
+				h3 = HASH_INIT;
+				k++;
+			}
+		}
+	}
+
+	/* if we have anything left then add it to the end. This
+	   ensures that the last part of the message is always
+	   considered */
+	if (h != 0) {
+		snprintf(&p[j], 2, "%02x", h2);
+		snprintf(&ret2[k], 2, "%02x", h3);
+	}
+
+	strcat(p+j, ":");
+	strcat(p+j, ret2);
+
+	/* our blocksize guess may have been way off - repeat if necessary */
+	if (bsize == 0 && block_size > MIN_BLOCKSIZE && j < SPAMSUM_HEX_LENGTH/2) {
+		block_size = block_size / 2;
+		goto again;
+	}
+
+	return ret;
+}
+
 
 /*
    we only accept a match if we have at least one common substring in

data/ext/spamsum.i

@@ -13,4 +13,5 @@
 
 int edit_distn(char *s1, int s1_len, char *s2, int s2_len);
 char *spamsum(char *str, unsigned int len, unsigned int flags=0, unsigned int bsize=0);
+char *spamsum_hex(char *str, unsigned int len, unsigned int flags=0, unsigned int bsize=0);
 unsigned int spamsum_match(char *s1, char *s2);

data/ext/spamsum_wrap.c

@@ -2087,6 +2087,61 @@ fail:
 }
 
 
+SWIGINTERN VALUE
+_wrap_spamsum_hex(int argc, VALUE *argv, VALUE self) {
+  char *arg1 = (char *) 0 ;
+  unsigned int arg2 ;
+  unsigned int arg3 = (unsigned int) 0 ;
+  unsigned int arg4 = (unsigned int) 0 ;
+  char *result = 0 ;
+  int res1 ;
+  char *buf1 = 0 ;
+  int alloc1 = 0 ;
+  unsigned int val2 ;
+  int ecode2 = 0 ;
+  unsigned int val3 ;
+  int ecode3 = 0 ;
+  unsigned int val4 ;
+  int ecode4 = 0 ;
+  VALUE vresult = Qnil;
+  
+  if ((argc < 2) || (argc > 4)) {
+    rb_raise(rb_eArgError, "wrong # of arguments(%d for 2)",argc); SWIG_fail;
+  }
+  res1 = SWIG_AsCharPtrAndSize(argv[0], &buf1, NULL, &alloc1);
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), Ruby_Format_TypeError( "", "char *","spamsum_hex", 1, argv[0] ));
+  }
+  arg1 = (char *)(buf1);
+  ecode2 = SWIG_AsVal_unsigned_SS_int(argv[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), Ruby_Format_TypeError( "", "unsigned int","spamsum_hex", 2, argv[1] ));
+  } 
+  arg2 = (unsigned int)(val2);
+  if (argc > 2) {
+    ecode3 = SWIG_AsVal_unsigned_SS_int(argv[2], &val3);
+    if (!SWIG_IsOK(ecode3)) {
+      SWIG_exception_fail(SWIG_ArgError(ecode3), Ruby_Format_TypeError( "", "unsigned int","spamsum_hex", 3, argv[2] ));
+    } 
+    arg3 = (unsigned int)(val3);
+  }
+  if (argc > 3) {
+    ecode4 = SWIG_AsVal_unsigned_SS_int(argv[3], &val4);
+    if (!SWIG_IsOK(ecode4)) {
+      SWIG_exception_fail(SWIG_ArgError(ecode4), Ruby_Format_TypeError( "", "unsigned int","spamsum_hex", 4, argv[3] ));
+    } 
+    arg4 = (unsigned int)(val4);
+  }
+  result = (char *)spamsum_hex(arg1,arg2,arg3,arg4);
+  vresult = SWIG_FromCharPtr((const char *)result);
+  if (alloc1 == SWIG_NEWOBJ) free((char*)buf1);
+  return vresult;
+fail:
+  if (alloc1 == SWIG_NEWOBJ) free((char*)buf1);
+  return Qnil;
+}
+
+
 SWIGINTERN VALUE
 _wrap_spamsum_match(int argc, VALUE *argv, VALUE self) {
   char *arg1 = (char *) 0 ;
@@ -2400,6 +2455,7 @@ SWIGEXPORT void Init_spamsum_swig(void) {
   SWIG_RubyInitializeTrackings();
   rb_define_module_function(mSpamsum_swig, "edit_distn", _wrap_edit_distn, -1);
   rb_define_module_function(mSpamsum_swig, "spamsum", _wrap_spamsum, -1);
+  rb_define_module_function(mSpamsum_swig, "spamsum_hex", _wrap_spamsum_hex, -1);
   rb_define_module_function(mSpamsum_swig, "spamsum_match", _wrap_spamsum_match, -1);
 }
 

data/spamsum.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{spamsum}
-  s.version = "0.1.1"
+  s.version = "0.1.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jonathan Wilkins"]

data/test.rb

@@ -3,31 +3,36 @@ s1 = "And she turned to me and took me by the hand and said, I've lost control a
 s2 = "And she screamed out kicking on her side and said, I've lost control again."
 s3 = "Control"
 
-puts "s1: #{s1}"
-puts "s2: #{s2}"
-puts "s3: #{s3}"
-puts "Spamsum.distance(s1, s2): #{Spamsum.distance(s1, s2)}"
-puts "Spamsum.distance(s1, s3): #{Spamsum.distance(s1, s3)}"
-puts "-"*40
+# XXX: seems to be a min length issue for compares, replace with other dist func?
+b1 = "\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb"
+b2 = "\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc"
+b3 = "\x00\xaa\xbb\xcc\xdd\x66\x77\x88\x99\x00\xaa"
 
-(1..3).each { |x|
-  eval("s = s#{x}")
 
-  eval("$sum#{x} = Spamsum.sum(s, s.length)")
-  puts "Spamsum.sum(s#{x}): #{eval("$sum#{x}")}"
-  puts "Spamsum.match(s1, s#{x}): #{Spamsum.match($sum1, eval("$sum#{x}"))}"
-}
+%w(s b).each { |t|
+  puts "-"*40
+  (1..3).each { |x|
+    puts "#{t}#{x}: #{eval("#{t}#{x}")}" if t == 's'
+    puts "Spamsum.distance(#{t}1, #{t}#{x}): #{Spamsum.distance(eval("#{t}1"), eval("#{t}#{x}"))}"
+    eval("s = #{t}#{x}")
+    puts "-"*20
 
-puts "-"*40
+    eval("$#{t}sum#{x} = Spamsum.sum(s)")
+    puts "Spamsum.sum(#{t}#{x}): #{eval("$#{t}sum#{x}")}"
+    puts "Spamsum.match(#{t}1, #{t}#{x}): #{Spamsum.match(eval("$#{t}sum1"), eval("$#{t}sum#{x}"))}"
+    puts "-"*20
 
-# XXX: seems to be a min length issue for compares, replace with other dist func?
-b1 = "\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb"
-b2 = "\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc"
-b3 = "\x00\xaa\xbb\xcc\xdd\x66\x77\x88\x99\x00\xaa"
-(1..3).each { |x|
-  eval("s = b#{x}")
+    eval("$#{t}hsum#{x} = Spamsum.sum_hex(s)")
+    puts "Spamsum.match(#{t}1, #{t}#{x}): #{Spamsum.match(eval("$#{t}hsum1"), eval("$#{t}hsum#{x}"))}"
+  }
+}
 
-  eval("$bsum#{x} = Spamsum.sum(s, s.length)")
-  puts "Spamsum.sum(b#{x}): #{eval("$bsum#{x}")}"
-  puts "Spamsum.match(b1, b#{x}): #{Spamsum.match($bsum1, eval("$bsum#{x}"))}"
+puts "-"*40
+%w(s b).each { |t|
+  (1..3).each { |x|
+    puts "Spamsum.sum_hex(#{t}#{x}): #{eval("$#{t}hsum#{x}")}"
+  }
 }
+
+puts "Spamsum.sum_hex(s1*1000): #{Spamsum.sum_hex(s1*1000)}"
+puts "Spamsum.sum_hex(s1*1000): #{Spamsum.sum_hex(open('random.bin').read)}"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: jwilkins-spamsum
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - Jonathan Wilkins