fuzz_ball 0.9.0

data/ext/fuzz_ball/duple_index/utstring.h

@@ -0,0 +1,148 @@
+/*
+Copyright (c) 2008-2011, Troy D. Hanson   http://uthash.sourceforge.net
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/* a dynamic string implementation using macros 
+ * see http://uthash.sourceforge.net/utstring
+ */
+#ifndef UTSTRING_H
+#define UTSTRING_H
+
+#define UTSTRING_VERSION 1.9.4
+
+#ifdef __GNUC__
+#define _UNUSED_ __attribute__ ((__unused__)) 
+#else
+#define _UNUSED_ 
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+#define oom() exit(-1)
+
+typedef struct {
+    char *d;
+    size_t n; /* allocd size */
+    size_t i; /* index of first unused byte */
+} UT_string;
+
+#define utstring_reserve(s,amt)                            \
+do {                                                       \
+  if (((s)->n - (s)->i) < (size_t)(amt)) {                 \
+     (s)->d = (char*)realloc((s)->d, (s)->n + amt);        \
+     if ((s)->d == NULL) oom();                            \
+     (s)->n += amt;                                        \
+  }                                                        \
+} while(0)
+
+#define utstring_init(s)                                   \
+do {                                                       \
+  (s)->n = 0; (s)->i = 0; (s)->d = NULL;                   \
+  utstring_reserve(s,100);                                 \
+  (s)->d[0] = '\0'; \
+} while(0)
+
+#define utstring_done(s)                                   \
+do {                                                       \
+  if ((s)->d != NULL) free((s)->d);                        \
+  (s)->n = 0;                                              \
+} while(0)
+
+#define utstring_free(s)                                   \
+do {                                                       \
+  utstring_done(s);                                        \
+  free(s);                                                 \
+} while(0)
+
+#define utstring_new(s)                                    \
+do {                                                       \
+   s = (UT_string*)calloc(sizeof(UT_string),1);            \
+   if (!s) oom();                                          \
+   utstring_init(s);                                       \
+} while(0)
+
+#define utstring_renew(s)                                  \
+do {                                                       \
+   if (s) {                                                \
+     utstring_clear(s);                                    \
+   } else {                                                \
+     utstring_new(s);                                      \
+   }                                                       \
+} while(0)
+
+#define utstring_clear(s)                                  \
+do {                                                       \
+  (s)->i = 0;                                              \
+  (s)->d[0] = '\0';                                        \
+} while(0)
+
+#define utstring_bincpy(s,b,l)                             \
+do {                                                       \
+  utstring_reserve(s,(l)+1);                               \
+  if (l) memcpy(&(s)->d[(s)->i], b, l);                    \
+  s->i += l;                                               \
+  s->d[s->i]='\0';                                         \
+} while(0)
+
+#define utstring_concat(dst,src)                           \
+do {                                                       \
+  utstring_reserve(dst,(src->i)+1);                        \
+  if (src->i) memcpy(&(dst)->d[(dst)->i], src->d, src->i); \
+  dst->i += src->i;                                        \
+  dst->d[dst->i]='\0';                                     \
+} while(0)
+
+#define utstring_len(s) ((unsigned)((s)->i))
+
+#define utstring_body(s) ((s)->d)
+
+_UNUSED_ static void utstring_printf_va(UT_string *s, const char *fmt, va_list ap) {
+   int n;
+   va_list cp;
+   while (1) {
+#ifdef _WIN32
+      cp = ap;
+#else
+      va_copy(cp, ap);
+#endif
+      n = vsnprintf (&s->d[s->i], s->n-s->i, fmt, cp);
+      va_end(cp);
+
+      if ((n > -1) && (n < (int)(s->n-s->i))) {
+        s->i += n;
+        return;
+      }
+
+      /* Else try again with more space. */
+      if (n > -1) utstring_reserve(s,n+1); /* exact */
+      else utstring_reserve(s,(s->n)*2);   /* 2x */
+   }
+}
+_UNUSED_ static void utstring_printf(UT_string *s, const char *fmt, ...) {
+   va_list ap;
+   va_start(ap,fmt);
+   utstring_printf_va(s,fmt,ap);
+   va_end(ap);
+}
+
+#endif /* UTSTRING_H */

data/ext/fuzz_ball/smith_waterman/SmithWaterman.c

@@ -0,0 +1,211 @@
+// Include the Ruby headers and goodies
+#include <ruby.h>
+#include <SmithWaterman.h>
+
+// The initialization method for this module
+void Init_smith_waterman() {
+	FuzzBall      = rb_define_module("FuzzBall");
+	SmithWaterman = rb_define_class_under(FuzzBall, "SmithWaterman", rb_cObject);
+
+	rb_define_method(SmithWaterman, "initialize", method_initialize, 2);
+	rb_define_attr(SmithWaterman, "alignment", 1, 0);
+	rb_define_attr(SmithWaterman, "score", 1, 0);
+}
+
+/* method_initialize
+ *
+ * We use the Smith-Waterman algorithm to align each candidate string with the
+ * needle string and to see how well the two fit. The Smith-Waterman algorithm
+ * is a dynamic programming algorith that keeps track of different alignments
+ * between two strings using a matrix. The best alignment is determined using
+ * a recursive search through the alignment matrix. For more information, see:
+ *
+ * http://en.wikipedia.org/wiki/Smith%E2%80%93Waterman_algorithm
+ */
+VALUE method_initialize(VALUE self, VALUE needle, VALUE candidate) {
+  int i, j, i_max, j_max;
+  int n_needle    = (int) RARRAY_LEN(needle);
+  int n_candidate = (int) RARRAY_LEN(candidate);
+  int *c_needle, *c_candidate;
+
+  double max_score;
+  double **mat;
+
+  VALUE alignment = rb_ary_new();
+
+  alloc_vars(&mat, &c_needle, &c_candidate, n_needle, n_candidate);
+
+  // Copy the needle / candidate strings from their ruby versions
+  // into plain old C-integer arrays.
+  for (i=0; i<n_needle; i++) {
+    c_needle[i] = NUM2INT( RARRAY_PTR(needle)[i] );
+  }
+
+  for (i=0; i<n_candidate; i++) {
+    c_candidate[i] = NUM2INT( RARRAY_PTR(candidate)[i] );
+  }
+
+  assign_cells(mat, c_needle, c_candidate, n_needle, n_candidate, &i_max, &j_max, &max_score);
+  recurse_optimal_path(mat, i_max, j_max, alignment);
+
+  rb_iv_set(self, "@alignment", alignment);
+  rb_iv_set(self, "@score", DBL2NUM(max_score));
+
+  free_vars(&mat, &c_needle, &c_candidate, n_needle, n_candidate);
+
+	return self;
+}
+
+// A few convenience methods for determining the max of 2, 3, and 4 doubles
+double max(double a, double b) {
+  return ( (a > b) ? a : b );
+}
+
+double max3(double a, double b, double c) {
+  return max(a, max(b, c));
+}
+
+double max4(double a, double b, double c, double d) {
+  return max(a, max3(b, c, d));
+}
+
+/* alloc_vars
+ *
+ * A simple function that allocates memory for the alignment matrix, as well
+ * as the arrays that store the characters of the needle and candidate strings
+ * given their lengths.
+ */
+void alloc_vars(
+    double ***mat,
+    int **needle,
+    int **candidate,
+    int n_needle,
+    int n_candidate
+) {
+  int i;
+
+  *mat = malloc(n_needle * sizeof(double *));
+  for (i=0; i<n_needle; i++) {
+    *((*mat) + i) = malloc(n_candidate * sizeof(double)); // ptr arithmetic FTW
+  }
+
+  *needle    = malloc(n_needle * sizeof(int));
+  *candidate = malloc(n_candidate * sizeof(int));
+}
+
+/* free_vars
+ *
+ * The method that frees memory associated with the alignment matrix, and the
+ * needle and candidate strings.
+ */
+void free_vars(
+    double ***mat,
+    int **needle,
+    int **candidate,
+    int n_needle,
+    int n_candidate
+) {
+
+  int i;
+
+  free(*needle);    *needle = NULL;
+  free(*candidate); *candidate = NULL;
+
+  for (i=0; i<n_needle; i++) {
+    free( (*mat)[i] );
+  }
+  free( *mat );
+  *mat = NULL;
+}
+
+/* assign_cells
+ *
+ * Called within the smith_waterman loop; this is the function that assigns
+ * each cell of the alignment matrix; the value of each cell represents the
+ * score of that given alignment, up to that point, taking into account any
+ * deletions, additions, etc.. that result in that alignment. For two strings
+ * of length m, n the alignment matrix is m x n large. As the values are
+ * assigned, we keep track of the cell with the higest score. At the end of
+ * the assignment, we start at this highest-scoring cell and recursively walk
+ * backwards through the cells, maximizing the score at each step. This
+ * becomes the highest scoring alignment
+ */
+void assign_cells(
+    double **mat,
+    int *needle,
+    int *candidate,
+    int n_needle,
+    int n_candidate,
+    int *i_max,
+    int *j_max,
+    double *max_score
+) {
+  int i, j;
+  double score, value;
+
+  for (i=0; i<n_needle; i++) {
+    for (j=0; j<n_candidate; j++) {
+      mat[i][j] = 0.0;
+    }
+  }
+
+  *max_score = -10000.0;
+  for (i=1; i<n_needle; i++) {
+    for (j=1; j<n_candidate; j++) {
+      if (needle[i-1] == candidate[j-1]) {
+        score = SCORE_MATCH;
+      } else {
+        score = SCORE_MISS;
+      }
+
+      value = max4(0.0, mat[i-1][j-1] + score, mat[i-1][j] + SCORE_DELETE, mat[i][j-1] + SCORE_INSERT);
+      mat[i][j] = value;
+
+      if (value > *max_score) {
+        *max_score = value;
+        *i_max     = i;
+        *j_max     = j;
+      }
+    }
+  }
+}
+
+/* recurse_optimal_path
+ *
+ * This method searches the alignment matrix from a starting point,
+ * and recursively walks backwards, maximizing the score at each step.
+ * This will result in an alignment of greatest score.
+ */
+void recurse_optimal_path(double **mat, int i, int j, VALUE alignment) {
+  int ii, jj;
+  double max_value;
+
+  // Push current position of the alignment into the array that stores it.
+  rb_ary_push(alignment, INT2NUM(i));
+  rb_ary_push(alignment, INT2NUM(j));
+
+  max_value = max3(mat[i-1][j-1], mat[i-1][j], mat[i][j-1]);
+
+  if (max_value == mat[i-1][j]) {
+    ii = i-1;
+    jj = j;
+  }
+
+  if (max_value == mat[i][j-1]) {
+    ii = i;
+    jj = j-1;
+  }
+
+  if (max_value == mat[i-1][j-1]) {
+    ii = i-1;
+    jj = j-1;
+  }
+
+  // The recursive loop. If we reach an edge (i.e., m(i,j) == 0), then stop
+  // the recursion. Otherwise, keep going!
+  if (mat[i][j] == 0) {
+    return;
+  } else {
+    return recurse_optimal_path(mat, ii, jj, alignment);
+  }
+}

data/ext/fuzz_ball/smith_waterman/SmithWaterman.h

@@ -0,0 +1,19 @@
+#define SCORE_INSERT -0.1
+#define SCORE_DELETE -1.0
+#define SCORE_MISS   -1.0
+#define SCORE_MATCH   2.0
+
+// Prototype some shit
+VALUE FuzzBall = Qnil;
+VALUE SmithWaterman = Qnil;
+void Init_smith_waterman();
+VALUE method_initialize(VALUE self, VALUE needle, VALUE candidate);
+
+void assign_cells(double **mat, int *needle, int *candidate, int n_needle, int n_candidate, int *i_max, int *j_max, double *max_score);
+void recurse_optimal_path(double **mat, int i, int j, VALUE alignment);
+void alloc_vars(double ***mat, int **needle, int **candidate, int n_needle, int n_candidate);
+void free_vars(double ***mat, int **needle, int **candidate, int n_needle, int n_candidate);
+
+double max(double a, double b);
+double max3(double a, double b, double c);
+double max4(double a, double b, double c, double d);

data/ext/fuzz_ball/smith_waterman/extconf.rb

@@ -0,0 +1,5 @@
+require 'mkmf'
+
+dir_config("fuzz_ball/smith_waterman")
+create_makefile("fuzz_ball/smith_waterman")
+

data/lib/fuzz_ball.rb

@@ -0,0 +1,6 @@
+require 'fuzz_ball/smith_waterman'
+require 'fuzz_ball/duple_index'
+
+module FuzzBall
+  autoload :Searcher, 'fuzz_ball/searcher'
+end

data/lib/fuzz_ball/searcher.rb

@@ -0,0 +1,92 @@
+module FuzzBall
+  class Searcher
+
+    attr_reader :files, :files_array, :options, :duple_index
+
+    def initialize(files, opts = {})
+      @options     = opts
+      @files       = files
+      @files_array = files.collect {|f| str2arr(f)}
+
+			index_duples!
+    end
+
+    def add( str )
+      str_arr = str2arr( str )
+
+      files       << str
+      files_array << str_arr
+
+      duple_index.add( files_array.count - 1, str_arr )
+
+      true
+    end
+
+    def search(needle, opts = {})
+
+      needle_ary = str2arr(needle)
+      results  = []
+
+      return results if (needle.length < 2)
+
+      decimate_strings!( needle_ary ).each do |candidate|
+        smith = SmithWaterman.new(needle_ary, candidate)
+
+        results << {
+          :alignment => smith.alignment,
+          :score     => (smith.score / candidate.length), # normalize by string length; this favors shorter strings even if a longer string has a higher smith score
+          :string    => candidate.pack("U*")
+        }
+      end
+
+      if (opts[:order] == :descending)
+        results.sort! {|a,b| b[:score] <=> a[:score]}
+      else
+        results.sort! {|a,b| a[:score] <=> b[:score]}
+      end
+
+      results = results.first(opts[:limit]) if opts[:limit].is_a?(Fixnum)
+
+      results
+    end
+
+    def inspect
+      %Q[<FuzzBall::Searcher n_files=#{files_array.count}>]
+    end
+
+    private
+
+		def index_duples!
+			@duple_index = DupleIndex.new
+			files_array.each_with_index do |str, index|
+				duple_index.add(index, str)
+			end
+		end
+
+    def decimate_strings!(needle)
+			matches_by_score = duple_index.match(needle)
+			max_score        = matches_by_score.keys.max
+			indices          = matches_by_score[max_score]
+
+			files_array.values_at(*indices)
+    end
+
+    def str2arr( str )
+      if options[:ignore]
+
+        regexp = options[:ignore].collect { |s|
+          Regexp.escape(s)
+        }.join("|")
+
+        regexp = Regexp.new("(#{regexp})")
+        str.gsub(regexp, "").unpack("U*")
+      else
+        str.unpack("U*")
+      end
+    end
+
+    def arr2str( arr )
+      arr.pack("U*")
+    end
+  end
+end