cassiopee 0.1.0 → 0.1.1

data/Changelog

@@ -1 +1,4 @@
-v0.1.0 : First version
+v0.1.1 : fix #1, add filter option
+  08/09/11 Olivier Sallou
+v0.1.0 : First version
+  Olivier Sallou

data/README

@@ -1,2 +1,2 @@
 Search an exact or approximate word (hamming or edit distance) in a string.
-Support cache mechanism and REST interface.
+Support index cache with incremental update for later searches

data/bin/cassie.rb

@@ -0,0 +1,139 @@
+#!/usr/bin/env ruby
+require File.join(File.dirname(__FILE__), '../lib/cassiopee')
+require 'optparse'
+require 'logger'
+ 
+options = {}
+ 
+optparse = OptionParser.new do|opts|
+   # Set a banner, displayed at the top
+   # of the help screen.
+   opts.banner = "Usage: cassie.rb [options]"
+ 
+   options[:verbose] = false
+   opts.on( '-v', '--verbose', 'Output more information' ) do
+     options[:verbose] = true
+   end
+
+   options[:filter] = nil
+   opts.on( '-f', '--filter FILTER', 'Filter matches between min and max positions ex. 100-150' ) do |filter|
+	options[:filter] = filter
+   end
+
+   options[:file] = nil
+   opts.on( '-i', '--index FILE', 'File to index' ) do |file|
+        options[:file] = file 
+   end
+
+   options[:fpattern] = nil
+   opts.on( '--fpattern FILE', 'File with pattern' ) do |file|
+        options[:fpattern] = file
+   end
+
+   options[:pattern] = nil
+   opts.on( '-p', '--pattern PATTERN', 'Search pattern' ) do |file|
+        options[:pattern] = file
+   end
+
+   options[:store] = nil
+   opts.on( '-s', '--store FILE', 'Store index to file' ) do |file|
+        options[:store] = file
+   end
+
+   options[:name] = nil
+   opts.on( '-n', '--name NAME', 'name of index, default [crawler]' ) do |name|
+        options[:name] = name
+   end
+
+   options[:exact] = false
+   opts.on( '-x', '--exact', 'Do exact search (default)' ) do
+        options[:exact] = true
+   end
+
+   options[:error] = 0
+   opts.on( '-m', '--hamming ERROR', 'Maximum number of error to search with Hamming distance' ) do |error|
+        options[:error] = error
+   end
+
+   opts.on( '-e', '--edit ERROR', 'Maximum number of error to search with edit(levenshtein) distance' ) do |error|
+        options[:error] = error * (-1)
+   end
+
+
+   opts.on( '-h', '--help', 'Display this screen' ) do
+     puts opts
+     exit
+   end
+
+end
+
+  optparse.parse!
+
+  if(options[:file]==nil)
+    puts "Error, input file is missing, use -h option for usage"
+    exit
+  elif(options[:verbose])
+    puts "Input sequence: " << options[:file].to_s 
+  end
+
+  if(options[:fpattern]==nil && options[:pattern]==nil)
+    puts "Error, pattern is missing, use -h option for usage"
+    exit
+  end
+
+
+ if(options[:error]==0)
+   options[:exact] = true
+ end
+
+
+crawler = Cassiopee::Crawler.new
+crawler.setLogLevel(Logger::INFO)
+if(options[:store])
+crawler.use_store = true
+end
+if(options[:name]!=nil)
+crawler.file_suffix = options[:name]
+end
+if(options[:filter]!=nil)
+positions = options[:filter].split('-')
+crawler.filter_position(positions[0],positions[1])
+end
+
+# String to index
+crawler.indexFile(options[:file])
+
+matches = nil
+
+if(options[:fpattern]==nil)
+	pattern = options[:pattern]
+else
+	pattern = ''
+	file = File.new(options[:fpattern], "r")
+        while (line = file.gets)
+        	input = line.downcase.chomp
+		pattern << input
+        end
+	file.close
+	if(pattern.length==0)
+		puts "Error pattern file is empty"
+		exit 
+        end
+end
+
+  if(options[:verbose])
+    puts "Search pattern " << pattern
+  end
+
+if(options[:exact])
+        puts "Search exact" unless !options[:verbose]
+	matches = crawler.searchExact(pattern)
+else
+        puts "Search approximate" unless !options[:verbose]
+	matches = crawler.searchApproximate(pattern,options[:errors])
+end
+
+# Go through matches
+while((match = crawler.next())!=nil)
+  puts "Match: " << match.inspect
+end

data/demo.rb

@@ -23,7 +23,7 @@ puts "Hamming: " << test.computeHamming("my strigg",1).to_s
 puts "Levenshtein: " << test.computeLevenshtein("mystriigg",3).to_s
 
 # Approcimate search, edit distance = 1
-crawler.searchApproximate("llu",-1)
+crawler.searchApproximate("llu",-2)
 
 # Go through matches
 while((match = crawler.next())!=nil)

data/lib/cassiopee.rb

@@ -1,6 +1,8 @@
 require 'digest/md5'
 require 'logger'
 require 'zlib'
+require 'rubygems'
+require 'text'
 
 module Cassiopee
 
@@ -33,6 +35,7 @@ module Cassiopee
     	end
     	return nberr
     end
+
     
     # Calculate the edit distance between string and pattern
     # Extend a String
@@ -40,41 +43,14 @@ module Cassiopee
     
     def computeLevenshtein(pattern,edit)
     	pattern = pattern.downcase
-    	matrix= Array.new(2)
-    	matrix[0] = Array.new(pattern.length+1)
-    	matrix[1] = Array.new(pattern.length+1)
-    	(0..(pattern.length)).each do |i|
-    		matrix[0][i]=i
-    		matrix[1][i]=i
-    	end
-    	c=0
-    	p=1
-    	(1..(self.length)).each do |i|
-    		c = i.modulo(2)
-    		p = (i+1).modulo(2)
-    		matrix[c][0] = i
-        	(1..(pattern.length)).each do |j|
-        		# Bellman's principle of optimality
-        		weight = 0
-    			if(pattern[i-1] != self[j-1])
-    					weight = 1
-    			end
-    			weight = matrix[p][j-1] + weight
-    			if(weight > matrix[p][j] +1)
-    				weight = matrix[p][j] +1
-    			end
-    			if(weight > matrix[c][j-1] +1)
-    				weight = matrix[c][j-1] +1
-    			end
-    			matrix[c][j] = weight
-    		end	
-    	end
-    	p = c
-    	c = (c + 1).modulo(2)
-    	if(matrix[p][pattern.length]>edit)
+    	
+    	distance = Text::Levenshtein.distance(self, pattern)
+    	
+   	
+    	if(distance>edit)
     		return -1
     	end
-    	return matrix[p][pattern.length]
+    	return distance
     	
     end
  
@@ -91,6 +67,9 @@ module Cassiopee
     # Use persistent suffix file ?
     attr_accessor  :use_store
     
+    @min_position = 0
+    @max_position = 0
+    
     FILE_SUFFIX_EXT = ".sfx"
     FILE_SUFFIX_POS = ".sfp"
     
@@ -119,9 +98,11 @@ module Cassiopee
         end
         
         # Clear suffixes in memory
+        # If using use_store, clear the store too
         
         def clear
         	@suffixes = Hash.new
+        	File.delete(@file_suffix+FILE_SUFFIX_POS) unless !File.exists?(@file_suffix+FILE_SUFFIX_POS)
         end
     
     	# Set Logger level
@@ -131,6 +112,7 @@ module Cassiopee
         end
         
         # Index an input file
+        # Clear existing indexes
         
         def indexFile(f)
          # Parse file, map letters to reduced alphabet
@@ -138,18 +120,34 @@ module Cassiopee
          # Take all suffix, order by length, link to position map on other file
          # Store md5 for easier compare? + 20 bytes per suffix
             @sequence = readSequence(f)
-            
+            clear()
+            @min_position = 0
+    		@max_position = 0
         end
         
         # Index an input string
+        # Clear existing indexes
         
         def indexString(s)
             @sequence = s
             File.open(@file_suffix+FILE_SUFFIX_EXT, 'w') do |data|
             	data.puts(@sequence)
             end
-            
-            
+            clear()
+            @min_position = 0
+    		@max_position = 0
+        end
+        
+        # Filter matches to be between min and max start position
+        # If not using use_store, search speed is improved but existing indexes are cleared
+        # If max=0, then max is string length
+        
+        def filter_position(min,max)
+            if(!use_store)
+        		clear()
+        	end
+        	@min_position = min
+        	@max_position = max
         end
         
         # Search exact match
@@ -208,7 +206,8 @@ module Cassiopee
         			next
         		end
         		if (md5val == matchmd5)
-                    match = Array[md5val, 0, posArray]
+        			filteredPosArray = filter(posArray)
+                    match = Array[md5val, 0, filteredPosArray]
 		    		$log.debug "Match: " << match.inspect
 		    		@matches << match
 		    	else
@@ -222,7 +221,8 @@ module Cassiopee
 		    			  errors = seq.computeLevenshtein(s,edit)
 		    			end
 		    			if(errors>=0)
-		    			    match = Array[md5val, errors, posArray]
+		    				filteredPosArray = filter(posArray)
+		    			    match = Array[md5val, errors, filteredPosArray]
 		    				$log.debug "Match: " << match.inspect
 		    				@matches << match
 		    			end
@@ -234,6 +234,29 @@ module Cassiopee
         	return @matches 
         end
         
+        # Filter the array of positions with defined position filter
+        
+        def filter(posArray)
+        	$log.debug("filter the position with " << @min_position.to_s << " and " << @max_position.to_s)
+        	if(@min_position==0 && @max_position==0)
+        		return posArray
+        	end
+        	filteredArray = Array.new
+        	i = 0
+        	posArray.each do |pos|
+        		if(i==0)
+        			# First elt of array is match length
+        			filteredArray << pos
+        		end
+        		if(i>0 && pos>=@min_position && pos<=@max_position)
+        			filteredArray << pos
+        		end
+        		i +=1
+        	end
+        	return filteredArray
+        end
+        
+        
         # Extract un suffix from suffix file based on md5 match
         
         def extractSuffix(start,len)
@@ -283,6 +306,17 @@ module Cassiopee
             	maxlen = @sequence.length
             end
             
+            if(!use_store)
+            	minpos = @min_position
+            	if(@max_position==0)
+            		maxpos = @sequence.length
+            	else
+            		maxpos = @max_position
+            	end
+            else
+            	minpos = 0
+            	maxpos = @sequence.length - minlen
+            end
             
             	suffixlen = nil
             	$log.info('Start indexing')
@@ -322,11 +356,15 @@ module Cassiopee
                 		next
                 	end
                 	changed = true
-               		 (0..(s.length-maxlen)).each do |j|
+               		 (minpos..(maxpos)).each do |j|
+               		 	# if position+length longer than sequence length, skip it
+               		 	if(j+i>=@sequence.length)
+               		 		next
+               		 	end
                     	@suffix = s[j,i]
                     	@suffixmd5 = Digest::MD5.hexdigest(@suffix)
                     	@position = j
-                    	#$log.debug("add "+@suffix+" at pos "+@position.to_s)
+                    	$log.debug("add "+@suffix+" at pos "+@position.to_s)
                     	nbSuffix += addSuffix(@suffixmd5, @position,i)
                     end
                     $log.debug("Nb suffix found: " << nbSuffix.to_s << ' for length ' << i.to_s)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: cassiopee
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 25
   prerelease: false
   segments: 
   - 0
   - 1
-  - 0
-  version: 0.1.0
+  - 1
+  version: 0.1.1
 platform: ruby
 authors: 
 - Olivier Sallou
@@ -17,8 +17,23 @@ cert_chain: []
 
 date: 2011-09-04 00:00:00 +02:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: text
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 0
+        - 2
+        - 0
+        version: 0.2.0
+  type: :runtime
+  version_requirements: *id001
 description: Cassiopee index one String and provide methods to search exact match or approximate matches with Hammming and/or edit distance.
 email: olivier.sallou@gmail.com
 executables: []
@@ -33,6 +48,7 @@ files:
 - LICENSE
 - demo.rb
 - lib/cassiopee.rb
+- bin/cassie.rb
 - tests/test-suite.rb
 has_rdoc: true
 homepage: https://github.com/osallou/cassiopee