cassiopee 0.1.4 → 0.1.5

data/Changelog

@@ -1,3 +1,4 @@
+v0.1.5 : add CrawlerCache for basic cache management (previous result only), several fixes
 v0.1.4 : fix 0.1.3 error on index load, add filter_position management in mt
 v0.1.3 : 09/11 Olivier Sallou
  add CrawlerMT in cassiopee-mt for multi thread support to speed up the search

data/lib/cassiopee-mt.rb

@@ -34,6 +34,7 @@ module CassiopeeMt
       crawler.setLogLevel($log.level)
       crawler.file_suffix = @file_suffix
       crawler.loadIndex()
+	  crawler.method = method
       #crawler.file_suffix = @file_suffix+"."+threadId.to_s
     end
 
@@ -55,16 +56,16 @@ module CassiopeeMt
         end
         nb = len.div(maxthread)
         (1..maxthread).each do |i|
-          crawler = Crawler.new
-	  setParams(crawler,i)
-          curmax = min + nb
-          if(i==maxthread)
-             curmax = max
-          end
-          crawler.filter_position(min,curmax)
-	  $log.debug("Start new Thread between " << min.to_s << " and " << curmax.to_s)
-          @th[i-1] = Thread.new{ Thread.current["matches"] = crawler.searchExact(pattern) }
-          min = curmax + 1
+			crawler = Crawler.new
+			setParams(crawler,i)
+			curmax = min + nb
+			if(i==maxthread)
+				curmax = max
+			end
+			crawler.filter_position(min,curmax)
+			$log.debug("Start new Thread between " << min.to_s << " and " << curmax.to_s)
+			@th[i-1] = Thread.new{ Thread.current["matches"] = crawler.searchExact(pattern) }
+			min = curmax + 1
         end
         @th.each {|t| t.join; t["matches"].each { |m| @matches << m }}
         return @matches

data/lib/cassiopee.rb

@@ -34,7 +34,6 @@ module Cassiopee
     # Compute Hamming distance but using a mapping matrix of alphabet ambiguity
     
     def computeHammingAmbiguous(pattern,hamming,ambiguous)
-    	pattern = pattern.downcase
     	nberr = 0
     	(0..(self.length-1)).each do |c|
     		if(!isAmbiguousEqual(pattern[c],self[c],ambiguous))
@@ -52,7 +51,6 @@ module Cassiopee
     # Return -1 if max is reached
     
     def computeHamming(pattern,hamming)
-    	pattern = pattern.downcase
     	nberr = 0
     	(0..(self.length-1)).each do |c|
     		if(pattern[c] != self[c])
@@ -72,11 +70,9 @@ module Cassiopee
     # Return -1 if max is reached
     
     def computeLevenshtein(pattern,edit)
-    	pattern = pattern.downcase
     	
     	distance = Text::Levenshtein.distance(self, pattern)
     	
-   	
     	if(distance>edit)
     		return -1
     	end
@@ -91,44 +87,42 @@ module Cassiopee
 	
     def computeLevenshteinAmbiguous(pattern, edit, ambiguous)
 	
-	  pattern = pattern.downcase
-      encoding = defined?(Encoding) ? self.encoding.to_s : $KCODE
+		encoding = defined?(Encoding) ? self.encoding.to_s : $KCODE
 
-    if Text.encoding_of(self) =~ /^U/i
-      unpack_rule = 'U*'
-    else
-      unpack_rule = 'C*'
-    end
+		if Text.encoding_of(self) =~ /^U/i
+			unpack_rule = 'U*'
+		else
+			unpack_rule = 'C*'
+		end
 
-    s = self.unpack(unpack_rule)
-    t = pattern.unpack(unpack_rule)
-    n = s.length
-    m = t.length
-    return m if (0 == n)
-    return n if (0 == m)
+		s = self.unpack(unpack_rule)
+		t = pattern.unpack(unpack_rule)
+		n = s.length
+		m = t.length
+		return m if (0 == n)
+		return n if (0 == m)
 
-    d = (0..m).to_a
-    x = nil
+		d = (0..m).to_a
+		x = nil
 
-    (0...n).each do |i|
-      e = i+1
-      (0...m).each do |j|
-        cost = (isAmbiguousEqual(s[i],t[j],ambiguous)) ? 0 : 1
-        x = [
-          d[j+1] + 1, # insertion
-          e + 1,      # deletion
-          d[j] + cost # substitution
-        ].min
-        d[j] = e
-        e = x
-      end
-      d[m] = x
-    end
-    if(x>edit)
-    return -1
-	end
-	
-	return x
+		(0...n).each do |i|
+			e = i+1
+			(0...m).each do |j|
+				cost = (isAmbiguousEqual(s[i],t[j],ambiguous)) ? 0 : 1
+				x = [
+					d[j+1] + 1, # insertion
+					e + 1,      # deletion
+					d[j] + cost # substitution
+				].min
+				d[j] = e
+				e = x
+			end
+			d[m] = x
+		end
+		if(x>edit)
+			return -1
+		end
+		return x
   end
   
   
@@ -150,6 +144,106 @@ module Cassiopee
 	end
   end
  
+	# Class maning cache of results
+ 
+	class CrawlerCache
+	
+		FILE_CACHE_EXT = ".sfc"
+
+		# Suffix files name/path
+		attr_accessor  :file_suffix
+		
+		# search exact: 0
+		# hamming     : 1
+		# edit        : 2
+		attr_accessor  :method
+	
+		# filter
+		attr_accessor  :min_position
+		attr_accessor  :max_position
+		
+		# max errors
+		attr_accessor	:errors
+		
+		attr_accessor	:cache
+	
+		$log = Logger.new(STDOUT)
+		$log.level = Logger::INFO		
+	
+		def setLogger(userlogger)
+			$log = userlogger
+		end
+	
+		def initialize
+			@file_suffix = "crawler"
+        end
+	
+		# Loads cache from file
+		def loadCache
+            	return Array.new unless File.exists?(@file_suffix+FILE_CACHE_EXT)
+                begin
+                  file = Zlib::GzipReader.open(@file_suffix+FILE_CACHE_EXT)
+                rescue Zlib::GzipFile::Error
+                  file = File.open(@file_suffix+FILE_CACHE_EXT, 'r')
+                ensure
+                    obj =  Marshal.load file.read
+                    file.close
+					if(method!=obj.method || min_position<obj.min_position || max_position>obj.max_position || errors>obj.errors)
+						return Array.new
+					end
+                    return filterCache(obj)
+                end			
+		end
+	
+		# Save self to cache, with cache object set from obj
+		def saveCache(obj)
+			self.cache = obj
+			marshal_dump = Marshal.dump(self)
+			sfxpos = File.new(@file_suffix+FILE_CACHE_EXT,'w')
+			sfxpos = Zlib::GzipWriter.new(sfxpos)
+			sfxpos.write marshal_dump
+			sfxpos.close
+		end
+		
+		def clearCache
+			File.delete(@file_suffix+FILE_CACHE_EXT) unless !File.exists?(@file_suffix+FILE_CACHE_EXT)
+		end
+		
+		private
+		
+		# filter cache according to settings
+		# obj: cache object
+		def filterCache(cacheobject)
+			
+			realmatches = Array.new
+			if(cacheobject==nil)
+				return realmatches
+			end
+			
+			cacheobject.cache.each do |obj|
+				if(obj[1]>self.errors)
+					next
+				end
+				realpos = Array.new
+				realpos << obj[2][0]
+				(1..obj[2].length-1).each do |i|
+				    curpos= obj[2][i]
+					if((curpos<=max_position || max_position==0) && curpos>=min_position)
+						realpos << curpos
+					end
+				end
+				if(realpos.length<=1)
+					next
+				end
+				realmatches << Array[obj[0],obj[1],realpos]
+				
+			end
+			return realmatches
+		end
+	
+	end
+ 
+ 
  	# Base class to index and search through a string 
  
     class Crawler
@@ -164,11 +258,30 @@ module Cassiopee
     attr_accessor  :use_store
 	# Array of comment characters to skip lines in input sequence file
 	attr_accessor  :comments
+	
+	# Manage basic cache to store previous match
+	attr_accessor  :useCache 
     
+	# Method for search FORCE or SUFFIX
+	# * SUFFIX loads all suffixes and search through them afterwards, interesting for multiple searches (suffixes are reused)
+	# * FORCE checks matches while crossing the suffixes. Does not keep parsed data for later search
+	#   FORCE method does not yet support optimal filters
+	attr_accessor	:method
+	
+	METHOD_DIRECT = 0
+	METHOD_SUFFIX = 1
+	
     @min_position = 0
     @max_position = 0
 	
+	# Previous position filter
+	@prev_min_position = 0
+	@prev_max_position = 0
+	
 	@ambiguous = nil
+	
+	@pattern = nil
+	
     
     FILE_SUFFIX_EXT = ".sfx"
     FILE_SUFFIX_POS = ".sfp"
@@ -177,27 +290,39 @@ module Cassiopee
 
     $maxthread = 1
 	
+	@cache = nil
+	
     
     $log = Logger.new(STDOUT)
     $log.level = Logger::INFO
     
         def initialize
             @useAmbiguity = false
+			@useCache = false
             @file_suffix = "crawler"
-            
+			
+			@method = 0
+			
+			@prev_min_position = 0
+			@prev_max_position = 0
+		
+			
             @suffix = nil
             @suffixmd5 = nil
             @position = 0
             
             @suffixes = Hash.new
             
-            @matches = nil
+            @matches = Array.new
             @curmatch = 0
             @use_store = false
             
             @sequence = nil
 			
 			@comments = Array["#"]
+			
+			@cache = Cassiopee::CrawlerCache.new
+
         end
 		
 		def filterLength
@@ -213,7 +338,11 @@ module Cassiopee
         
         def clear
         	@suffixes = Hash.new
-		@matches = Array.new
+			@matches.clear
+			@pattern = nil
+			@prev_max_position = 0
+			@prev_min_position = 0
+			@cache.clearCache()
         	File.delete(@file_suffix+FILE_SUFFIX_POS) unless !File.exists?(@file_suffix+FILE_SUFFIX_POS)
         end
     
@@ -300,37 +429,57 @@ module Cassiopee
         # Filter matches to be between min and max start position
         # If not using use_store, search speed is improved but existing indexes are cleared
         # If max=0, then max is string length
+		# Must be called after index creation or load
         
         def filter_position(min,max)
             if(!use_store)
         		clear()
         	end
+			@prev_min_position = @min_position
+			@prev_max_position = @max_position
         	@min_position = min
         	@max_position = max
         end
         
         # Search exact match
         
-        def searchExact(pattern)
+        def searchExact(s)
+		
 		if(@useAmbiguity)
-		  return searchApproximate(pattern,0)
+		  return searchApproximate(s,0)
+		end
+		
+        s = s.downcase
+        
+		updateCache(0,0)
+		@matches = @cache.loadCache()
+		
+		if(@matches.length>0)
+			return cache?(@matches)
 		end
-        pattern = pattern.downcase
-        parseSuffixes(@sequence,pattern.length,pattern.length)
+		
+		#@matches.clear
+		
+		@pattern = Digest::MD5.hexdigest(s)
+		
+		parseSuffixes(@sequence,s.length,s.length,0,s)
         
-         @matches = Array.new
+		return @matches unless(method == METHOD_SUFFIX)
+		
          # Search required length, compare (compare md5?)
          # MD5 = 128 bits, easier to compare for large strings
-            matchsize = pattern.length
-            matchmd5 = Digest::MD5.hexdigest(pattern)
+            
+			
+			matchsize = @pattern.length
+			
             @suffixes.each do |md5val,posArray|
-                if (md5val == matchmd5)
+                if (isMatchEqual?(md5val))
                     match = Array[md5val, 0, posArray]
 		    		$log.debug "Match: " << match.inspect
 		    		@matches << match
                 end
             end
-        return @matches 
+        return cache?(@matches) 
         
         end
         
@@ -342,32 +491,48 @@ module Cassiopee
         
         
         def searchApproximate(s,edit)
+		
         	if(edit==0 && !@useAmbiguity) 
         		return searchExact(s)
         	end
-        
+			allowederrors = edit
         	if(edit>=0)
         	  useHamming = true
         	  minmatchsize = s.length
         	  maxmatchsize = s.length
+			  updateCache(1,edit)
+			  @matches = @cache.loadCache()
         	else
         	  useHamming = false
         	  edit = edit * (-1)
               minmatchsize = s.length - edit
               maxmatchsize = s.length + edit
+			  updateCache(2,edit)
+			  @matches = @cache.loadCache()
             end
+			
+			if(@matches.length>0)
+				return @matches
+			end
+			
+			s = s.downcase
             
-            parseSuffixes(@sequence,minmatchsize,maxmatchsize)
+			
+            #@matches.clear
+			@pattern = Digest::MD5.hexdigest(s)
+			
+			parseSuffixes(@sequence,minmatchsize,maxmatchsize,allowederrors,s)
             
-            matchmd5 = Digest::MD5.hexdigest(s)
+			return cache?(@matches) unless(method == METHOD_SUFFIX)
+			
+ 
             
-        	@matches = Array.new
-        	
+        	      	
         	@suffixes.each do |md5val,posArray|
         		if(md5val == SUFFIXLEN)
         			next
         		end
-        		if (md5val == matchmd5)
+        		if (md5val == @pattern)
         			filteredPosArray = filter(posArray)
                     match = Array[md5val, 0, filteredPosArray]
 		    		$log.debug "Match: " << match.inspect
@@ -376,20 +541,8 @@ module Cassiopee
 		    		if(posArray[0]>= minmatchsize && posArray[0] <= maxmatchsize)
 		    			# Get string
 		    			seq = extractSuffix(posArray[1],posArray[0])
-		    			seq.extend(Cassiopee)
-		    			if(useHamming)
-						  if(@useAmbiguity && @ambiguous!=nil)
-						    errors = seq.computeHammingAmbiguous(s,edit,@ambiguous)
-						  else
-		    			    errors = seq.computeHamming(s,edit)
-						  end
-		    			else
-						  if(@useAmbiguity && @ambigous!=nil)
-						    errors = seq.computeLevenshteinAmbiguous(s,edit,@ambigous)
-						  else
-		    			    errors = seq.computeLevenshtein(s,edit)
-						  end						
-		    			end
+						errors = isApproximateEqual?(seq,s,useHamming,edit)
+						
 		    			if(errors>=0)
 		    				filteredPosArray = filter(posArray)
 		    			    match = Array[md5val, errors, filteredPosArray]
@@ -401,7 +554,7 @@ module Cassiopee
         	
         	end
         	
-        	return @matches 
+        	return cache?(@matches) 
         end
         
         # Filter the array of positions with defined position filter
@@ -488,14 +641,68 @@ module Cassiopee
         end
         
         private
+		
+			# If cache is used, store results for later retrieval, else return matches directly
+			def cache?(results)
+				if(@useCache)
+					@cache.saveCache(results)
+				end
+				
+				return results
+			end
+		
+			# Update cache object with current object parameters
+			# * method: 0 -> exact, 1 -> hamming, 2 -> edit
+			def updateCache(method,errors)
+				@cache.file_suffix = @file_suffix
+				@cache.min_position = @min_position
+				@cache.max_position = @max_position
+				@cache.method = method
+				@cache.errors = errors
+			end
+
+		
+			# check if md5 is equal to pattern
+			def isMatchEqual?(s)
+				if(@pattern == s)
+					return true
+				end
+				return false
+			end
+			
+			# check if string is approximatly equal to pattern
+			# s: string to compare
+			# pattern: base pattern used
+			# useHamming: use Hamming or edit distance
+			# edit : allowed errors
+			def isApproximateEqual?(s,pattern,useHamming,edit)
+				errors = -1
+						s.extend(Cassiopee)
+		    			if(useHamming)
+						  if(@useAmbiguity && @ambiguous!=nil)
+						    errors = s.computeHammingAmbiguous(pattern,edit,@ambiguous)
+						  else
+		    			    errors = s.computeHamming(pattern,edit)
+						  end
+		    			else
+						  if(@useAmbiguity && @ambiguous!=nil)
+						    errors = s.computeLevenshteinAmbiguous(pattern,edit,@ambigous)
+						  else
+		    			    errors = s.computeLevenshtein(pattern,edit)
+						  end						
+		    			end
+			end
+		
+
+		
         
         	# Parse input string
         	#
         	# * creates a suffix file
         	# * creates a suffix position file
         	
-            def parseSuffixes(s,minlen,maxlen)
-            
+            def parseSuffixes(s,minlen,maxlen,edit=0,pat=nil)
+            						
             # Controls
             if(minlen<=0) 
             	minlen = 1
@@ -554,6 +761,7 @@ module Cassiopee
                 		next
                 	end
                 	changed = true
+					prev_progress = -1
                		 (minpos..(maxpos)).each do |j|
                		 	# if position+length longer than sequence length, skip it
                		 	if(j+i>=@sequence.length)
@@ -562,10 +770,46 @@ module Cassiopee
                     	@suffix = s[j,i]
                     	@suffixmd5 = Digest::MD5.hexdigest(@suffix)
                     	@position = j
-                    	$log.debug("add "+@suffix+" at pos "+@position.to_s)
-                    	nbSuffix += addSuffix(@suffixmd5, @position,i)
+						progress = (@position * 100).div(@sequence.length)
+						if((progress % 10) == 0 && progress > prev_progress)
+							prev_progress = progress
+							$log.debug("progress: " << progress.to_s)
+						end
+					
+						if(method==METHOD_DIRECT)
+						
+							if(edit==0 && !@useAmbiguity)
+								if(isMatchEqual?(@suffixmd5))
+									errors = 0
+								else
+									errors = -1
+								end
+							else
+							
+							if(edit>=0)
+								useHamming = true
+								allowederrors = edit
+							else
+								useHamming = false
+								allowederrors = edit * (-1)
+							end
+								errors = isApproximateEqual?(@suffix,pat,useHamming,allowederrors)
+							end
+							
+							
+							if(errors>=0)
+								match = Array[@suffixmd5, errors, Array[i,j]]
+								$log.debug "Match: " << match.inspect
+								@matches << match
+							end
+							
+							
+							
+						else
+                    	  nbSuffix += addSuffix(@suffixmd5, @position,i)
+						end
                     end
-                    $log.debug("Nb suffix found: " << nbSuffix.to_s << ' for length ' << i.to_s)
+                    $log.debug("Nb suffix found: " << nbSuffix.to_s << ' for length ' << i.to_s) unless method==METHOD_DIRECT
                 end
             
                 
@@ -615,7 +859,6 @@ module Cassiopee
                             	input = line.downcase.chomp
 								skip = false
 								comments.each do |c|
-								$log.debug("skip line ?" << c << " == " << input[0])
 									if(input[0] == c[0])
 										# Line start with a comment char, skip it
 										$log.debug("skip line")

data/tests/test-suite.rb

@@ -9,13 +9,13 @@ class TestCrawler < Test::Unit::TestCase
  
   def test_exactsearch
     crawler = Cassiopee::Crawler.new
-    crawler.setLogLevel(Logger::ERROR)
+    #crawler.setLogLevel(Logger::DEBUG)
     crawler.indexString('my sample example')
     matches = crawler.searchExact('ampl')
-    assert_equal(1,matches.length)
+    assert_equal(2,matches.length)
     # Minus 1, because first element is len of match
-    match = crawler.next()
-    assert_equal(2,match[2].length-1)
+    #match = crawler.next()
+    #assert_equal(2,match[2].length-1)
   end
 
   def test_ambiguous 
@@ -41,6 +41,23 @@ class TestCrawler < Test::Unit::TestCase
     assert_equal(1,matches.length)
   end
 
+  def test_directmethod
+    crawler = Cassiopee::Crawler.new
+    crawler.method = Cassiopee::Crawler::METHOD_DIRECT
+    crawler.indexString('my sample example')
+    matches = crawler.searchApproximate('ebampl',1)
+    assert_equal(1,matches.length)
+  end
+
+
+  def test_suffixmethod
+    crawler = Cassiopee::Crawler.new
+    crawler.method = Cassiopee::Crawler::METHOD_SUFFIX
+    crawler.indexString('my sample example')
+    matches = crawler.searchApproximate('ebampl',1)
+    assert_equal(1,matches.length)
+  end
+
   def  test_multithreadsearch
     crawler = CassiopeeMt::CrawlerMt.new
     crawler.maxthread=3
@@ -49,6 +66,36 @@ class TestCrawler < Test::Unit::TestCase
     assert_equal(1,matches.length)
   end
 
+  def test_cache
+
+    crawler = Cassiopee::Crawler.new
+    crawler.indexString('my sample example')
+    matches = crawler.searchApproximate('ebampl',-1)
+
+    cache = Cassiopee::CrawlerCache.new
+    cache.method = 2
+    cache.min_position = 0
+    cache.max_position = 0
+    cache.errors = 1
+    cache.saveCache(matches)
+
+    cache = Cassiopee::CrawlerCache.new
+    cache.method = 2
+    cache.min_position = 0
+    cache.max_position = 0
+    cache.errors = 1
+    cachematches = cache.loadCache
+    assert_equal(1,cachematches.length)
+
+    cache = Cassiopee::CrawlerCache.new
+    cache.method = 2
+    cache.min_position = 0
+    cache.max_position = 0
+    cache.errors = 2
+    cachematches = cache.loadCache
+    assert_equal(0,cachematches.length)
+
+  end
 end
 
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: cassiopee
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: false
   segments: 
   - 0
   - 1
-  - 4
-  version: 0.1.4
+  - 5
+  version: 0.1.5
 platform: ruby
 authors: 
 - Olivier Sallou
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-19 00:00:00 +02:00
+date: 2011-09-20 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency