cassiopee 0.1.2 → 0.1.3

data/Changelog

@@ -1,3 +1,5 @@
+v0.1.3 : 09/11 Olivier Sallou
+ add CrawlerMT in cassiopee-mt for multi thread support to speed up the search
 v0.1.2 : 09/11 Olivier Sallou
  add possibility to reload an "index" instead of using index method again
  fix comment mngt (comments attribute)

data/demo-mt.rb

@@ -0,0 +1,28 @@
+require File.join(File.dirname(__FILE__), 'lib/cassiopee-mt')
+require 'rubygems'
+require 'logger'
+
+# Instanciate a new crawler
+crawler = CassiopeeMt::CrawlerMt.new
+crawler.setLogLevel(Logger::INFO)
+crawler.maxthread=3
+#crawler.use_store = true
+
+# String to index
+crawler.indexString('iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiisallou salluiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii')
+# Search pattern in indexed string
+crawler.searchExact('llo')
+
+# Go through matches
+while((match = crawler.next())!=nil)
+  puts "got an exact  match " << match.inspect
+end
+
+crawler.clear()
+
+crawler.searchApproximate('llo',1)
+
+# Go through matches
+while((match = crawler.next())!=nil)
+  puts "got an approximate  match " << match.inspect
+end

data/lib/cassiopee-mt.rb

@@ -0,0 +1,79 @@
+require 'digest/md5'
+require 'logger'
+require 'zlib'
+#require 'rubygems'
+#require 'text'
+#require 'text/util'
+require File.join(File.dirname(__FILE__), 'cassiopee')
+
+include Cassiopee
+
+# Module managing multi threads to search in strings, extending Cassiopee
+module CassiopeeMt
+
+    # Multi threaded search using a Crawler per thread
+    # Filtering is used to split the input data according to maxthread
+    # Matches of each thread are merge to matches of CrawlerMT
+    class CrawlerMt < Crawler
+ 
+    # Max number fo threads to use
+    attr_accessor  :maxthread
+
+    @th = []
+
+    def initialize
+	super
+	@th = []
+	@matches = Array.new
+    end
+
+
+    def setParams(crawler,threadId)
+      crawler.setLogLevel($log.level)
+      crawler.file_suffix = @file_suffix
+      crawler.loadIndex()
+      #crawler.file_suffix = @file_suffix+"."+threadId.to_s
+      crawler.indexString(@sequence)
+    end
+
+   def searchExact(pattern)
+        nb = @sequence.length.div(maxthread)
+        min = 0
+        (1..maxthread).each do |i|
+          crawler = Crawler.new
+	  setParams(crawler,i)
+          max = min + nb
+          if(i==maxthread)
+             max = @sequence.length
+          end
+          crawler.filter_position(min,max)
+	  $log.debug("Start new Thread between " << min.to_s << " and " << max.to_s)
+          @th[i-1] = Thread.new{ Thread.current["matches"] = crawler.searchExact(pattern) }
+          min = max + 1
+        end
+        @th.each {|t| t.join; t["matches"].each { |m| @matches << m }}
+        return @matches
+   end
+
+   def searchApproximate(s,edit)
+        nb = @sequence.length.div(maxthread)
+        min = 0
+        (1..maxthread).each do |i|
+          crawler = Crawler.new
+          setParams(crawler,i)
+          max = min + nb
+          if(i==maxthread)
+             max = @sequence.length
+          end
+          crawler.filter_position(min,max)
+          $log.debug("Start new Thread between " << min.to_s << " and " << max.to_s)
+          @th[i-1] = Thread.new{ Thread.current["matches"] = crawler.searchApproximate(s,edit) }
+          min = max + 1
+        end
+        @th.each {|t| t.join; t["matches"].each { |m| @matches << m }}
+        return @matches
+   end 
+
+   end
+
+end

data/lib/cassiopee.rb

@@ -136,15 +136,14 @@ module Cassiopee
   # * ambigous is a Hash of char/Array of char mapping
   
   def isAmbiguousEqual(a,b,ambiguous)
-	if(ambiguous==nil || ambiguous[a.chr]==nil)
+	if(ambiguous==nil || (ambiguous[a.chr]==nil && ambiguous[b.chr]==nil ))
 	  if(a==b)
 	    return true
 	  else
 	    return false
 	  end
 	end
-	vin = "" << a.chr
-	if(ambiguous[a.chr].index(b.chr)!=nil)
+	if(ambiguous[a.chr].index(b.chr)!=nil || ambiguous[b.chr].index(a.chr)!=nil || a==b)
 	   return true
     else
 	   return false
@@ -214,6 +213,7 @@ module Cassiopee
         
         def clear
         	@suffixes = Hash.new
+		@matches = Array.new
         	File.delete(@file_suffix+FILE_SUFFIX_POS) unless !File.exists?(@file_suffix+FILE_SUFFIX_POS)
         end
     

data/tests/test-suite.rb

@@ -1,4 +1,5 @@
 require File.join(File.dirname(__FILE__), '../lib/cassiopee')
+require File.join(File.dirname(__FILE__), '../lib/cassiopee-mt')
 require 'rubygems'
 require 'logger'
 require 'test/unit'
@@ -40,6 +41,14 @@ class TestCrawler < Test::Unit::TestCase
     assert_equal(1,matches.length)
   end
 
+  def  test_multithreadsearch
+    crawler = CassiopeeMt::CrawlerMt.new
+    crawler.maxthread=3
+    crawler.indexString('iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiimy sample exampleiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii')
+    matches = crawler.searchExact('exam')
+    assert_equal(1,matches.length)
+  end
+
 end
 
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: cassiopee
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 29
   prerelease: false
   segments: 
   - 0
   - 1
-  - 2
-  version: 0.1.2
+  - 3
+  version: 0.1.3
 platform: ruby
 authors: 
 - Olivier Sallou
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-09 00:00:00 +02:00
+date: 2011-09-19 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -47,7 +47,9 @@ files:
 - Changelog
 - LICENSE
 - demo.rb
+- demo-mt.rb
 - lib/cassiopee.rb
+- lib/cassiopee-mt.rb
 - bin/cassie.rb
 - tests/test-suite.rb
 - tests/amb.map