thomaspeklak-OfflineSearch 0.2.2

data/lib/temporary_storage.rb

@@ -0,0 +1,257 @@
+# temporary storage
+# class to store crawled data before the date is written to a file
+#
+# options:
+#   datebase
+#     sqlite, mysql
+#     filesystem
+#
+# * $Author$
+# * $Rev$
+# * $LastChangedDate$
+
+class Temporary_Storage
+  attr_reader :storage_handler
+  
+  # initializes the storage handler
+  def initialize(mode)
+    @storage_handler=case
+      when mode=='sqlite': Sqlite.new('storage.db')
+      when mode=='memory': Memory.new
+      else
+        $logger.error("no appropriate stroage is selected\nvalid options:\n\tsqlite\n\tmemory")
+        exit
+    end
+  end
+
+  # stores file name, title and page rank.
+  # start pagerank with 1 to enable mutliplicatoin pagerank
+  def store_file(filename,title,pagerank=1)
+     @storage_handler.store_file(filename.to_s,title.to_s.gsub('"','"'),pagerank)
+  end
+  
+  #stores the term and term rank
+  def store_term(term,rank)
+    @storage_handler.store_term(term,rank)
+  end
+
+  # stores an array of links
+  def store_link(links)
+    @storage_handler.store_link(links)
+  end
+
+  # returns a hash of a stored file
+  def get_file(filename)
+    f=@storage_handler.get_file(filename)
+    { 'filename'=>f[0],
+      'titel'=>f[1],
+      'pagerank'=>f[2] }
+  end
+  
+  # returns a hash of stored files
+  def get_files
+    f=@storage_handler.get_files
+  end
+  
+  # returns an array of links
+  def get_links
+    @storage_handler.get_links
+  end
+  
+  # returns a hash of terms
+  def get_terms
+    @storage_handler.get_terms
+  end
+  
+  # calculates the page rank
+  # the page rank equals the number of inbound links or if none 1 
+  def calculate_pageranks_from_links
+    @storage_handler.calculate_pageranks_from_links
+  end
+  
+  private
+  
+  # implements a storage handler in the memory
+  class Memory
+    def initialize
+      @files = Hash.new
+      @terms = Terms.new
+      @links = Links.new
+      @current_doc = nil
+    end
+    
+    #stores the file in the files hash
+    def store_file(filename,title,pagerank=1)
+      @files[filename] = @current_document = Document.new(filename, title, pagerank)
+    end
+    
+    # stores a term in the terms class
+    def store_term(term,rank)
+      @terms.store(term, Term2Document.new(@current_document,rank))
+    end
+    
+    # stores a link in the link class
+    def store_link(links)
+      @links.add(links)
+    end
+    
+    def get_file(filename)
+      
+    end
+    
+    # returns the files hash
+    def get_files
+      @files
+    end
+    
+    # returns a terms hash
+    def get_terms
+      @terms.get_all
+    end
+    
+    # returns an array of links
+    def get_links
+      @links.get_all
+    end
+    
+    # calculates the page rank
+    # the page rank equals the number of inbound links or if none 1     
+    def calculate_pageranks_from_links
+      @links.get_all.each do |link, rank|
+        @files[link].page_rank=rank if @files.has_key?(link)
+      end
+    end
+    
+    private
+    
+    # represents a document. stores an internal id, the file name, title and page rank. all attributes are accessible
+    class Document
+      @@ID=0
+      attr_accessor :ID, :name, :title, :page_rank
+      def initialize(name,title,page_rank)
+        @ID= @@ID+=1
+        @name = name
+        @title = title
+        @page_rank = page_rank
+      end
+    end
+    
+    # represents a hash of terms and their corresponding documents
+    class Terms
+      def initialize
+        @terms = Hash.new
+      end
+      
+      # stores a term in the terms hash with the corresponding document or adds the document to a term if the term already exists in the hash
+      def store(term,term2document)
+        @terms.has_key?(term) ? @terms[term] << term2document : @terms[term]=[term2document]
+      end
+
+      # returns a term hash
+      def get_one(term)
+        @terms[term]
+      end
+      
+      # returns the terms hash
+      def get_all
+        @terms
+      end
+    end
+    
+    # represents a link from a term to a document. the link includes the semantic value of the term. all attributes are accesible
+    class Term2Document
+      attr_accessor :document, :rank
+      def initialize(document, rank)
+        @document = document
+        @rank = rank
+      end
+    end
+    
+    # represents unique links of all indexed documents
+    class Links
+      def initialize
+        @links = Hash.new
+      end
+      
+      # adds a link to the hash or increases the link value by one if the link already exists
+      def add(links)
+        links.each{ |link| @links.has_key?(link)? @links[link]+=1 : @links[link]=1 }
+      end
+      
+      # returns all links
+      def get_all
+        @links
+      end
+    end
+  end
+  
+  class Sqlite
+    def initialize(db)
+      require 'rubygems'
+      require 'sqlite3'
+      @current_file_id = nil
+      begin
+        File.delete(db)
+      rescue
+      end
+      @db = SQLite3::Database.new(db)
+      @db.type_translation = true
+      sql = "        
+        create table files( 
+          id integer not null primary key autoincrement,
+          filename varchar2(255), 
+          title varchar2(255), 
+          pagerank integer
+        );
+        create table terms(
+          id integer not null primary key autoincrement,
+          term varchar2(255) unique not null
+        );
+        create table files_terms(
+          file_id integer not null,
+          term_id integer not null,
+          rank integer not null
+        );
+        create table links(
+          link varchar2(255) not null primary key,
+          links_in integer
+        )
+        "
+      @db.execute_batch(sql)
+    end
+    def store_file(filename, title, pagerank)
+      @db.execute( "insert into files (filename, title, pagerank) values ( ?, ?, ? )", filename, title, pagerank)
+      @current_file_id = @db.last_insert_row_id()
+    end
+    
+    def store_term(term, rank)
+      unless (term_id=@db.get_first_value('select id from terms where term = ?',term)) :
+        @db.execute("insert into terms (term) values (?)", term)
+        term_id=@db.last_insert_row_id()
+      end
+      @db.execute("insert into files_terms values (?,?,?)", @current_file_id,term_id,rank)
+    end
+    
+    def store_link(links)
+      links.each do |link|
+        if(links_in  = @db.get_first_value("select links_in from links where link = ?",link))
+          @db.execute("update links set links_in =? where link = ? ",links_in+1,link)
+        else
+          @db.execute("insert into links values (?,?)", link, 1)
+        end
+      end
+    end
+    
+    def get_file(filename)
+      @db.get_first_row("select * from files where filename = ?", filename)
+    end
+
+    def get_files
+      @db.execute("select * from files f join files_term ft on f.id = ft.id")
+    end
+    
+    def get_links
+      @db.execute("select * from links")
+    end
+  end
+end

data/templates/base+double_metaphone/jQueryDoubleMetaphone.js

@@ -0,0 +1,290 @@
+(function(){
+	String.prototype.doubleMetaphone = function(){
+		var primary = new Array, secondary = new Array, current = 0;
+		var original = this.toUpperCase()+'     ', length = this.length, last = length -1;
+		if(/^GN|KN|PN|WR|PS$/.test(original.substr(0,2)))
+      current += 1;
+    if('X' == original.substr(0, 1)){
+			primary.push('S');
+			secondary.push('S');
+			current += 1;
+		}
+		//main loop
+		var resultset = new Array;
+		while ((primary.length<4 || secondary.length<4) && current <= length){
+			resultset = double_metaphone_lookup(original, current, length, last);
+			if(resultset[0]) primary.push(resultset[0]);
+			if(resultset[1]) secondary.push(resultset[1]);
+			current += resultset[2];
+		}
+		primary = primary.join('').substr(0,4);
+		secondary = secondary.join('').substr(0,4);
+		return [primary, (primary == secondary)? null : secondary];
+	};
+	String.prototype.slavo_germanic = function(){
+		return /W|K|CZ|WITZ/.test(this);
+	};
+	String.prototype.vowel = function() {
+		return /^[AEIOUY]$/.test(this);
+	};
+	var A='A',B='B',C='C',D='D',E='E',F='F',G='G',H='H',I='I',J='J',K='K',L='L',M='M',N='N',O='O',P='P',Q='Q',R='R',S='S',T='T',U='U',V='V',W='W',X='X',Y='Y',Z='Z';
+	function double_metaphone_lookup(str, pos, length, last){
+		var cl = str.charAt(pos); // current letter
+		switch (true){
+			case cl.vowel():
+				return (pos) ? [null, null, 1] : [A,A,1];
+			case cl==B:
+				return [P,P,(B == str.charAt(pos+1))?2:1];
+			case cl=='Ç':
+				return [S,S,1];
+			case cl==C:
+				if(pos>1 && !str.charAt(pos-2).vowel() && 'ACH' == str.substr(pos-1,3) && str.charAt(pos+2) != I && (str.charAt(pos+2) != E || /^(B|M)ACHER$/.test(str.substr(pos-2,6))))
+					return [K,K,2];
+				else if(!pos && 'CAESAR' == str.substr(pos,6))
+					return [S,S,2];
+				else if('CHIA' == str.substr(pos, 4))
+					return [K,K,2];
+				else if('CH' == str.substr(pos,2)){
+					if(pos && 'CHAE' == str.substr(pos,4))
+						return [K,X,2];
+					else if(!pos && ($.in_array(['HARAC', 'HARIS'],str.substr(pos+1,5)) || $.in_array(['HOR', 'HYM', 'HIA', 'HEM'],str.substr(pos+1,3))) && str.substr(0,5) != 'CHORE')
+						return [K,K,2];
+					else if($.in_array(['VON','VAN'],str.substr(0,4)) || 'SCH' == str.substr(0,3) || $.in_array(['ORCHES','ARCHIT','ORCHID'],str.substr(pos-2,6)) || /^T|S$/.test(str.charAt(pos+2)) || ((!pos || /^[AOUE]$/.test(str.charAt(pos-1))) && /^[LRNMBHFVW ]$/.test(str.charAt(pos+2))))
+						return [K,K,2];
+					else if(pos)
+						return [('MC' == str.substr(0,2))?K:X,K,2];
+					else return [X,X,2];
+				}
+				else if(Z == str.charAt(pos+1) && 'WI' != str.substr(pos-2,2))
+					return [S,X,2];
+				else if('CIA' == str.substr(pos+1,3))
+					return [X,X,3];
+				else if(C == str.charAt(pos+1) && 1 != pos && M != str.charAt(0)){
+					if(/^[IEH]$/.test(str.charAt(pos+2)) && 'HU' != str.substr(pos+2,2)){
+						if ((1 == pos && A == str.charAt(pos-1)) || /^UCCE(E|S)$/.test(str.substr(pos-1,5)))
+							return ['KS','KS',3];
+						else
+							return [X,X,3];
+					}
+					else
+						return [K,K,2];
+				}
+				else if(/^[KGQ]$/.test(str.charAt(pos+1)))
+					return [K,K,2];
+				else if(/^[IEY]$/.test(str.charAt(pos+1)))
+					return [S,(/^I(O|E|A)$/.test(str.substr(pos+1, 2)) ? X : S), 2];
+				else{
+					if(/^ (C|Q|G)$/.test(str.substr(pos+1,2)))
+						return [K,K,3];
+					else
+						return [K,K,(/^[CKQ]$/.test(str.charAt(pos+1)) && !($.in_array(['CE','CI'],str.substr(pos+1,2))))? 2: 1];
+				}
+			case cl==D:
+				if(str.charAt(pos+1)==G){
+					if(/^[IEY]$/.test(str.charAt(pos+2)))
+						return [J,J,3];
+					else
+						return ['TK','TK',2];
+				}
+				else
+					return [T,T,(/^[DT]$/.test(str.charAt(pos+1)))? 2:1];
+			case cl==F:
+				return [F,F,(F==str.charAt(pos+1))?2:1];
+			case cl==G:
+				if(H==str.charAt(pos+1)){
+					if(pos && !str.charAt(pos-1).vowel())
+						return [K,K,2];
+					else if(!pos){
+						if(I == str.charAt(pos+2))
+							return [J,J,2];
+						else
+							return [K,K,2];
+					}
+					else if((pos>1 && /^[BHD]$/.test(str.charAt(pos-2))) || (pos>2 && /^[BHD]$/.test(str.charAt(pos-3))) || (pos>3 && /^B|H$/.test(str.charAt(pos-4))))
+						return [null,null,2];
+					else{
+						if(pos>2 && U == str.charAt(pos-1) && /^[CGLRT]$/.test(str.charAt(pos-3)))
+							return [F,F,2];
+						else{
+							if(pos && I != str.charAt(pos-1))
+								return [K,K,2];
+							else
+								return [null,null,2];
+						}
+					}
+				}
+				else if(N==str.charAt(pos+1)){
+					if(1==pos && str.charAt(0).vowel() && !str.slavo_germanic())
+						return ['KN',N,2];
+					else{
+						if('EY' != str.substr(pos+2,2) && Y != str.charAt(pos+1) && !str.slavo_germanic())
+							return [N,'KN',2];
+						else
+							return ['KN',N,2];
+					}
+				}
+				else if('LI'==str.substr(pos+1,2))
+					return ['KL',L,2];
+				else if(!pos && (Y==str.charAt(pos+1) || /^(E(S|P|B|L|Y|I|R)|I(B|L|N|E))$/.test(str.substr(pos+1,2))))
+					return [K,J,2];
+				else if(('ER' == str.substr(pos+1,2) || Y == str.charAt(pos+1)) && !/^(D|R|M)ANGER$/.test(str.substr(0,6)) && !/^E|I$/.test(str.charAt(pos-1)) && !/^(R|O)GY$/.test(str.substr(pos-1,3)))
+					return [K,J,2];
+				else if(/^[EIY]$/.test(str.charAt(pos+1)) || /^(A|O)GGI$/.test(str.substr(pos-1,4))){
+					if(/^V(A|O)N $/.test(str.substr(0,4)) || 'SCH' == str.substr(0,3) || 'ET' == str.substr(pos+1,2))
+						return [K,K,2];
+					else{
+						if ('IER ' == str.substr(pos+1,4))
+							return [J,J,2];
+						else
+							return [J,K,2];
+					}
+				}
+				else if(G==str.charAt(pos+1))
+					return [K,K,2];
+				else
+					return [K,K,1];
+			case cl==H:
+				if(!pos || str.charAt(pos-1).vowel() && str.charAt(pos+1).vowel())
+					return [H,H,2];
+				else
+					return [null,null,1];
+			case cl==J:
+				if ('OSE' == str.substr(pos+1,3) || 'SAN ' == str.substr(0,4)){
+					if((!pos && ' ' == str.charAt(pos+4)) || 'SAN ' == str.substr(0,4))
+						return [H,H,1];
+					else
+						return [J,H,1];
+				}
+				else{
+					var current = (J==str.charAt(pos+1))? 2 : 1;
+					if(!pos && 'OSE' != str.substr(pos+1,3))
+						return [J,A,current];
+					else{
+						if (str.charAt(pos-1).vowel() && !str.slavo_germanic() && /^A|O$/.test(str.charAt(pos+1)))
+							return [J,H,current];
+						else{
+							if(last == pos)
+								return [J,null, current];
+							else{
+								if (!/^[LTKSNMBZ]$/.test(str.charAt(pos+1)) && !/^[SKL]$/.test(str.charAt(pos-1)))
+									return [J,J,current];
+								else
+									return [null,null,current];
+							}
+						}
+					}
+				}
+			case cl==K:
+				return [K,K,(K==str.charAt(pos+1))? 2 : 1];
+			case cl==L:
+				if(L==str.charAt(pos+1)){
+					if(((length-3)==pos && /^(ILL(O|A)|ALLE)$/.test(str.substr(pos-1,4))) || (/^(A|O)S$/.test(str.substr(last-1,2)) || /^A|O$/.test(str.charAt(last)) && 'ALLE'== str.substr(pos-1,4)))
+						return [L,null,2];
+					else
+						return [L,L,2];
+				}
+				else
+					return [L,L,1];
+			case cl==M:
+				if(('UMB' == str.substr(pos-1,3) && (last-1 == pos || 'ER' == str.substr(pos+2,2))) || M == str.charAt(pos+1))
+					return [M,M,2];
+				else
+					return [M,M,1];
+			case cl==N:
+				return [N,N,(N==str.charAt(pos+1))? 2: 1];
+			case cl=='Ñ':
+				return [N,N,1];
+			case cl==P:
+				if(H==str.charAt(pos+1))
+					return [F,F,2];
+				else
+					return [P,P,(/^P|B$/.test(str.charAt(pos+1)))? 2 : 1];
+			case cl==Q:
+				return [K,K,(Q==str.charAt(+1))? 2: 1];
+			case cl==R:
+				var current =(R==str.charAt(pos+1))? 2 : 1;
+				if(last == pos && !str.slavo_germanic() && 'IE' == str.substr(pos-2,2) && !/^M(E|A)$/.test(str.substr(pos-4,2)))
+					return [null,R,current];
+				else
+					return [R,R,current];
+			case cl==S:
+				if(/^(I|Y)SL$/.test(str.substr(pos-1,3)))
+					return [null,null,1];
+				else if (H==str.charAt(pos+1)){
+					if (/^H(EIM|OEK|OLM|OLZ)$/.test(str.substr(pos+1,4)))
+						return [S,S,2];
+					else
+						return [X,X,2];
+				}
+				else if (/^I(O|A)$/.test(str.substr(pos+1,2)))
+					return [S,(str.slavo_germanic())? S : X, 3];
+				else if ((!pos && /^[MNLW]$/.test(str.charAt(+1))) || Z==str.charAt(pos+1))
+					return [S,X,(Z==str.charAt(pos+1))? 2 : 1];
+				else if (C== str.charAt(pos+1)){
+					if (H== str.charAt(pos+2)){
+						if (/^OO|ER|EN|UY|ED|EM$/.test(str.substr(pos+3,2)))
+							return [(/^E(R|N)$/.test(str.substr(pos+3,2)))? X : 'SK','SK',3];
+						else
+							return [X,((!pos && !str.charAt(3).vowel()) && (W != str.charAt(pos+3)))? S : X,3];
+					}
+					else if (/^[IEY]$/.test(str.charAt(pos+2)))
+						return [S,S,3];
+					else
+						return ['SK','SK',3];
+				}
+				else
+					return [(last == pos && /^(A|O)I$/.test(str.substr(pos-2,2)))? null : S,S,(/^S|Z$/.test(str.charAt(pos+1)))? 2 : 1];
+			case cl==T:
+				if ('ION' == str.substr(pos+1,3) || /^IA|CH$/.test(str.substr(pos+1,2)))
+					return [X,X,3];
+				else if(H==str.charAt(pos+1) || 'TH' == str.substr(pos+1,2)){
+					if(/^(O|A)M$/.test(str.substr(pos+2,2)) || /^V(A|O)N $/.test(str.substr(0,4)) || 'SCH'== str.substr(0,3))
+						return [T,T,2];
+					else
+						return['0',T,2];
+				}
+				else return [T,T,(/^T|D$/.test(str.charAt(pos+1)))? 2 : 1];
+			case cl==V:
+				return [F,F,(V==str.charAt(pos+1))? 2 : 1];
+			case cl==W:
+				if(R==str.charAt(pos+1))
+					return [R,R,2];
+				var pri = '';
+				var sec = '';
+				if(!pos && str.charAt(pos+1).vowel() || H==str.charAt(pos+1)){
+					pri = A;
+					sec = (str.charAt(pos+1).vowel())? F : A;
+				}
+				if(last == pos && str.charAt(pos-1).vowel() || 'SCH' == str.substr(0,3) || /^EWSKI|EWSKY|OWSKI|OWSKY$/.test(str.substr(pos-1,5)))
+					return [pri,sec+F,1];
+				else if(/^I(C|T)Z$/.test(str.substr(pos+1,3)))
+					return [pri+'TS',sec+'FX',4];
+				else
+					return [pri,sec,1];
+			case cl==X:
+				var current = (/^C|X$/.test(str.charAt(pos+1)))? 2 : 1;
+				if (last == pos && (/^(I|E)AU$/.test(str.substr(pos-3,3)) || /^(A|O)U$/.test(str.substr(pos-2, 2))))
+					return [null,null, current];
+				else
+					return ['KS','KS',current];
+			case cl==Z:
+				if(H==str.charAt(pos+1))
+					return [J,J,2];
+				else{
+					var current = (Z==str.charAt(pos+1)) ? 2 : 1;
+					if(/^Z(O|I|A)$/.test(str.substr(pos+1,2)) || (str.slavo_germanic() && (pos > 0 && T != str.charAt(pos-1))))
+						return [S,'TS',current];
+					else
+						return [S,S,current];
+				}
+		}
+		return [null,null,1];
+	};
+	$.in_array = function(arr,p_val){
+		for(var i = 0, l = arr.length; i < l; i++){
+			if(arr[i] == p_val)
+				return true;
+		}
+		return false;
+	};
+})();