thomaspeklak-OfflineSearch 0.2.2

data/lib/entity_converter.rb

@@ -0,0 +1,33 @@
+# extends the string class to convert html entities
+# use carefully, can not convert entities back, as some entities are just skipped, because the are not useful for the search generation process
+#
+# * $Author$
+# * $Rev$
+# * $LastChangedDate$
+
+class String
+  # this method converts encoded entities to their utf-8 euqivalent. be careful this method strips out all unknown entities because they are of no special use for the semantic search
+  def decode_html_entities
+    mgsub([[/ä/,'ä'],[/Ä/,'Ä'],[/ö/,'ö'],[/Ö/,'Ö'],[/ü/,'ü'],[/Ü/,'Ü'],[/ß/,'ß'],[/&[a-zA-Z]{4,6};/,' ']])
+  end
+
+  # encodes html entities
+  def encode_html_entities
+    mgsub([[/ä/,'ä'],[/Ä/,'Ä'],[/ö/,'ö'],[/Ö/,'Ö'],[/ü/,'ü'],[/U/,'Ü'],[/ß/,'ß']])    
+  end
+  
+  # converts uppercase umlauts to downcase
+  def umlaut_to_downcase
+    mgsub([[/Ä/,'ä'],[/Ö/,'ö'],[/Ü/,'ü']])
+  end
+  
+  private
+
+  # method to substitute multiple strings at once. [Author: Ruby Cookbook]
+  def mgsub(key_value_pairs=[].freeze)
+    regexp_fragments = key_value_pairs.collect { |k,v| k }
+    gsub(Regexp.union(*regexp_fragments)) do |match|
+    key_value_pairs.detect{|k,v| k =~ match}[1]
+  end
+end  
+end

data/lib/generate_default_config.rb

@@ -0,0 +1,9 @@
+# generates a default configuration file in the current path
+#
+# * $Author$
+# * $Rev$
+# * $LastChangedDate$
+#
+require 'fileutils'
+include FileUtils
+cp(File.dirname(__FILE__)+'/config_default.yaml','./')

data/lib/generate_default_stopwords.rb

@@ -0,0 +1,18 @@
+# generates the default stopword list in the current directory
+# the language is taken from the language switch
+#
+# * $Author$
+# * $Rev$
+# * $LastChangedDate$
+#
+require 'fileutils'
+include FileUtils
+
+language = ['german','english']
+unless(defined?($config) && language.include?($config['language']))
+  $logger.error('language must be english or german')
+  exit
+end
+
+
+cp(File.dirname(__FILE__) +"/stoplist/#{$config['language']}/stopwords.txt",'./')

data/lib/generate_default_template.rb

@@ -0,0 +1,38 @@
+# generates template files
+# currently only one template is supported
+#
+# * $Author$
+# * $Rev$
+# * $LastChangedDate$
+#
+
+class TemplateGenerator
+  def initialize(template)
+    @template = template
+    find_files
+    copy_files_to_current_path
+  end
+  
+  private
+  # serach the given docpath for files
+  # returns an array of files
+  def find_files()
+    require 'find'
+    directory = File.dirname(__FILE__) + '/../templates/' + @template
+    @files = Array.new()
+    Find.find(directory) do |f|
+      if FileTest.file?f
+        @files.push(f)
+      end
+    end
+    @files
+  end
+
+  #copies the found files in the current path
+  def copy_files_to_current_path()
+    require 'fileutils'
+    @files.each do |f|
+      FileUtils::cp(f,'./')
+    end
+  end
+end

data/lib/log_init.rb

@@ -0,0 +1,16 @@
+# configures the ruby logger
+#
+# * $Author$
+# * $Rev$
+# * $LastChangedDate$
+#
+
+require 'logger'
+unless ($config.has_key?('logger'))
+  $logger = Logger.new(STDOUT)
+  $logger.level = Logger::INFO
+else
+  $logger = ($config['logger']['file'] == 'STDOUT')? Logger.new(STDOUT) : Logger.new($config['logger']['file'])
+  $logger.level = eval("Logger::#{$config['logger']['level'].upcase}")
+end
+

data/lib/offline_search.rb

@@ -0,0 +1,11 @@
+# Start point for OS
+#
+# * $Author$
+# * $Rev$
+# * $LastChangedDate$
+#
+
+require 'option_parser'
+
+require 'action_controller'
+ActionController.new

data/lib/option_parser.rb

@@ -0,0 +1,61 @@
+#parses command line options and merges them into the config file
+#
+# * $Author$
+# * $Rev$
+# * $LastChangedDate$
+#
+require "YAML"
+
+require 'optparse'
+$config = Hash.new
+OptionParser.new do |opts|
+  opts.banner = "Usage: OfflineSearch [options]"
+  opts.on('-c', '--config=CONFIG_FILE', String,'configuration file for the offline search') do |c|
+    if (File.exists?(c))
+      $config = YAML.load_file(c)
+    else
+      $logger.error('config file not found')
+      exit      
+    end
+  end
+  opts.separator ""
+  opts.separator "Generators"
+  opts.on('-g','--generate-default-config','creates a default config file in the current directory') do
+    $action = 'generate_default_config'
+  end
+  opts.on('-w','--generate-default-stopwords','creates a default stopword list in the current directory. Language flag is required.') do
+    $action = 'generate_default_stopwords'
+  end
+  opts.on('-t','--generate-template=TEMPLATE','creates search template files in the current directory. Possible values: base, base+double_metaphone') do |t|
+    $action = 'generate_template'
+		$config['template']=t
+  end
+  opts.on('-o','--generate-search-data','crawler the documents in the given docpath and generates the search data file') do
+    $action = 'generate_search'
+  end
+  opts.separator ""
+  opts.separator "Optional arguments"
+  opts.separator "can also be specified in the config file"
+  opts.separator "command line arguments will overwrite any given value in the config file"
+  opts.on('-d', '--docpath=DOCPATH', String,'path of the documents') do |d|
+    $config['crawler']['docpath'] = d
+  end 
+  opts.on('-f', '--search-data-file=SEARCH_DATA_FILE', String,'path and name of the search data file') do |f|
+    $config['search_generator']['search_data_file'] = f
+  end
+  opts.on('-s', '--stopword-list=STOPWORD_LIST', String,'stopword list, if none is specified the default stop word list is used') do |s|
+    $config['crawler']['stopwords'] = s
+  end
+  opts.on('-l','--language=LANGUAGE',String,'required if you want to generate a default stopword list') do |l|
+    $config['language'] = l
+  end  
+  opts.separator ""
+  opts.on_tail('-h','--help','Show this message') do
+    puts opts
+    exit
+  end
+  if (opts.default_argv.size == 0)
+    puts opts
+    exit
+  end
+end.parse!

data/lib/option_validator.rb

@@ -0,0 +1,53 @@
+# checks if all required arguments are specified and if their values are correct
+#
+# * $Author$
+# * $Rev$
+# * $LastChangedDate$
+#
+class OptionValidator
+  def initialize
+    storage = ['memory','sqlite']
+    language = ['german','english']
+
+    if ($config['crawler']['stopwords'].nil?) then
+      $config['crawler']['stopwords'] = File.dirname(__FILE__) +"/stoplist/#{$config['language']}/stopwords.txt"
+    end
+
+    unless(language.include?($config['language']))
+      $logger.error('language must be english or german')
+      exit
+    end
+    unless(storage.include?($config['storage']))
+      $logger.error('storage must be memory or sqlite')
+      exit
+    end
+    unless($config['crawler']['docs'].size>0)
+      $logger.error('doc types must be specified')
+      exit
+    end
+    unless (File.exists?($config['crawler']['stopwords']))
+      $logger.error('stopwords file does not exist')
+      exit
+    end
+    
+    unless (directory_exists?($config['crawler']['docpath']))
+      $logger.error('docpath does not exist')
+    end
+    
+    unless (base_directory_exists?($config['search_generator']['search_data_file']))
+      $logger.error('path to the search data file does not exits. Please create the directory first')
+    end
+    
+    unless (base_directory_exists?($config['search_generator']['output_frequency_to']))
+      $logger.error('path to the frequency file does not exits. Please create the directory first')
+    end
+  end
+  
+  private
+  def base_directory_exists?(file)
+    FileTest.directory?(File.dirname(file))
+  end 
+  def directory_exists?(dir)
+    FileTest.directory?(dir)
+  end 
+end

data/lib/search_generator.rb

@@ -0,0 +1,112 @@
+# generates the data for the search
+#
+# * $Author$
+# * $Rev$
+# * $LastChangedDate$
+#
+
+class SearchGenerator
+  # needs files and terms and an entry in the config representing the location of the javascript file
+  def initialize(files, terms)
+    @files = files
+    @terms = terms
+    $logger.info("writing data to #{$config['search_generator']['search_data_file']}")
+    @search_data_file = File.new($config['search_generator']['search_data_file'],'w')
+  end
+  
+  # generates the search data
+  def generate
+    ($config['search_generator']['use_double_metaphone'] && $config['search_generator']['use_double_metaphone'] == true)? generate_terms_for_dm : generate_terms
+    generate_files
+    generate_relative_path
+    generate_frequency_file if ($config['search_generator']['output_frequency_to'])
+    generate_double_metaphone if ($config['search_generator']['use_double_metaphone'] && $config['search_generator']['use_double_metaphone'] == true)
+    cleanup    
+  end
+  
+  private
+  # generates a javascript hash of the indexed terms and writes it to the javascript file
+  # term => document id, rank
+  def generate_terms
+    $logger.info("generating term base")
+    out = Array.new
+    out << "var terms = {"
+    @terms.each do |term, reference|
+      out << "'#{term}':["
+      docs = Hash.new
+      reference.each { |r| docs.has_key?(r.document.ID)? docs[r.document.ID]+=r.rank : docs[r.document.ID] = r.rank }
+			# because of a javascript performance issue with nested arrays, the page id and the page rank are put into a string and split in the javascript search an demand
+      docs.sort{ |a,b| a[1]<=>b[1]}.reverse.each{ |doc_ID, rank| out << "'#{doc_ID}-#{rank}',"}
+      out << "],"
+    end
+    @search_data_file.puts out.join.gsub(',]',']')[0..-2] + "};"
+  end
+	
+
+  # generates a javascript hash of the indexed terms and writes it to the javascript file
+  # term => document id, rank
+  def generate_terms_for_dm
+    $logger.info("generating term base")
+    outTerms = Array.new
+    outTerms << "var terms = {"
+    out = Array.new
+		out << "var ranks = ["
+		i = 0
+    @terms.each do |term, reference|
+      outTerms << "'#{term}':#{i},"
+			i += 1
+			out<<"["
+      docs = Hash.new
+      reference.each { |r| docs.has_key?(r.document.ID)? docs[r.document.ID]+=r.rank : docs[r.document.ID] = r.rank }
+			# because of a javascript performance issue with nested arrays, the page id and the page rank are put into a string and split in the javascript search an demand
+      docs.sort{ |a,b| a[1]<=>b[1]}.reverse.each{ |doc_ID, rank| out << "'#{doc_ID}-#{rank}',"}
+      out << "],"
+    end
+		@search_data_file.puts outTerms.join.gsub(',]',']')[0..-2] + '};'
+    @search_data_file.puts out.join.gsub(',]',']')[0..-2] + "];"
+  end
+  
+  # generates a javascript hash of file ids => title, file name, pagerank
+  def generate_files
+    $logger.info("generating file base")
+    out = Array.new
+    out << "var files = {"
+    @files.each_value do |f|
+      out << "#{f.ID}:[\"#{f.title}\",'#{f.name[1..-1]}',#{f.page_rank}],"
+    end
+    @search_data_file.puts out.join[0..-2] + "};"    
+  end
+  
+  # stores the relative path in a vairable
+  def generate_relative_path
+    $logger.info("generating relative path")
+    @search_data_file.puts "var rel_path = '#{$config['search_generator']['relative_path_to_files'].gsub(/\/$/,'')}/';" if $config['search_generator'].has_key?('relative_path_to_files')
+  end
+  
+  def generate_frequency_file
+    $logger.info("generating frequency file")
+    File.open($config['search_generator']['output_frequency_to'],'w') do |f|
+      @terms.each do |term,reference|
+        f.puts "#{term} #{reference.size}"
+      end
+      
+    end
+  end
+
+  def generate_double_metaphone()
+    $logger.info("generating double metaphone data")
+    require 'Text'
+    out = Array.new
+    out << 'var dm_data = ['
+    @terms.each do |t,r|
+      temp =  Text::Metaphone.double_metaphone(t)
+      out <<  "['#{temp[0]}'#{(temp[1])? ',\''+temp[1]+'\'':nil}],"
+		end
+    @search_data_file.puts out.join[0..-2] + "];"
+  end
+  
+  # performs cleanup operations
+  def cleanup
+    @search_data_file.close
+  end
+end

data/lib/stop_words.rb

@@ -0,0 +1,7 @@
+$stop_words = Hash.new
+
+File.open($config['crawler']['stopwords']) do |f|
+  while line = f.gets
+    $stop_words[line.chomp]=nil
+  end
+end

data/lib/stoplist/english/stopwords.txt

@@ -0,0 +1,317 @@
+a
+abandon
+abandoned
+abc
+able
+about
+above
+absence
+absent
+absolute
+absolutely
+ac
+accordance
+according
+accordingly
+acct
+across
+active
+actively
+actual
+actually
+ad
+adb
+adds
+adequate
+adequately
+adjacent
+adversely
+after
+afterwards
+again
+against
+ago
+ahead
+alike
+all
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+an
+analog
+and
+another
+any
+anybody
+anymore
+anyone
+anything
+anyway
+anywhere
+apart
+approx
+are
+around
+as
+aside
+at
+auto
+avail
+available
+aware
+away
+awhile
+badly
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+being
+below
+beside
+besides
+better
+between
+beyond
+brief
+briefly
+but
+by
+bye
+can
+come
+comes
+comfortable
+common
+commonly
+completely
+cons
+continually
+continue
+continued
+continues
+continuing
+continuous
+continuously
+could
+couple
+course
+current
+currently
+definitely
+despite
+did
+directly
+do
+does
+doing
+done
+during
+each
+either
+empty
+enough
+even
+eventually
+ever
+every
+everybody
+everyone
+everything
+everywhere
+ex
+exact
+exactly
+for
+from
+generally
+had
+has
+have
+he
+hello
+her
+hereby
+herein
+hi
+him
+himself
+his
+how
+however
+ideally
+if
+in
+indeed
+inner
+into
+is
+it
+its
+itself
+just
+like
+lot
+lots
+many
+may
+maybe
+me
+mean
+mere
+merely
+might
+more
+most
+mostly
+must
+my
+myself
+neither
+not
+obvious
+obviously
+of
+off
+on
+once
+one
+ones
+ongoing
+only
+onto
+other
+others
+otherwise
+ought
+our
+ours
+out
+outer
+over
+particular
+particularly
+please
+previous
+previously
+ready
+really
+recent
+recently
+relative
+relatively
+same
+see
+seldom
+self
+serious
+seriously
+set
+similar
+since
+sincerely
+so
+some
+somebody
+someday
+somehow
+someone
+someplace
+something
+sometime
+sometimes
+somewhat
+somewhere
+sorely
+sorry
+successful
+successfully
+such
+suddenly
+suitable
+sure
+surely
+than
+thank
+thanks
+that
+thats
+the
+their
+them
+theme
+themselves
+then
+there
+thereby
+therefore
+these
+they
+this
+thorough
+thoroughly
+those
+though
+through
+throughout
+throughput
+to
+today
+together
+total
+totally
+toward
+towards
+typical
+typically
+until
+up
+upon
+usually
+various
+very
+want
+was
+we
+well
+went
+were
+what
+whatever
+whats
+whatsoever
+when
+where
+which
+while
+who
+whoever
+whole
+whom
+whose
+why
+will
+with
+within
+without
+worth
+worthwhile
+worthy
+would
+yes
+yet
+you
+your
+yours
+yourself