xapian_db 0.3.2 → 0.3.3

data/lib/xapian_db/stopwords/update_stopwords.rb

@@ -0,0 +1,35 @@
+require "fileutils"
+
+LANGUAGE_MAP = {:danish     => :da,
+                :dutch      => :nl,
+                :english    => :en,
+                :finnish    => :fi,
+                :french     => :fr,
+                :german     => :de,
+                :hungarian  => :hu,
+                :italian    => :it,
+                :norwegian  => :no,
+                :portuguese => :pt,
+                :russian    => :ru,
+                :spanish    => :es,
+                :swedish    => :sv}
+
+# 1. Load the stop words files from snowball.tartarus.org
+LANGUAGE_MAP.keys.reject{|k| k == :russian}.each { |l| system("curl http://snowball.tartarus.org/algorithms/%s/stop.txt | iconv -f ISO-8859-1 -t UTF-8 > %s.txt" % [l, l]) }
+system("curl http://snowball.tartarus.org/algorithms/russian/stop.txt | iconv -f KOI8-R -t UTF-8 > russian.txt")
+
+# 2. Clean up the files (remove comments) and write a new file with the iso name
+LANGUAGE_MAP.keys.each do |lang|
+  open("#{LANGUAGE_MAP[lang]}.txt", "w") do |outfile|
+    open("#{lang}.txt", "r") do |infile|
+      while line = infile.gets
+        outfile.puts line.split(" ", 2).first.downcase.strip  unless line =~ /^ +|^$|^\|/
+      end
+    end
+  end
+end
+
+# 3. Remove the downloaded files
+LANGUAGE_MAP.keys.each {|lang| FileUtils.rm_rf "#{lang}.txt"}
+
+

data/lib/xapian_db.rb

@@ -1,17 +1,35 @@
 # encoding: utf-8
 
-require 'digest/sha1'
-require 'xapian'
-require 'yaml'
-
 # This is the top level module of xapian_db. It allows you to
 # configure XapianDB, create / open databases and perform
 # searches.
 
 # @author Gernot Kogler
 
+require 'xapian'
+require 'yaml'
+
 module XapianDb
 
+  # Supported languages
+  LANGUAGE_MAP = {:da => :danish,
+                  :nl => :dutch,
+                  :en => :english,
+                  :fi => :finnish,
+                  :fr => :french,
+                  :de => :german2, # Normalises umlauts and ß
+                  :hu => :hungarian,
+                  :it => :italian,
+                  :nb => :norwegian,
+                  :nn => :norwegian,
+                  :no => :norwegian,
+                  :pt => :portuguese,
+                  :ro => :romanian,
+                  :ru => :russian,
+                  :es => :spanish,
+                  :sv => :swedish,
+                  :tr => :turkish}
+
   # Global configuration for XapianDb. See {XapianDb::Config.setup}
   # for available options
   def self.setup(&block)
@@ -61,16 +79,9 @@ module XapianDb
 
 end
 
-require File.dirname(__FILE__) + '/xapian_db/config'
-require File.dirname(__FILE__) + '/xapian_db/adapters/generic_adapter'
-require File.dirname(__FILE__) + '/xapian_db/adapters/datamapper_adapter'
-require File.dirname(__FILE__) + '/xapian_db/adapters/active_record_adapter'
-require File.dirname(__FILE__) + '/xapian_db/index_writers/direct_writer'
-require File.dirname(__FILE__) + '/xapian_db/database'
-require File.dirname(__FILE__) + '/xapian_db/document_blueprint'
-require File.dirname(__FILE__) + '/xapian_db/indexer'
-require File.dirname(__FILE__) + '/xapian_db/query_parser'
-require File.dirname(__FILE__) + '/xapian_db/resultset'
+do_not_require = %w(update_stopwords.rb railtie.rb)
+files = Dir.glob("#{File.dirname(__FILE__)}/**/*.rb").reject{|path| do_not_require.include?(File.basename(path))}
+files.each {|file| require file}
 
 # Configure XapianDB if we are in a Rails app
 require File.dirname(__FILE__) + '/xapian_db/railtie' if defined?(Rails)

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 3
-  - 2
-  version: 0.3.2
+  - 3
+  version: 0.3.3
 platform: ruby
 authors: 
 - Gernot Kogler
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-10 00:00:00 +01:00
+date: 2010-12-13 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -66,7 +66,24 @@ files:
 - lib/xapian_db/indexer.rb
 - lib/xapian_db/query_parser.rb
 - lib/xapian_db/railtie.rb
+- lib/xapian_db/repositories/stemmer.rb
+- lib/xapian_db/repositories/stopper.rb
 - lib/xapian_db/resultset.rb
+- lib/xapian_db/stopwords/da.txt
+- lib/xapian_db/stopwords/de.txt
+- lib/xapian_db/stopwords/en.txt
+- lib/xapian_db/stopwords/es.txt
+- lib/xapian_db/stopwords/fi.txt
+- lib/xapian_db/stopwords/fr.txt
+- lib/xapian_db/stopwords/hu.txt
+- lib/xapian_db/stopwords/it.txt
+- lib/xapian_db/stopwords/nl.txt
+- lib/xapian_db/stopwords/no.txt
+- lib/xapian_db/stopwords/pt.txt
+- lib/xapian_db/stopwords/README
+- lib/xapian_db/stopwords/ru.txt
+- lib/xapian_db/stopwords/sv.txt
+- lib/xapian_db/stopwords/update_stopwords.rb
 - lib/xapian_db.rb
 - LICENSE
 - README.rdoc