picky 0.3.0 → 0.9.0

data/lib/picky/sources/csv.rb

@@ -1,5 +1,3 @@
-require 'csv'
-
 module Sources
   
   # Describes a CSV source, a file with csv in it.
@@ -12,6 +10,7 @@ module Sources
     attr_reader :file_name, :field_names
     
     def initialize *field_names, options
+      require 'csv'
       @field_names = field_names
       @file_name   = Hash === options && options[:file] || raise_no_file_given(field_names)
     end

data/lib/picky/sources/db.rb

@@ -50,7 +50,6 @@ module Sources
     #
     def connect_backend
       configure @options
-      return if PICKY_ENVIRONMENT.to_s == 'test' # TODO Unclean.
       raise "Database backend not configured" unless connection_options
       database.establish_connection connection_options
     end

data/lib/picky/sources/delicious.rb

@@ -0,0 +1,41 @@
+module Sources
+  
+  class Delicious < Base
+    
+    def initialize username, password
+      require 'www/delicious'
+      @username = username
+      @password = password
+    end
+    
+    # Harvests the data to index.
+    #
+    def harvest _, field
+      get_data do |uid, data|
+        indexed_id = uid
+        text = data[field.name]
+        next unless text
+        text.force_encoding 'utf-8' # TODO Still needed?
+        yield indexed_id, text
+      end
+    end
+    
+    #
+    #
+    def get_data
+      @generated_id ||= 0
+      @posts ||= WWW::Delicious.new(@username, @password).posts_recent(:count => 100)
+      @posts.each do |post|
+        data = {
+          :title => post.title,
+          :tags  => post.tags.join(' '),
+          :url   => post.url.to_s
+        }
+        @generated_id += 1
+        yield @generated_id, data
+      end
+    end
+    
+  end
+  
+end

data/lib/picky/tokenizers/base.rb

@@ -4,75 +4,80 @@ module Tokenizers
   #
   class Base
     
+    # TODO use frozen EMPTY_STRING for ''
+    #
+    
     # Stopwords.
     #
-    def self.stopwords regexp
-      define_method :remove_stopwords do |text|
-        text.gsub! regexp, ''
-      end
-      # Use this method if you don't want to remove
-      # stopwords if it is just one word.
-      #
-      @@non_single_stopword_regexp = /^\b[\w:]+?\b[\.\*\~]?\s?$/
-      define_method :remove_non_single_stopwords do |text|
-        return text if text.match @@non_single_stopword_regexp
-        remove_stopwords text
-      end
+    def stopwords regexp
+      @remove_stopwords_regexp = regexp
+    end
+    def remove_stopwords text
+      text.gsub! @remove_stopwords_regexp, '' if @remove_stopwords_regexp
+      text
+    end
+    @@non_single_stopword_regexp = /^\b[\w:]+?\b[\.\*\~]?\s?$/
+    def remove_non_single_stopwords text
+      return text if text.match @@non_single_stopword_regexp
+      remove_stopwords text
     end
-    def remove_stopwords text; end
     
     # Contraction.
     #
-    def self.contracts_expressions what, to_what
-      define_method :contract do |text|
-        text.gsub! what, to_what
-      end
+    def contracts_expressions what, to_what
+      @contract_what    = what
+      @contract_to_what = to_what
+    end
+    def contract text
+      text.gsub! @contract_what, @contract_to_what if @contract_what
     end
-    def contract text; end
     
     # Illegals.
     #
     # TODO Should there be a legal?
     #
-    def self.removes_characters regexp
-      define_method :remove_illegals do |text|
-        text.gsub! regexp, ''
-      end
+    def removes_characters regexp
+      @removes_characters_regexp = regexp
+    end
+    def remove_illegals text
+      text.gsub! @removes_characters_regexp, '' if @removes_characters_regexp
+      text
     end
-    def remove_illegals text; end
     
     # Splitting.
     #
-    def self.splits_text_on regexp
-      define_method :split do |text|
-        text.split regexp
-      end
+    def splits_text_on regexp
+      @splits_text_on_regexp = regexp
+    end
+    def split text
+      text.split @splits_text_on_regexp
     end
-    def split text; end
     
     # Normalizing.
     #
-    def self.normalizes_words regexp_replaces
-      define_method :normalize_with_patterns do |text|
-        regexp_replaces.each do |regex, replace|
-          # This should be sufficient
-          #
-          text.gsub!(regex, replace) and break
-        end
-        remove_after_normalizing_illegals text
-        text
+    def normalizes_words regexp_replaces
+      @normalizes_words_regexp_replaces = regexp_replaces
+    end
+    def normalize_with_patterns text
+      return text unless @normalizes_words_regexp_replaces
+      
+      @normalizes_words_regexp_replaces.each do |regex, replace|
+        # This should be sufficient
+        #
+        text.gsub!(regex, replace) and break
       end
+      remove_after_normalizing_illegals text
+      text
     end
-    def normalize_with_patterns text; end
     
     # Illegal after normalizing.
     #
-    def self.removes_characters_after_splitting regexp
-      define_method :remove_after_normalizing_illegals do |text|
-        text.gsub! regexp, ''
-      end
+    def removes_characters_after_splitting regexp
+      @removes_characters_after_splitting_regexp = regexp
+    end
+    def remove_after_normalizing_illegals text
+      text.gsub! @removes_characters_after_splitting_regexp, '' if @removes_characters_after_splitting_regexp
     end
-    def remove_after_normalizing_illegals text; end
     
     # Returns a number of tokens, generated from the given text.
     #
@@ -93,6 +98,10 @@ module Tokenizers
     
     def initialize substituter = UmlautSubstituter.new
       @substituter = substituter
+      
+      # TODO Default handling.
+      #
+      splits_text_on(/\s/)
     end
     
     # Hooks.

data/lib/picky/tokenizers/default/index.rb

@@ -0,0 +1,7 @@
+module Tokenizers
+  module Default
+    # Default is always an instance.
+    #
+    Index = ::Tokenizers::Index.new
+  end
+end

data/lib/picky/tokenizers/default/query.rb

@@ -0,0 +1,7 @@
+module Tokenizers
+  module Default
+    # Default is always an instance.
+    #
+    Query = ::Tokenizers::Query.new
+  end
+end

data/lib/picky/tokenizers/index.rb

@@ -5,15 +5,6 @@ module Tokenizers
   #
   class Index < Base
     
-    # Default handling definitions. Override in config.
-    #
-    removes_characters(//)
-    stopwords(//)
-    contracts_expressions(//, '')
-    splits_text_on(/\s/)
-    normalizes_words([])
-    removes_characters_after_splitting(//)
-    
     # Default indexing preprocessing hook.
     #
     # Does:

data/lib/picky/tokenizers/query.rb

@@ -13,15 +13,6 @@ module Tokenizers
   #
   class Query < Base
     
-    # Default query tokenizer behaviour. Override in config.
-    #
-    removes_characters(//)
-    stopwords(//)
-    contracts_expressions(//, '')
-    splits_text_on(/\s/)
-    normalizes_words([])
-    removes_characters_after_splitting(//)
-    
     def preprocess text
       remove_illegals text             # Remove illegal characters
       remove_non_single_stopwords text # remove stop words

data/lib/tasks/application.rake

@@ -1,4 +1,4 @@
-desc "Loads the application, including its configuration."
+# desc "Loads the application, including its configuration."
 task :application => :framework do
   puts "Running rake task 'application'."
   Loader.load_application

data/lib/tasks/cache.rake

@@ -1,53 +1,46 @@
 namespace :cache do
   
-  namespace :structure do
-    
-    desc 'create the directory structure for the cache indexes'
-    task :create => :application do
-      Indexes.create_directory_structure
-      puts "Directory structure generated."
-    end
-    
-  end
-  
-  desc "Generates the index cache files."
-  task :generate => :application do
-    Indexes.generate_caches
-    puts "Caches generated."
-  end
+  # Move to index namespace.
+  #
+  
+  # desc "Generates the index cache files."
+  # task :generate => :application do
+  #   Indexes.generate_caches
+  #   puts "Caches generated."
+  # end
 
-  desc "Generates a specific index cache file like field=books:title. Note: Index tables need to be there. Will generate just the cache."
-  task :only => :application do
-    type_and_field = ENV['FIELD'] || ENV['field']
-    type, field = type_and_field.split ':'
-    Indexes.generate_cache_only type.to_sym, field.to_sym
-  end
-  
-  
-  desc 'Checks the index cache files'
-  task :check => :application do
-    Indexes.check_caches
-    puts "All caches look ok."
-  end
-  
-  
-  desc "Removes the index cache files."
-  task :clear => :application do
-    Indexes.clear_caches
-    puts "All index cache files removed."
-  end
-  
-  
-  desc 'Backup the index cache files'
-  task :backup => :application do
-    Indexes.backup_caches
-    puts "Index cache files moved to the backup directory"
-  end
-  
-  desc 'Restore the index cache files'
-  task :restore => :application do
-    Indexes.restore_caches
-    puts "Index cache files restored from the backup directory"
-  end
+  # desc "Generates a specific index cache file like field=books:title. Note: Index tables need to be there. Will generate just the cache."
+  # task :only => :application do
+  #   type_and_field = ENV['FIELD'] || ENV['field']
+  #   type, field = type_and_field.split ':'
+  #   Indexes.generate_cache_only type.to_sym, field.to_sym
+  # end
+  
+  
+  # desc 'Checks the index cache files'
+  # task :check => :application do
+  #   Indexes.check_caches
+  #   puts "All caches look ok."
+  # end
+  
+  
+  # desc "Removes the index cache files."
+  # task :clear => :application do
+  #   Indexes.clear_caches
+  #   puts "All index cache files removed."
+  # end
+  
+  
+  # desc 'Backup the index cache files'
+  # task :backup => :application do
+  #   Indexes.backup_caches
+  #   puts "Index cache files moved to the backup directory"
+  # end
+  
+  # desc 'Restore the index cache files'
+  # task :restore => :application do
+  #   Indexes.restore_caches
+  #   puts "Index cache files restored from the backup directory"
+  # end
   
 end

data/lib/tasks/framework.rake

@@ -1,4 +1,4 @@
-desc "Loads the framework."
+# desc "Loads the framework."
 task :framework do
   require File.expand_path '../../picky', __FILE__
 end

data/lib/tasks/index.rake

@@ -2,24 +2,34 @@
 #
 namespace :index do
   
-  desc "Generates the temp tables: Copies the data from the public entries."
-  task :generate_temp_tables => :application do
-    Indexes.take_snapshot
+  # Not necessary anymore.
+  #
+  # namespace :directories do
+  #   desc 'Creates the directory structure for the indexes.'
+  #   task :make => :application do
+  #     Indexes.create_directory_structure
+  #     puts "Directory structure generated."
+  #   end
+  # end
+  
+  desc "Takes a snapshot, indexes, and caches."
+  task :generate, [:order] => :application do |_, options|
+    randomly = (options.order == 'ordered') ? false : true
+    Indexes.index randomly
   end
   
-  desc "rake index:tables:update"
-  task :prepare => :application do
+  desc "Generates the index snapshots."
+  task :generate_snapshots => :application do
     Indexes.take_snapshot
-    Indexes.configuration.index
   end
   
-  desc "E.g. Generates a specific index table. Note: temp tables need to be there. Will generate just the index table."
-  task :only, [:type, :field] => :application do |_, options|
-    type, field = options.type, options.field
-    Indexes.generate_index_only type.to_sym, field.to_sym
-  end
+  # desc "E.g. Generates a specific index table. Note: intermediate indexes need to be there."
+  # task :only, [:type, :field] => :application do |_, options|
+  #   type, field = options.type, options.field
+  #   Indexes.generate_index_only type.to_sym, field.to_sym
+  # end
   
-  desc "E.g. Generates a specific index cache file. Note: temp tables need to be there. Will generate index table and cache."
+  desc "Generates a specific index from index snapshots."
   task :specific, [:type, :field] => :application do |_, options|
     type, field = options.type, options.field
     Indexes.generate_index_only type.to_sym, field.to_sym

data/lib/tasks/server.rake

@@ -11,7 +11,7 @@ namespace :server do
     pid.blank? ? nil : pid.chomp
   end
   
-  desc "Start the unicorns. Weheee!"
+  desc "Start the unicorns. (Wehee!)"
   task :start => :framework do
     chdir_to_root
     # Rake::Task[:"solr:start"].invoke # TODO Move to better place.
@@ -21,13 +21,13 @@ namespace :server do
     exec command
   end
   
-  desc "Stop the unicorns. Blam!"
+  desc "Stop the unicorns. (Blam!)"
   task :stop => :framework do
     `kill -QUIT #{current_pid}` if current_pid
     # Rake::Task[:"solr:stop"].invoke # TODO Move to better place.
   end
   
-  desc "Restart the unicorns!"
+  desc "Restart the unicorns."
   task :restart do
     Rake::Task[:"server:stop"].invoke
     sleep 5

data/lib/tasks/shortcuts.rake

@@ -1,6 +1,13 @@
-desc "Shortcut for indexing and caching."
+desc "Shortcut for index:generate."
 task :index => :application do
-  Indexes.index
+  Rake::Task[:'index:generate'].invoke
+end
+
+desc "Shortcut for try:both"
+task :try, [:text, :type_and_field] => :application do |_, options|
+  text, type_and_field = options.text, options.type_and_field
+  
+  Rake::Task[:'try:both'].invoke text, type_and_field
 end
 
 desc "shortcut for server:start"

data/lib/tasks/statistics.rake

@@ -1,13 +1,13 @@
 namespace :statistics do
 
-  desc "start the server"
-  task :start => :application do
-    Statistics.start unless PICKY_ENVIRONMENT == 'test'
-  end
+  # desc "start the server"
+  # task :start => :application do
+  #   Statistics.start unless PICKY_ENVIRONMENT == 'test'
+  # end
   
-  desc "stop the server"
-  task :stop => :application do
-    Statistics.stop unless PICKY_ENVIRONMENT == 'test'
-  end
+  # desc "stop the server"
+  # task :stop => :application do
+  #   Statistics.stop unless PICKY_ENVIRONMENT == 'test'
+  # end
   
 end

data/lib/tasks/try.rake

@@ -6,7 +6,7 @@ namespace :try do
   task :index, [:text, :type_and_field] => :application do |_, options|
     text, type_and_field = options.text, options.type_and_field
     
-    tokenizer = type_and_field ? Indexes.find(*type_and_field.split(':')).tokenizer : Tokenizers::Index.new
+    tokenizer = type_and_field ? Indexes.find(*type_and_field.split(':')).tokenizer : Tokenizers::Default::Index
     
     puts "\"#{text}\" is index tokenized as #{tokenizer.tokenize(text).to_a}"
   end
@@ -17,7 +17,9 @@ namespace :try do
     
     # TODO tokenize destroys the original text...
     #
-    puts "\"#{text}\" is query tokenized as #{Tokenizers::Query.new.tokenize(text.dup).to_a.map(&:to_s)}"
+    # TODO Use the Query Tokenizer.
+    #
+    puts "\"#{text}\" is query tokenized as #{Tokenizers::Default::Query.tokenize(text.dup).to_a.map(&:to_s).map(&:to_sym)}"
   end
   
   desc "Try the given text with both the index and the query (type:field optional)."

data/project_prototype/Gemfile

@@ -2,7 +2,7 @@ source :gemcutter
 
 # Gems required by Picky.
 #
-gem 'picky',            '~> 0.3.0'
+gem 'picky',            '~> 0.9.0'
 gem 'bundler',          '>= 0.9.26'
 gem 'rack',             '~> 1.2.1'
 gem 'rack-mount',       '~> 0.6.9'

data/project_prototype/app/application.rb

@@ -18,9 +18,13 @@ class PickySearch < Application
                       Sources::CSV.new(:title, :author, :isbn, :year, :publisher, :subjects, :file => 'app/library.csv'),
                       # Use a database as source:
                       # Sources::DB.new('SELECT id, title, author, isbn13 as isbn FROM books', :file => 'app/db.yml'), 
-                      field(:title, :similarity => Similarity::DoubleLevenshtone.new(3)), # Up to three similar title word indexed.
-                      field(:author),
-                      field(:isbn,  :partial => Partial::None.new) # Partially searching on an ISBN makes not much sense.
+                      field(:title,
+                            :partial => Partial::Subtoken.new(:down_to => 1), # Index partial down to character 1 (default: -3),
+                                                                              # e.g. florian -> floria, flori, flor, flo, fl, f
+                                                                              # Like this, you'll find florian even when entering just an "f".
+                            :similarity => Similarity::DoubleLevenshtone.new(3)), # Up to three similar title word indexed.
+                      field(:author, :partial => Partial::Subtoken.new(:down_to => 1)),
+                      field(:isbn,   :partial => Partial::None.new) # Partially searching on an ISBN makes not much sense, neither does similarity.
   
   # Defines the maximum tokens (words) that pass through to the engine.
   #

data/spec/lib/cacher/partial/default_spec.rb

@@ -6,7 +6,7 @@ describe Cacher::Partial::Default do
     Cacher::Partial::Default.should be_kind_of(Cacher::Partial::Subtoken)
   end
   it "should be a the right down to" do
-    Cacher::Partial::Default.down_to.should == 1
+    Cacher::Partial::Default.down_to.should == -3
   end
   it "should be a the right starting at" do
     Cacher::Partial::Default.starting_at.should == -1

data/spec/lib/cacher/partial/none_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+
+describe Cacher::Partial::None do
+  
+  it "has the right superclass" do
+    Cacher::Partial::None.should < Cacher::Partial::Strategy
+  end
+  it "returns an empty index" do
+    Cacher::Partial::None.new.generate_from(:unimportant).should == {}
+  end
+  
+end

data/spec/lib/cacher/partial/subtoken_spec.rb

@@ -122,7 +122,7 @@ describe Cacher::Partial::Subtoken do
         end
       end
     end
-    context 'starting_at -1' do
+    context 'starting_at set' do
       before(:each) do
         @cacher = Cacher::Partial::Subtoken.new :down_to => 4, :starting_at => -2
       end
@@ -148,6 +148,34 @@ describe Cacher::Partial::Subtoken do
         end
       end
     end
+    context 'starting_at set' do
+      before(:each) do
+        @cacher = Cacher::Partial::Subtoken.new :down_to => 4, :starting_at => 0
+      end
+      describe 'starting_at' do
+        it 'should return the right value' do
+          @cacher.starting_at.should == 0
+        end
+      end
+      describe 'down_to' do
+        it 'should return the right value' do
+          @cacher.down_to.should == 4
+        end
+      end
+      describe 'generate_from' do
+        it 'should generate the right index' do
+          @cacher.generate_from( :florian => [1], :flavia => [2] ).should == {
+            :florian => [1],
+            :floria => [1],
+            :flori => [1],
+            :flor => [1],
+            :flavia => [2],
+            :flavi => [2],
+            :flav => [2]
+          }
+        end
+      end
+    end
   end
   
 end