picky 0.11.2 → 0.12.0

data/lib/picky/indexing/{types.rb → indexes.rb}

@@ -1,8 +1,8 @@
 module Indexing
   
-  class Types
+  class Indexes
     
-    attr_reader :types
+    attr_reader :indexes
     
     each_delegate :take_snapshot,
                   :generate_caches,
@@ -11,7 +11,7 @@ module Indexing
                   :check_caches,
                   :clear_caches,
                   :create_directory_structure,
-                  :to => :types
+                  :to => :indexes
     
     def initialize
       clear
@@ -20,13 +20,13 @@ module Indexing
     # TODO Spec.
     #
     def clear
-      @types = []
+      @indexes = []
     end
     
     # TODO Spec. Superclass?
     #
-    def register type
-      self.types << type
+    def register index
+      self.indexes << index
     end
     
     # Runs the indexers in parallel (index + cache).
@@ -39,9 +39,9 @@ module Indexing
       # Run in parallel.
       #
       timed_exclaim "INDEXING USING #{Cores.max_processors} PROCESSORS, IN #{randomly ? 'RANDOM' : 'GIVEN'} ORDER."
-      Cores.forked self.types, { randomly: randomly } do |type|
-        type.index
-        type.cache
+      Cores.forked self.indexes, { randomly: randomly } do |an_index|
+        an_index.index
+        an_index.cache
       end
       timed_exclaim "INDEXING FINISHED."
     end
@@ -51,36 +51,36 @@ module Indexing
     def index_for_tests
       take_snapshot
       
-      self.types.each do |type|
-        type.index
-        type.cache
+      self.indexes.each do |an_index|
+        an_index.index
+        an_index.cache
       end
     end
     
     # TODO Spec
     #
-    def generate_index_only type_name, category_name
-      found = find type_name, category_name
+    def generate_index_only index_name, category_name
+      found = find index_name, category_name
       found.index if found
     end
-    def generate_cache_only type_name, category_name
-      found = find type_name, category_name
+    def generate_cache_only index_name, category_name
+      found = find index_name, category_name
       found.generate_caches if found
     end
     
     # TODO Spec
     #
-    def find type_name, category_name
-      type_name     = type_name.to_sym
+    def find index_name, category_name
+      index_name     = index_name.to_sym
       
-      types.each do |type|
-        next unless type.name == type_name
+      indexes.each do |index|
+        next unless index.name == index_name
         
-        found = type.categories.find category_name
+        found = index.categories.find category_name
         return found if found
       end
       
-      raise %Q{Index "#{type_name}" not found. Possible indexes: "#{types.map(&:name).join('", "')}".}
+      raise %Q{Index "#{index_name}" not found. Possible indexes: "#{indexes.map(&:name).join('", "')}".}
     end
     
   end

data/lib/picky/loader.rb

@@ -104,9 +104,9 @@ module Loader
     load_relative 'helpers/cache'
     load_relative 'helpers/measuring'
     
-    # Character Substitution
+    # Character Substituters
     #
-    load_relative 'character_substitution/european'
+    load_relative 'character_substituters/west_european'
     
     # Signal handling
     #
@@ -119,8 +119,7 @@ module Loader
     # Index generation strategies.
     #
     load_relative 'indexers/no_source_specified_error'
-    load_relative 'indexers/base'
-    load_relative 'indexers/default'
+    load_relative 'indexers/serial'
     #
     # load_relative 'indexers/solr'
     
@@ -132,7 +131,7 @@ module Loader
     #
     load_relative 'cacher/partial/strategy'
     load_relative 'cacher/partial/none'
-    load_relative 'cacher/partial/subtoken'
+    load_relative 'cacher/partial/substring'
     load_relative 'cacher/partial/default'
 
     # Weight index generation strategies.
@@ -167,27 +166,27 @@ module Loader
     load_relative 'index/file/json'
     load_relative 'index/files'
     
-    # Index types.
+    # Indexing and Indexed things.
     #
-    load_relative 'bundle'
+    load_relative 'index/bundle'
     
     load_relative 'indexing/bundle'
     load_relative 'indexing/category'
     load_relative 'indexing/categories'
-    load_relative 'indexing/type'
-    load_relative 'indexing/types'
+    load_relative 'indexing/index'
+    load_relative 'indexing/indexes'
     
-    load_relative 'index/bundle'
-    load_relative 'index/category'
-    load_relative 'index/categories'
-    load_relative 'index/type'
-    load_relative 'index/types'
+    load_relative 'indexed/bundle'
+    load_relative 'indexed/category'
+    load_relative 'indexed/categories'
+    load_relative 'indexed/index'
+    load_relative 'indexed/indexes'
     
-    load_relative 'types'
+    load_relative 'indexes_api'
     load_relative 'alias_instances'
-    load_relative 'type'
+    load_relative 'index_api'
     
-    load_relative 'index/wrappers/exact_first'
+    load_relative 'indexed/wrappers/exact_first'
     
     # Tokens.
     #
@@ -240,11 +239,7 @@ module Loader
     
     # Configuration.
     #
-    load_relative 'configuration/indexes'
-    
-    # ... in Application.
-    #
-    load_relative 'configuration/queries'
+    load_relative 'configuration/index'
     
     # Application and routing.
     #

data/lib/picky/query/base.rb

@@ -19,7 +19,7 @@ module Query
     #
     def initialize *index_type_definitions
       options      = Hash === index_type_definitions.last ? index_type_definitions.pop : {}
-      indexes      = index_type_definitions.map &:index
+      indexes      = index_type_definitions.map &:indexed
       
       @weigher     = options[:weigher]   || Weigher.new(indexes)
       @tokenizer   = options[:tokenizer] || Tokenizers::Query.default

data/lib/picky/rack/harakiri.rb

@@ -4,8 +4,12 @@ module Rack
   #
   # Use as follows in e.g. your rackup File:
   #
-  # Rack::Harakiri.after = 50
-  # use Rack::Harakiri
+  #   Rack::Harakiri.after = 100
+  #   use Rack::Harakiri
+  #
+  # Then the Unicorn will commit suicide after 100 requests (50 is the default).
+  #
+  # The Master Unicorn process forks a new child Unicorn to replace the old one.
   #
   class Harakiri
     
@@ -21,6 +25,9 @@ module Rack
       @quit_after_requests = self.class.after || 50
     end
     
+    # Harakiri is a middleware, so it passes the call on after checking if it
+    # is time to honorably retire.
+    #
     def call env
       harakiri
       @app.call env

data/lib/picky/signals.rb

@@ -1,7 +1,7 @@
 # TODO Cleanup and move to project_prototype.
 #
 # Signal.trap 'USR1' do
-#   Indexes.reload
+#   Indexed.reload
 # end
 # Signal.trap 'USR2' do
 #   Loader.reload

data/lib/picky/sources/base.rb

@@ -10,22 +10,9 @@ module Sources
   # * take_snapshot: Optional, called once for each type.
   class Base
     
-    # Note: Methods listed for illustrative purposes.
+    # Note: Default methods do nothing.
     #
     
-    # Called by the indexer when gathering data.
-    #
-    # Yields the data (id, text for id) for the given type and field.
-    #
-    # When implementing or overriding your own,
-    # be sure to <tt>yield</tt> (or <tt>block.call</tt>) an id (as string or integer)
-    # and a corresponding text for the given type symbol and
-    # category symbol.
-    #
-    def harvest type, category
-      # yields nothing
-    end
-    
     # Connect to the backend.
     #
     # Note: Called once per index/category combination
@@ -47,6 +34,19 @@ module Sources
       
     end
     
+    # Called by the indexer when gathering data.
+    #
+    # Yields the data (id, text for id) for the given type and category.
+    #
+    # When implementing or overriding your own,
+    # be sure to <tt>yield</tt> (or <tt>block.call</tt>) an id (as string or integer)
+    # and a corresponding text for the given type symbol and
+    # category symbol.
+    #
+    def harvest type, category
+      # yields nothing
+    end
+    
   end
   
 end

data/lib/picky/sources/couch.rb

@@ -8,9 +8,9 @@ module Sources
 
   class Couch < Base
     
-    def initialize *field_names, options
+    def initialize *category_names, options
       check_gem
-      Hash === options && options[:url] || raise_no_db_given(field_names)
+      Hash === options && options[:url] || raise_no_db_given(category_names)
       @db = RestClient::Resource.new options.delete(:url), options
     end
     
@@ -23,9 +23,10 @@ module Sources
 
     # Harvests the data to index.
     #
-    def harvest type, field
+    def harvest type, category
+      category_name = category.name.to_s
       get_data do |doc|
-        yield doc['_id'].to_i, doc[field.name.to_s] || next
+        yield doc['_id'].to_i, doc[category_name] || next
       end
     end
 
@@ -35,9 +36,9 @@ module Sources
         map{|row| row['doc']}.
         each &block
     end
-
-    def raise_no_db_given field_names
-      raise NoCouchDBGiven.new(field_names.join(', '))
+    
+    def raise_no_db_given category_names
+      raise NoCouchDBGiven.new(category_names.join(', '))
     end
   end
 end

data/lib/picky/sources/csv.rb

@@ -1,32 +1,32 @@
 module Sources
   
   # Describes a CSV source, a file with csv in it.
-  # Give it a sequence of field names and a file option with the filename.
+  # Give it a sequence of category names and a file option with the filename.
   #
   class NoCSVFileGiven < StandardError; end
   
   class CSV < Base
     
-    attr_reader :file_name, :field_names
+    attr_reader :file_name, :category_names
     
-    def initialize *field_names, options
+    def initialize *category_names, options
       require 'csv'
-      @field_names = field_names
-      @file_name   = Hash === options && options[:file] || raise_no_file_given(field_names)
+      @category_names = category_names
+      @file_name   = Hash === options && options[:file] || raise_no_file_given(category_names)
     end
     
     #
     #
-    def raise_no_file_given field_names
-      raise NoCSVFileGiven.new(field_names.join(', '))
+    def raise_no_file_given category_names
+      raise NoCSVFileGiven.new(category_names.join(', '))
     end
     
     # Harvests the data to index.
     #
-    def harvest _, field
-      index = field_names.index field.name
+    def harvest _, category
+      index = category_names.index category.name
       get_data do |ary|
-        indexed_id = ary.shift.to_i
+        indexed_id = ary.shift.to_i # TODO is to_i necessary?
         text       = ary[index]
         next unless text
         text.force_encoding 'utf-8' # TODO Still needed?

data/lib/picky/sources/db.rb

@@ -93,11 +93,11 @@ module Sources
     # Example:
     #   "SELECT indexed_id, value FROM bla_table st WHERE kind = 'bla'"
     #
-    def harvest type, field
+    def harvest type, category
       connect_backend
       
       (0..count(type)).step(chunksize) do |offset|
-        get_data(type, field, offset).each do |indexed_id, text|
+        get_data(type, category, offset).each do |indexed_id, text|
           next unless text
           text.force_encoding 'utf-8' # TODO Still needed?
           yield indexed_id, text
@@ -107,16 +107,16 @@ module Sources
     
     # Gets database from the backend.
     #
-    def get_data type, field, offset
-      database.connection.execute harvest_statement_with_offset(type, field, offset)
+    def get_data type, category, offset
+      database.connection.execute harvest_statement_with_offset(type, category, offset)
     end
     
     # Builds a harvest statement for getting data to index.
     #
     # TODO Use the adapter for this.
     #
-    def harvest_statement_with_offset type, field, offset
-      statement = harvest_statement type, field
+    def harvest_statement_with_offset type, category, offset
+      statement = harvest_statement type, category
       
       statement += statement.include?('WHERE') ? ' AND' : ' WHERE'
       
@@ -125,8 +125,8 @@ module Sources
     
     # Base harvest statement for dbs.
     #
-    def harvest_statement type, field
-      "SELECT indexed_id, #{field.name} FROM #{snapshot_table_name(type)} st"
+    def harvest_statement type, category
+      "SELECT indexed_id, #{category.name} FROM #{snapshot_table_name(type)} st"
     end
     
     # Override in subclasses.

data/lib/picky/sources/delicious.rb

@@ -16,10 +16,10 @@ module Sources
     
     # Harvests the data to index.
     #
-    def harvest _, field
+    def harvest _, category
       get_data do |uid, data|
         indexed_id = uid
-        text = data[field.name]
+        text = data[category.name]
         next unless text
         text.force_encoding 'utf-8' # TODO Still needed?
         yield indexed_id, text

data/lib/picky/sources/wrappers/location.rb

@@ -38,9 +38,9 @@ module Sources
         @min = 1.0/0
       end
       
-      # Yield the data (id, text for id) for the given type and field.
+      # Yield the data (id, text for id) for the given type and category.
       #
-      def harvest type, field
+      def harvest type, category
         reset
         
         # Cache. TODO Make option?
@@ -49,7 +49,7 @@ module Sources
         
         # Gather min/max.
         #
-        backend.harvest type, field do |indexed_id, location|
+        backend.harvest type, category do |indexed_id, location|
           location = location.to_f
           @min = location if location < @min
           locations << [indexed_id, location]

data/lib/picky/tokenizers/base.rb

@@ -22,16 +22,6 @@ module Tokenizers
       remove_stopwords text
     end
     
-    # Contraction.
-    #
-    def contracts_expressions what, to_what
-      @contract_what    = what
-      @contract_to_what = to_what
-    end
-    def contract text
-      text.gsub! @contract_what, @contract_to_what if @contract_what
-    end
-    
     # Illegals.
     #
     # TODO Should there be a legal?
@@ -83,7 +73,7 @@ module Tokenizers
     #
     # Default is European Character substitution.
     #
-    def substitutes_characters_with substituter = CharacterSubstitution::European.new
+    def substitutes_characters_with substituter = CharacterSubstituters::WestEuropean.new
       # TODO Raise if it doesn't quack substitute?
       @substituter = substituter
     end

data/lib/picky/tokenizers/index.rb

@@ -25,7 +25,6 @@ module Tokenizers
       text = substitute_characters text
       text.downcase!
       remove_illegals text
-      contract text
       # we do not remove single stopwords for an entirely different
       # reason than in the query tokenizer.
       # An indexed thing with just name "UND" (a stopword) should not lose its name.

data/lib/picky/tokenizers/query.rb

@@ -30,7 +30,6 @@ module Tokenizers
     def preprocess text
       remove_illegals text             # Remove illegal characters
       remove_non_single_stopwords text # remove stop words
-      contract text                    # contract st sankt etc
       text
     end
     

data/lib/tasks/index.rake

@@ -17,10 +17,10 @@ namespace :index do
   end
   
   desc "Generates a specific index from index snapshots."
-  task :specific, [:type, :field] => :application do |_, options|
-    type, field = options.type, options.field
-    Indexes.generate_index_only type.to_sym, field.to_sym
-    Indexes.generate_cache_only type.to_sym, field.to_sym
+  task :specific, [:index, :category] => :application do |_, options|
+    index, category = options.index, options.category
+    Indexes.generate_index_only index.to_sym, category.to_sym
+    Indexes.generate_cache_only index.to_sym, category.to_sym
   end
   
   desc 'Checks the index files for files that are suspiciously small or missing.'

data/lib/tasks/shortcuts.rake

@@ -3,11 +3,11 @@ task :index => :application do
   Rake::Task[:'index:randomly'].invoke
 end
 
-desc "Try the given text in the indexer/query (type:field optional)."
-task :try, [:text, :type_and_field] => :application do |_, options|
-  text, type_and_field = options.text, options.type_and_field
+desc "Try the given text in the indexer/query (index:category optional)."
+task :try, [:text, :index_and_category] => :application do |_, options|
+  text, index_and_category = options.text, options.index_and_category
   
-  Rake::Task[:'try:both'].invoke text, type_and_field
+  Rake::Task[:'try:both'].invoke text, index_and_category
 end
 
 desc "Start the server."

data/lib/tasks/try.rake

@@ -2,11 +2,11 @@
 #
 namespace :try do
   
-  # desc "Try how a given word would be tokenized when indexing (type:field optional)."
-  task :index, [:text, :type_and_field] => :application do |_, options|
-    text, type_and_field = options.text, options.type_and_field
+  # desc "Try how a given word would be tokenized when indexing (type:category optional)."
+  task :index, [:text, :index_and_category] => :application do |_, options|
+    text, index_and_category = options.text, options.index_and_category
     
-    tokenizer = type_and_field ? Indexes.find(*type_and_field.split(':')).tokenizer : Tokenizers::Index.default
+    tokenizer = index_and_category ? Indexes.find(*index_and_category.split(':')).tokenizer : Tokenizers::Index.default
     
     puts "\"#{text}\" is index tokenized as #{tokenizer.tokenize(text.dup).to_a}"
   end
@@ -18,11 +18,11 @@ namespace :try do
     puts "\"#{text}\" is query tokenized as #{Tokenizers::Query.default.tokenize(text.dup).to_a.map(&:to_s).map(&:to_sym)}"
   end
   
-  # desc "Try the given text with both the index and the query (type:field optional)."
-  task :both, [:text, :type_and_field] => :application do |_, options|
-    text, type_and_field = options.text, options.type_and_field
+  # desc "Try the given text with both the index and the query (type:category optional)."
+  task :both, [:text, :index_and_category] => :application do |_, options|
+    text, index_and_category = options.text, options.index_and_category
     
-    Rake::Task[:"try:index"].invoke text, type_and_field
+    Rake::Task[:"try:index"].invoke text, index_and_category
     Rake::Task[:"try:query"].invoke text
   end
   

data/project_prototype/Gemfile

@@ -2,7 +2,7 @@ source :gemcutter
 
 # Gems required by Picky.
 #
-gem 'picky',            '~> 0.11.0'
+gem 'picky',            '~> 0.12.0'
 gem 'rake'
 gem 'bundler'
 gem 'rack',             '~> 1.2.1'

data/project_prototype/app/application.rb

@@ -9,32 +9,33 @@
 class PickySearch < Application
   
   # Indexing: How text is indexed.
-  # Querying: How query text is handled.
   #
   default_indexing removes_characters: /[^a-zA-Z0-9\s\/\-\"\&\.]/,
                    stopwords:          /\b(and|the|of|it|in|for)\b/,
                    splits_text_on:     /[\s\/\-\"\&\.]/
                    
+  # Querying: How query text is handled.
+  #
   default_querying removes_characters: /[^a-zA-Z0-9\s\/\-\,\&\"\~\*\:]/, # Picky needs control chars *"~: to pass through.
                    stopwords:          /\b(and|the|of|it|in|for)\b/,
                    splits_text_on:     /[\s\/\-\,\&]+/,
                    
-                   maximum_tokens: 5, # Max amount of tokens passing into a query. 5 is the default.
-                   substitutes_characters_with: CharacterSubstitution::European.new # Normalizes special user input, Ä -> Ae, ñ -> n etc.
+                   maximum_tokens: 5, # Amount of tokens passing into a query (5 = default).
+                   substitutes_characters_with: CharacterSubstituters::WestEuropean.new # Normalizes special user input, Ä -> Ae, ñ -> n etc.
                    
   # Define an index. Use a database etc. source?
   # See http://github.com/floere/picky/wiki/Sources-Configuration#sources
   #
   books_index = index :books, Sources::CSV.new(:title, :author, :isbn, file: 'app/library.csv')
-  books_index.category :title,
-                       similarity: Similarity::Phonetic.new(3), # Up to three similar title word indexed (default: No similarity).
-                       partial: Partial::Substring.new(from: 1) # Indexes substrings upwards from character 1 (default: -3),
-                                                                # You'll find "picky" even when entering just a "p".
-  books_index.category :author,
-                       partial: Partial::Substring.new(from: 1)
-  books_index.category :isbn,
-                       partial: Partial::None.new # Partial substring searching on an ISBN does not make
-                                                  # much sense, neither does similarity.
+  books_index.define_category :title,
+                              similarity: Similarity::Phonetic.new(3), # Up to three similar title word indexed (default: No similarity).
+                              partial: Partial::Substring.new(from: 1) # Indexes substrings upwards from character 1 (default: -3),
+                                                                       # You'll find "picky" even when entering just a "p".
+  books_index.define_category :author,
+                              partial: Partial::Substring.new(from: 1)
+  books_index.define_category :isbn,
+                              partial: Partial::None.new # Partial substring searching on an ISBN does not make
+                                                         # much sense, neither does similarity.
   
   query_options = { :weights => { [:title, :author] => +3, [:title] => +1 } } # +/- points for ordered combinations.