picky 3.0.0.pre1 → 3.0.0.pre2

data/lib/picky/adapters/rack/search.rb

@@ -7,7 +7,7 @@ module Picky
     # adapter for it.
     #
     # For example, if you give it a query, it will extract the query param etc.
-    # and call search_with_text on it if it is called by Rack.
+    # and call search on it if it is called by Rack.
     #
     module Rack
 
@@ -35,9 +35,9 @@ module Picky
           lambda do |env|
             params  = ::Rack::Request.new(env).params
 
-            results = query.search_with_text *extracted(params)
+            results = query.search *extracted(params)
 
-            PickyLog.log results.to_log(params[query_key])
+            Picky.logger && Picky.logger.info(results.to_log(params[query_key]))
 
             respond_with results.to_response, content_type
           end

data/lib/picky/adapters/rack.rb

@@ -8,7 +8,7 @@ module Picky
     # adapter for it.
     #
     # For example, if you give it a query, it will extract the query param etc.
-    # and call search_with_text on it if it is called by Rack.
+    # and call search on it if it is called by Rack.
     #
     # Usage:
     #   Adapters::Rack.app_for(thing, options)
@@ -18,8 +18,8 @@ module Picky
       # Generates the appropriate app for Rack.
       #
       @@mapping = {
-        :search_with_text => Search,
-        :parameters       => LiveParameters
+        :search     => Search,
+        :parameters => LiveParameters
       }
       def self.app_for thing, options = {}
         @@mapping.each_pair do |method, adapter|

data/lib/picky/application.rb

@@ -175,7 +175,10 @@ module Picky
 
       # Routes.
       #
-      delegate :route, :to => :rack_adapter
+      def route options
+        raise "Warning: block passed into #route method, not into Search.new!" if block_given?
+        rack_adapter.route options
+      end
 
       # A Picky application implements the Rack interface.
       #
@@ -194,8 +197,7 @@ module Picky
       # Reloads & finalizes the apps.
       #
       def reload
-        Loader.load_user 'app'             # Sinatra app_file.
-        Loader.load_user 'app/logging'     # Standard Picky logging.
+        Loader.load_user 'app'             # Sinatra app_file. TODO Needed?
         Loader.load_user 'app/application' # Standard Picky appfile.
         finalize_apps
         exclaim "Application #{apps.map(&:name).join(', ')} loaded."

data/lib/picky/backend/file/json.rb

@@ -17,6 +17,13 @@ module Picky
         #
         def load
           Yajl::Parser.parse ::File.open(cache_path, 'r'), symbolize_keys: true
+
+          # Note: Circumvents the yajl symbolize utf-8 characters problem.
+          #
+          # Yajl::Parser.parse(::File.open(cache_path, 'r')).inject({}) do |hash, (k, v)|
+          #   hash[k.to_sym] = v
+          #   hash
+          # end
         end
         # Dumps the index hash in json format.
         #

data/lib/picky/backend/file/text.rb

@@ -36,7 +36,7 @@ module Picky
         def retrieve
           id    = nil
           token = nil
-          ::File.open(cache_path, 'r:binary') do |file|
+          ::File.open(cache_path, 'r:utf-8') do |file|
             file.each_line do |line|
               id, token = line.split ?,, 2
               yield id, (token.chomp! || token).to_sym
@@ -47,7 +47,7 @@ module Picky
         #
         #
         def open &block
-          ::File.open cache_path, 'w:binary', &block
+          ::File.open cache_path, 'w:utf-8', &block
         end
 
 

data/lib/picky/character_substituters/west_european.rb

@@ -1,5 +1,8 @@
 # encoding: utf-8
 #
+# TODO Should probably be renamed, as it also removes
+# diactritics from japanese characters, like べ to へ.
+#
 module Picky
 
   module CharacterSubstituters # :nodoc:all

data/lib/picky/extensions/symbol.rb

@@ -1,20 +1,20 @@
 # Extending the Symbol class.
 #
 class Symbol # :nodoc:all
-  
+
   # :keys.each_subtoken    # => yields each of [:keys, :key, :ke, :k]
   # :keys.each_subtoken(2) # => yields each of [:keys, :key, :ke]
   #
   def each_subtoken from_length = 1
     sub = self.id2name
-    
+
     size = sub.size
     from_length = size + from_length + 1 if from_length < 0
     from_length = size if size < from_length
     from_length = 1 if from_length < 1
-    
+
     yield self
     size.downto(from_length + 1) { yield sub.chop!.intern }
   end
-  
+
 end

data/lib/picky/indexes/index.rb

@@ -99,41 +99,25 @@ module Picky
       # === Parameters
       # * name: A name that will be used for the index directory and in the Picky front end.
       #
-      # === Options
+      # === Options (all are used in the block, see examples)
       # * source: Where the data comes from, e.g. Sources::CSV.new(...). Optional, can be defined in the block using #source.
       # * result_identifier: Use if you'd like a different identifier/name in the results than the name of the index.
       # * after_indexing: As of this writing only used in the db source. Executes the given after_indexing as SQL after the indexing process.
-      # * tokenizer: The tokenizer to use for this index. Optional, can be defined in the block using #indexing.
-      # * key_format: The format the ids of this index are in. Optional, can be defined in the block using #key_format.
-      #
-      # Examples:
-      #   my_index = Indexes::Memory.new(:my_index, source: some_source) do
-      #     category :bla
-      #   end
+      # * tokenizer: Call and pass either a tokenizer (responds to #tokenize) or the options for a tokenizer..
+      # * key_format: Call and pass in a format method for the ids (default is #to_i).
       #
+      # Example:
       #   my_index = Indexes::Memory.new(:my_index) do
-      #     source   Sources::CSV.new(file: 'data/index.csv')
-      #     category :bla
+      #     source            Sources::CSV.new(file: 'data/index.csv')
+      #     key_format        :to_sym
+      #     category          :bla
+      #     result_identifier :my_special_results
       #   end
       #
-      #
       def initialize name, options = {}
-        check_name name
-        @name = name.to_sym
-
-        check_options options
-
-        @source = options[:source]
-
-        @after_indexing        = options[:after_indexing]
-        @tokenizer             = options[:tokenizer]
-        @key_format            = options[:key_format]
-
-        # Indexed.
-        #
-        @result_identifier    = options[:result_identifier] || name
+        @name              = name.to_sym
 
-        # TODO Move ignore_unassigned_tokens to query, somehow.
+        # TODO Move ignore_unassigned_tokens to query, somehow. Then, remove options.
         #
         @categories = Categories.new ignore_unassigned_tokens: (options[:ignore_unassigned_tokens] || false)
 
@@ -141,13 +125,7 @@ module Picky
         #
         Indexes.register self
 
-        #
-        #
         instance_eval(&Proc.new) if block_given?
-
-        # Check if any source has been given in the block or the options.
-        #
-        check_source @source
       end
 
       # Default bundles.
@@ -319,58 +297,6 @@ module Picky
       end
       alias define_geo_categories geo_categories
 
-      #
-      # Since this is an API, we fail hard quickly.
-      #
-      def check_name name # :nodoc:
-        raise ArgumentError.new(<<-NAME
-
-
-  The index identifier (you gave "#{name}") for Indexes::Memory/Indexes::Redis should be a Symbol/String,
-  Examples:
-    Indexes::Memory.new(:my_cool_index) # Recommended
-    Indexes::Redis.new("a-redis-index")
-  NAME
-
-
-  ) unless name.respond_to?(:to_sym)
-      end
-      def check_options options # :nodoc:
-        raise ArgumentError.new(<<-OPTIONS
-
-
-  Sources are not passed in as second parameter for #{self.class.name} anymore, but either
-  * as :source option:
-    #{self.class.name}.new(#{name.inspect}, source: #{options})
-  or
-  * given to the #source method inside the config block:
-    #{self.class.name}.new(#{name.inspect}) do
-      source #{options}
-    end
-
-  Sorry about that breaking change (in 2.2.0), didn't want to go to 3.0.0 yet!
-
-  All the best
-    -- Picky
-
-
-  OPTIONS
-  ) unless options.respond_to?(:[])
-      end
-      def check_source source # :nodoc:
-        raise ArgumentError.new(<<-SOURCE
-
-
-  The index "#{name}" should use a data source that responds to either the method #each, or the method #harvest, which yields(id, text), OR it can be a lambda/block, returning such a source.
-  Or it could use one of the built-in sources:
-    Sources::#{(Sources.constants - [:Base, :Wrappers, :NoCSVFileGiven, :NoCouchDBGiven]).join(',
-    Sources::')}
-
-
-  SOURCE
-  ) unless source.respond_to?(:each) || source.respond_to?(:harvest) || source.respond_to?(:call)
-      end
-
       def to_stats # :nodoc:
         stats = <<-INDEX
   #{name} (#{self.class}):
@@ -390,7 +316,7 @@ module Picky
       #
       #
       def to_s
-        "#{self.class}(#{name}, result_id: #{result_identifier}, source: #{source}, categories: #{categories})"
+        "#{self.class}(#{name}, result_id: #{result_identifier}, source: #{@source}, categories: #{categories})"
       end
 
     end

data/lib/picky/indexes/index_indexed.rb

@@ -6,8 +6,7 @@ module Picky
     #
     class Index
 
-      attr_reader :result_identifier,
-                  :combinator
+      attr_reader :combinator
 
       delegate :load_from_cache,
                :analyze,
@@ -17,6 +16,18 @@ module Picky
 
       alias reload load_from_cache
 
+      # Define how the results of this index are identified.
+      # (Shown in the client, for example)
+      #
+      # Default is the name of the index.
+      #
+      def result_identifier result_identifier = nil
+        result_identifier ? define_result_identifier(result_identifier) : (@result_identifier || @name)
+      end
+      def define_result_identifier result_identifier
+        @result_identifier = result_identifier
+      end
+
     end
 
   end

data/lib/picky/indexes/index_indexing.rb

@@ -6,8 +6,7 @@ module Picky
     #
     class Index
 
-      attr_reader :after_indexing,
-                  :bundle_class
+      attr_reader :bundle_class
 
       # Delegators for indexing.
       #
@@ -35,6 +34,19 @@ module Picky
         end
       end
 
+      # Define an index tokenizer on the index.
+      #
+      # Parameters are the exact same as for indexing.
+      #
+      def indexing options = {}
+        @tokenizer = if options.respond_to?(:tokenize)
+          options
+        else
+          options && Tokenizers::Index.new(options)
+        end
+      end
+      alias define_indexing indexing
+
       # Check if the given enumerable source is empty.
       #
       # Note: Checking as early as possible to tell the
@@ -68,15 +80,6 @@ module Picky
         categories.each &:cache
       end
 
-      # Define an index tokenizer on the index.
-      #
-      # Parameters are the exact same as for indexing.
-      #
-      def indexing options = {}
-        @tokenizer = Tokenizers::Index.new options
-      end
-      alias define_indexing indexing
-
       # Returns the installed tokenizer or the default.
       #
       def tokenizer
@@ -102,6 +105,7 @@ module Picky
         @source = @source.respond_to?(:call) ? @source.call : @source
       end
       def define_source source
+        check_source source
         @source = source
       end
       def raise_no_source
@@ -119,6 +123,19 @@ module Picky
         NO_SOURCE
   )
       end
+      def check_source source # :nodoc:
+        raise ArgumentError.new(<<-SOURCE
+
+
+  The index "#{name}" should use a data source that responds to either the method #each, or the method #harvest, which yields(id, text), OR it can be a lambda/block, returning such a source.
+  Or it could use one of the built-in sources:
+    Sources::#{(Sources.constants - [:Base, :Wrappers, :NoCSVFileGiven, :NoCouchDBGiven]).join(',
+    Sources::')}
+
+
+  SOURCE
+  ) unless source.respond_to?(:each) || source.respond_to?(:harvest) || source.respond_to?(:call)
+      end
 
       # Define a key_format on the index.
       #
@@ -131,6 +148,16 @@ module Picky
         @key_format = key_format
       end
 
+      # Define what to do after indexing.
+      # (Only used in the Sources::DB)
+      #
+      def after_indexing after_indexing = nil
+        after_indexing ? define_after_indexing(after_indexing) : @after_indexing
+      end
+      def define_after_indexing after_indexing
+        @after_indexing = after_indexing
+      end
+
     end
 
   end

data/lib/picky/loader.rb

@@ -209,10 +209,6 @@ module Picky
       #
       load_relative 'character_substituters/west_european'
 
-      # Logging.
-      #
-      load_relative 'loggers/search'
-
       # Convenience accessors for generators.
       #
       load_relative 'generators/aliases'

data/lib/picky/search.rb

@@ -1,14 +1,16 @@
+# encoding: utf-8
+#
 module Picky
 
-  # = Picky Queries
+  # = Picky Searches
   #
   # A Picky Search is an object which:
-  # * holds one or more indexes
-  # * offers an interface to query these indexes.
+  #  * holds one or more indexes
+  #  * offers an interface to query these indexes.
   #
   # You connect URL paths to indexes via a Query.
   #
-  # We recommend not to use this directly, but connect it to an URL and query through one of these
+  # We recommend to not use this directly, but connect it to an URL and query through one of these
   # (Protip: Use "curl 'localhost:8080/query/path?query=exampletext')" in a Terminal.
   #
   class Search
@@ -16,39 +18,40 @@ module Picky
     include Helpers::Measuring
 
     attr_reader   :indexes
-    attr_writer   :tokenizer
-    attr_accessor :weights
+    attr_accessor :tokenizer, :weights
 
     # Takes:
     # * A number of indexes
-    # * Options hash (optional) with:
-    #   * tokenizer: Tokenizers::Query.default by default.
-    #   * weights:   A hash of weights, or a Query::Weights object.
     #
     # TODO Add identifiers_to_remove (rename) and reduce_allocations_to_amount (rename).
     # TODO categories_to_remove ?
     #
     # It is also possible to define the tokenizer and weights like so.
     # Example:
-    #   Search.new(index1, index2, index3) do
+    #   search = Search.new(index1, index2, index3) do
     #     searching removes_characters: /[^a-z]/, etc.
     #     weights [:author, :title] => +3, [:title, :isbn] => +1
     #   end
     #
     def initialize *index_definitions
-      options = Hash === index_definitions.last ? index_definitions.pop : {}
-
       @indexes = Query::Indexes.new *index_definitions, combinations_type_for(index_definitions)
-      searching options[:tokenizer]
-      boost     options[:weights]
 
       instance_eval(&Proc.new) if block_given?
+
+      @tokenizer ||= Tokenizers::Query.default
+      @weights   ||= Query::Weights.new
+
+      self
     end
 
-    # Example:
-    #   Search.new(index1, index2, index3) do
-    #     searching removes_characters: /[^a-z]/, etc.
-    #     weights [:author, :title] => +3, [:title, :isbn] => +1
+    # Examples:
+    #   search = Search.new(index1, index2, index3) do
+    #     searching removes_characters: /[^a-z]/,
+    #               # etc.
+    #   end
+    #
+    #   search = Search.new(index1, index2, index3) do
+    #     searching MyTokenizerThatRespondsToTheMethodTokenize.new
     #   end
     #
     def searching options
@@ -59,63 +62,17 @@ module Picky
       end
     end
 
-    # Returns the tokenizer if set or if not, the query tokenizer.
-    #
-    def tokenizer
-      @tokenizer || Tokenizers::Query.default
-    end
-
     # Example:
-    #   Search.new(index1, index2, index3) do
-    #     searching removes_characters: /[^a-z]/, etc.
-    #     boost [:author, :title] => +3, [:title, :isbn] => +1
+    #   search = Search.new(books_index, dvd_index, mp3_index) do
+    #     boost [:author, :title] => +3,
+    #           [:title, :isbn]   => +1
     #   end
     #
-    def boost options
-      weights  = options || Query::Weights.new
+    def boost weights
+      weights  ||= Query::Weights.new
       @weights = Hash === weights ? Query::Weights.new(weights) : weights
     end
 
-    # Returns the right combinations strategy for
-    # a number of query indexes.
-    #
-    # Currently it isn't possible using Memory and Redis etc.
-    # indexes in the same query index group.
-    #
-    # Picky will raise a Query::Indexes::DifferentTypesError.
-    #
-    @@mapping = {
-      Indexes::Memory => Query::Combinations::Memory,
-      Indexes::Redis  => Query::Combinations::Redis
-    }
-    def combinations_type_for index_definitions_ary
-      index_types = extract_index_types index_definitions_ary
-      !index_types.empty? && @@mapping[*index_types] || Query::Combinations::Memory
-    end
-    def extract_index_types index_definitions_ary
-      index_types = index_definitions_ary.map(&:class)
-      index_types.uniq!
-      check_index_types index_types
-      index_types
-    end
-    def check_index_types index_types
-      raise_different index_types if index_types.size > 1
-    end
-    # Currently it isn't possible using Memory and Redis etc.
-    # indexes in the same query index group.
-    #
-    class DifferentTypesError < StandardError
-      def initialize types
-        @types = types
-      end
-      def to_s
-        "Currently it isn't possible to mix #{@types.join(" and ")} Indexes in the same Search instance."
-      end
-    end
-    def raise_different index_types
-      raise DifferentTypesError.new(index_types)
-    end
-
     # This is the main entry point for a query.
     # Use this in specs and also for running queries.
     #
@@ -126,19 +83,15 @@ module Picky
     #
     # Note: The Rack adapter calls this method after unravelling the HTTP request.
     #
-    # TODO Rename to search.
-    #
-    def search_with_text text, ids = 20, offset = 0
-      search tokenized(text), ids.to_i, offset.to_i
+    def search text, ids = 20, offset = 0
+      search_with tokenized(text), ids.to_i, offset.to_i
     end
 
     # Runs the actual search using Query::Tokens.
     #
-    # Note: Internal method, use #search_with_text.
+    # Note: Internal method, use #search
     #
-    # TODO Rename to search_with.
-    #
-    def search tokens, ids = 20, offset = 0
+    def search_with tokens, ids = 20, offset = 0
       results = nil
 
       duration = timed do
@@ -151,7 +104,7 @@ module Picky
 
     # Execute a search using Query::Tokens.
     #
-    # Note: Internal method, use #search_with_text.
+    # Note: Internal method, use #search.
     #
     def execute tokens, ids, offset
       Results.from ids, offset, sorted_allocations(tokens)
@@ -172,6 +125,46 @@ module Picky
       indexes.prepared_allocations_for tokens, weights
     end
 
+    # Returns the right combinations strategy for
+    # a number of query indexes.
+    #
+    # Currently it isn't possible using Memory and Redis etc.
+    # indexes in the same query index group.
+    #
+    # Picky will raise a Query::Indexes::DifferentTypesError.
+    #
+    @@mapping = {
+      Indexes::Memory => Query::Combinations::Memory,
+      Indexes::Redis  => Query::Combinations::Redis
+    }
+    def combinations_type_for index_definitions_ary
+      index_types = extract_index_types index_definitions_ary
+      !index_types.empty? && @@mapping[*index_types] || Query::Combinations::Memory
+    end
+    def extract_index_types index_definitions_ary
+      index_types = index_definitions_ary.map(&:class)
+      index_types.uniq!
+      check_index_types index_types
+      index_types
+    end
+    def check_index_types index_types
+      raise_different index_types if index_types.size > 1
+    end
+    # Currently it isn't possible using Memory and Redis etc.
+    # indexes in the same query index group.
+    #
+    class DifferentTypesError < StandardError
+      def initialize types
+        @types = types
+      end
+      def to_s
+        "Currently it isn't possible to mix #{@types.join(" and ")} Indexes in the same Search instance."
+      end
+    end
+    def raise_different index_types
+      raise DifferentTypesError.new(index_types)
+    end
+
     # Display some nice information for the user.
     #
     def to_s

data/lib/picky/tokenizers/base.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+#
 module Picky
 
   module Tokenizers # :nodoc:all
@@ -120,7 +122,7 @@ Case sensitive?            #{@case_sensitive ? "Yes." : "-"}
       #
       # Note: Currently only for indexing.
       #
-      def reject_token_if &condition
+      def rejects_token_if &condition
         @reject_condition = condition
       end
       def reject tokens
@@ -170,7 +172,7 @@ Case sensitive?            #{@case_sensitive ? "Yes." : "-"}
         # Defaults.
         #
         splits_text_on options[:splits_text_on] || /\s/
-        reject_token_if &(options[:reject_token_if] || options[:rejects_token_if] || :blank?) # TODO Decide on using an s or not.
+        rejects_token_if &(options[:rejects_token_if] || :blank?)
       end
 
       # Default preprocessing hook.

data/lib/picky.rb

@@ -1,5 +1,11 @@
 module Picky
 
+  # This is only used in the classic project style.
+  #
+  class << self
+    attr_accessor :logger
+  end
+
   # Require the constants.
   #
   require ::File.expand_path '../picky/constants', __FILE__