picky 0.0.9 → 0.1.0

data/lib/picky/application.rb

@@ -1,40 +1,41 @@
+# The Picky application wherein the indexing and querying is defined.
+#
 class Application
-  
-  # An application simply delegates to the routing to handle a request.
-  #
-  def self.routing
-    @routing ||= Routing.new
+  class << self
+    
+    # An application simply delegates to the routing to handle a request.
+    #
+    def call env
+      routing.call env
+    end
+    
+    # Freezes the routes.
+    #
+    def finalize
+      routing.freeze
+    end
+    def routing
+      @routing ||= Routing.new
+    end
+    # Routes.
+    #
+    delegate :route, :root, :to => :routing
+    
+    # 
+    #
+    def indexing
+      @indexing ||= Configuration::Indexes.new
+    end
+    def index *args
+      self.type *args
+    end
+    delegate :type, :field, :to => :indexing
+    
+    #
+    #
+    def querying
+      @queries ||= Configuration::Queries.new
+    end
+    
   end
-  def self.call env
-    routing.call env
-  end
-  
-  # 
-  #
-  def self.indexes &block
-    indexes_configuration.instance_eval &block
-    # TODO Uglyyyyyy.
-    ::Indexes.configuration = indexes_configuration
-    ::Indexes.setup # TODO Think about setup/reload.
-  end
-  def self.indexes_configuration
-    @indexes || reset_indexes
-  end
-  def self.reset_indexes
-    @indexes = Configuration::Indexes.new # Is instance a problem?
-  end
-  
-  # 
-  #
-  def self.queries &block
-    queries_configuration.instance_eval &block
-    routing.freeze
-  end
-  def self.queries_configuration
-    @queries || reset_queries
-  end
-  def self.reset_queries
-    @queries = Configuration::Queries.new routing # Is instance a problem?
-  end
-  
 end

data/lib/picky/cacher/partial/default.rb

@@ -1,7 +1,5 @@
 module Cacher
   module Partial
-    # Default is Subtoken, down to 1.
-    #
-    Default = Subtoken.new :down_to => 1
+    Default = Subtoken.new :down_to => 1, :starting_at => -1
   end
 end

data/lib/picky/cacher/partial/subtoken.rb

@@ -2,6 +2,29 @@ module Cacher
 
   module Partial
     
+    # Generates the right subtokens for use in the subtoken strategy.
+    #
+    class SubtokenGenerator
+      
+      attr_reader :down_to, :starting_at
+      
+      def initialize down_to, starting_at
+        @down_to, @starting_at = down_to, starting_at
+        
+        if @starting_at.zero?
+          def each_subtoken token, &block
+            token.each_subtoken @down_to, &block
+          end
+        else
+          def each_subtoken token, &block
+            token[0..@starting_at].intern.each_subtoken @down_to, &block
+          end
+        end
+        
+      end
+      
+    end
+    
     # The subtoken partial strategy.
     #
     # If given
@@ -17,23 +40,27 @@ module Cacher
     #
     class Subtoken < Strategy
       
-      attr_reader :down_to, :starting_at
-      
       # Down to is how far it will go down in generating the subtokens.
       #
       # Examples:
-      # With :hello, and starting_at 0
+      # With :hello, and starting_at -1
       # * down to == 1: [:hello, :hell, :hel, :he, :h]
       # * down to == 4: [:hello, :hell]
       #
-      # With :hello, and starting_at -1
+      # With :hello, and starting_at -2
       # * down to == 1: [:hell, :hel, :he, :h]
       # * down to == 4: [:hell]
       #
       def initialize options = {}
-        @down_to     = options[:down_to] || 1
-        starting_at  = options[:starting_at] || 0
-        @starting_at = starting_at.zero? ? 0 : starting_at - 1
+        down_to     = options[:down_to] || 1
+        starting_at = options[:starting_at] || -1
+        @generator = SubtokenGenerator.new down_to, starting_at
+      end
+      def down_to
+        @generator.down_to
+      end
+      def starting_at
+        @generator.starting_at
       end
       
       # Generates a partial index from the given index.
@@ -43,12 +70,12 @@ module Cacher
         
         # Generate for each key token the subtokens.
         #
-        i = 5000
+        i = 0
         index.each_key do |token|
-          i -= 1
-          if i == 0
+          i += 1
+          if i == 5000
             puts "#{Time.now}: Generating partial tokens for token #{token}. This appears every 5000 tokens."
-            i = 5000
+            i = 0
           end
           generate_for token, index, result
         end
@@ -58,7 +85,7 @@ module Cacher
         # TODO If it is unique for a subtoken, it is
         #      unique for all derived longer tokens.
         #
-        result.each_value &:uniq! # Removed because of the set combination operation below
+        result.each_value &:uniq!
         
         result
       end
@@ -74,18 +101,17 @@ module Cacher
         # TODO Could be improved by appending the aforegoing ids?
         #
         def generate_for token, index, result
-          clipped_token = starting_at.zero? ? token : token[0..starting_at].to_sym
-          clipped_token.subtokens(down_to).each do |subtoken|
+          @generator.each_subtoken(token) do |subtoken|
             if result[subtoken]
               result[subtoken] += index[token] # unique
             else
-              result[subtoken] = index[token].dup
+              result[subtoken] = index[token].dup # TODO Spec this dup
             end
           end
         end
-
+        
     end
-
+    
   end
-
+  
 end

data/lib/picky/configuration/field.rb

@@ -1,5 +1,8 @@
 module Configuration
-
+  
+  # Describes the configuration of a "field", a category
+  # (title is a category of a books index, for example).
+  #
   class Field
     attr_reader :name, :indexed_name, :virtual
     attr_accessor :type # convenience
@@ -15,7 +18,8 @@ module Configuration
       @indexed_name   = options.delete(:indexed_field) || name # TODO Rename to indexed_as?
       @virtual        = options.delete(:virtual)       || false
       
-      Query::Qualifiers.add(name, options[:qualifiers]) if options[:qualifiers]
+      qualifiers = options[:qualifiers]
+      Query::Qualifiers.add(name, qualifiers) if qualifiers
       
       # @remove          = options[:remove]        || false
       # @filter          = options[:filter]        || true

data/lib/picky/configuration/indexes.rb

@@ -1,11 +1,13 @@
 module Configuration
-
+  
+  # Describes the container for all index configurations.
+  #
   class Indexes
     
     attr_reader :types
     
-    def initialize *types
-      @types = types
+    def initialize
+      @types = []
     end
     
     #
@@ -16,12 +18,19 @@ module Configuration
     
     # Delegates
     #
-    delegate :illegal_characters, :contract_expressions, :stopwords, :split_text_on, :normalize_words, :illegal_characters_after_splitting, :to => :default_index
+    delegate :removes_characters, :contracts_expressions, :stopwords, :splits_text_on, :normalizes_words, :removes_characters_after_splitting, :to => :default_index
     
-    def type name, *fields
-      types << Type.new(name, *fields)
+    # TODO Rewrite all this configuration handling.
+    #
+    def type name, source, *fields
+      new_type = Type.new name, source, *fields
+      types << new_type
+      ::Indexes.configuration ||= self
+      
+      generated = new_type.generate
+      ::Indexes.add generated
+      generated
     end
-    alias add_index type
     def field name, options = {}
       Field.new name, options
     end

data/lib/picky/configuration/queries.rb

@@ -1,31 +1,21 @@
 module Configuration
   
+  # 
+  #
   class Queries
     
-    attr_reader :routing
-    
-    #
-    #
-    def initialize routing
-      @routing = routing
-    end
-    
     #
     #
     def default_index
       Tokenizers::Query
     end
-    
-    # Routes.
-    #
-    delegate :defaults, :route, :live, :full, :root, :default, :to => :routing
+    delegate :removes_characters, :contracts_expressions, :stopwords, :splits_text_on, :normalizes_words, :removes_characters_after_splitting, :to => :default_index
     
     # Delegates.
     #
     def maximum_tokens amount
       Query::Tokens.maximum = amount
     end
-    delegate :illegal_characters, :contract_expressions, :stopwords, :split_text_on, :normalize_words, :illegal_characters_after_splitting, :to => :default_index
     
   end
   

data/lib/picky/extensions/symbol.rb

@@ -2,17 +2,32 @@
 #
 class Symbol
   
-  # :keys.subtokens    # => [:key, :ke, :k]
-  # :keys.subtokens(2) # => [:key, :ke]
+  # :keys.subtokens    # => [:keys, :key, :ke, :k]
+  # :keys.subtokens(2) # => [:keys, :key, :ke]
   #
   def subtokens down_to_length = 1
-    sub, result = self.to_s, [self]
+    sub = self.id2name
     
     size = sub.size
+    down_to_length = size + down_to_length if down_to_length < 0
     down_to_length = size if size < down_to_length
     
-    size.downto(down_to_length + 1) { result << sub.chop!.to_sym }
+    result = [self]
+    size.downto(down_to_length + 1) { result << sub.chop!.intern }
     result
   end
   
+  # TODO Duplicate code.
+  #
+  def each_subtoken down_to_length = 1
+    sub = self.id2name
+    
+    size = sub.size
+    down_to_length = size + down_to_length if down_to_length < 0
+    down_to_length = size if size < down_to_length
+    
+    yield self
+    size.downto(down_to_length + 1) { yield sub.chop!.intern }
+  end
+  
 end

data/lib/picky/generator.rb

@@ -2,6 +2,10 @@ require 'fileutils'
 
 module Picky
   
+  # Thrown when no generator for the command
+  #   picky <command> <options>
+  # is found.
+  #
   class NoGeneratorException < Exception; end
   
   # This is a very simple project generator.
@@ -43,6 +47,11 @@ module Picky
       klass.new *args
     end
     
+    # Generates a new Picky project.
+    #
+    # Example:
+    #   > picky project my_lovely_project
+    #
     class Project
       
       attr_reader :name, :prototype_project_basedir

data/lib/picky/helpers/measuring.rb

@@ -5,12 +5,12 @@ module Helpers
     
     # Returns a duration in seconds.
     #
-    def timed(*args, &block)
-      block_to_be_measured = lambda(&block)
+    def timed *args, &block
+      block_to_be_measured = lambda &block
       
       time_begin = Time.now.to_f
       
-      block_to_be_measured.call(*args)
+      block_to_be_measured.call *args
       
       Time.now.to_f - time_begin
     end

data/lib/picky/index/bundle.rb

@@ -103,10 +103,11 @@ module Index
     # Copies the indexes to the "backup" directory.
     #
     def backup
-      FileUtils.mkdir backup_path unless Dir.exists?(backup_path)
-      FileUtils.cp index_cache_path, backup_path, :verbose => true
-      FileUtils.cp similarity_cache_path, backup_path, :verbose => true
-      FileUtils.cp weights_cache_path, backup_path, :verbose => true
+      target = backup_path
+      FileUtils.mkdir target unless Dir.exists?(target)
+      FileUtils.cp index_cache_path,      target, :verbose => true
+      FileUtils.cp similarity_cache_path, target, :verbose => true
+      FileUtils.cp weights_cache_path,    target, :verbose => true
     end
     def backup_path
       File.join File.dirname(index_cache_path), 'backup'

data/lib/picky/index/category.rb

@@ -22,8 +22,8 @@ module Index
       @full    = options[:full_bundle]    || Bundle.new(:full,    self, type, Cacher::Partial::None.new, weights, similarity)
       @partial = options[:partial_bundle] || Bundle.new(:partial, self, type, partial, weights, Cacher::Similarity::None.new)
       
-      @full    = options[:full_lambda].call(@full, @partial)    if options[:full_lambda]
-      @partial = options[:partial_lambda].call(@full, @partial) if options[:partial_lambda]
+      @full    = full_lambda.call(@full, @partial)    if full_lambda = options[:full_lambda]
+      @partial = partial_lambda.call(@full, @partial) if partial_lambda = options[:partial_lambda]
     end
     
     # Loads the index from cache.
@@ -40,13 +40,13 @@ module Index
     # Generates all caches for this category.
     #
     def generate_caches
-      exclaim "#{Time.now}: Loading data from db for #{identifier}."
+      timed_exclaim "Loading data from db for #{identifier}."
       generate_caches_from_db
-      exclaim "#{Time.now}: Generating partial for #{identifier}."
+      timed_exclaim "Generating partial for #{identifier}."
       generate_partial
-      exclaim "#{Time.now}: Generating caches from memory for #{identifier}."
+      timed_exclaim "Generating caches from memory for #{identifier}."
       generate_caches_from_memory
-      exclaim "#{Time.now}: Dumping all caches for #{identifier}."
+      timed_exclaim "Dumping all caches for #{identifier}."
       dump_caches
     end
     def generate_caches_from_db
@@ -62,10 +62,17 @@ module Index
       full.dump
       partial.dump
     end
+    # TODO move to Kernel?
+    #
+    def timed_exclaim text
+      exclaim "#{Time.now}: #{text}"
+    end
+    # TODO move to Kernel?
+    #
     def exclaim text
       puts text
     end
-
+    
     # Used for testing.
     #
     def generate_indexes_from_full_index

data/lib/picky/index/combined.rb

@@ -2,7 +2,12 @@
 #
 module Index
 
-  # 
+  # This index combines a full and partial index.
+  # It serves to order the results such that exact (full) hits are found first.
+  #
+  # TODO Rename full -> exact. exact/partial?
+  #
+  # TODO Need to use the right subtokens. Bake in?
   #
   class Combined < Bundle