picky 0.0.0 → 0.0.2

data/bin/picky

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+
+begin
+  require 'picky/generator'
+rescue LoadError => e
+  require 'rubygems'
+  picky_path = File.expand_path('../../lib', __FILE__)
+  $:.unshift(picky_path) if File.directory?(picky_path) && !$:.include?(picky_path)
+  require 'picky/generator'
+end
+
+generator = Picky::Generator.new
+generator.generate ARGV

data/lib/bundling.rb

@@ -0,0 +1,10 @@
+# TODO Remove?
+#
+begin
+  require 'bundler'
+rescue LoadError => e
+  require 'rubygems'
+  require 'bundler'
+end
+Bundler.setup SEARCH_ENVIRONMENT
+Bundler.require

data/lib/constants.rb

@@ -0,0 +1,9 @@
+# Set constants.
+#
+
+# Use rack's environment for the search engine.
+#
+ENV['SEARCH_ENV'] ||= ENV['RACK_ENV']
+
+SEARCH_ENVIRONMENT = ENV['SEARCH_ENV'] || 'development' unless defined? SEARCH_ENVIRONMENT
+SEARCH_ROOT        = Dir.pwd unless defined? SEARCH_ROOT

data/lib/deployment.rb

@@ -0,0 +1,212 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'constants'))
+
+module Picky
+  module Capistrano
+    
+    # Include all 
+    #
+    module All
+      def self.extended cap_config
+        
+        cap_config.instance_eval do
+          
+          # Executes a rake task on the server.
+          #
+          # Options:
+          #  * env: The SEARCH_ENV. Will not set if set explicitly to false. Default: production.
+          #  * All other options get passed on to the Capistrano run task.
+          #
+          def execute_rake_task name, options = {}, &block
+            env = options.delete :env
+            env = env == false ? '' : "SEARCH_ENV=#{env || 'production'}"
+            run "cd #{current_path}; rake #{name} #{env}", options, &block
+          end
+          
+        end
+        
+        cap_config.extend Standard
+        cap_config.extend Deploy
+        cap_config.extend Caching
+        cap_config.extend Overrides
+        
+      end
+    end
+    
+    # Removes unneeded Rails defaults.
+    #
+    module Overrides
+      def self.extended cap_config
+        cap_config.instance_eval do
+          
+          namespace :deploy do
+            tasks.delete :check
+            tasks.delete :cold
+            tasks.delete :migrations
+            tasks.delete :migrate
+            tasks.delete :upload
+            
+            namespace :web do
+              tasks.delete :enable
+              tasks.delete :disable
+            end
+          end
+          
+        end
+      end
+    end
+    
+    module Standard
+      def self.extended cap_config
+        cap_config.load 'standard'
+        cap_config.load 'deploy'
+      end
+    end
+    
+    module Deploy
+      
+      def self.extended cap_config
+        cap_config.instance_eval do
+          
+          namespace :deploy do
+            %w(start stop).each do |action|
+              desc "#{action} the Servers"
+              task action.to_sym, :roles => :app do
+                execute_rake_task "server:#{action}"
+              end
+            end
+            desc "Restart the Servers sequentially"
+            task :restart, :roles => :app do
+              find_servers(:roles => :app).each do |server|
+                execute_rake_task "server:restart", :hosts => server.host
+              end
+            end
+            
+            desc 'Hot deploy the code'
+            task 'hot', :roles => :app do
+              update
+              execute_rake_task 'server:usr1', :env => false # No env needed.
+            end
+            
+            desc "Setup a GitHub-style deployment."
+            task :setup, :roles => :app do
+              cmd = "git clone #{repository} #{current_path}-clone-cache &&" +
+                    "rm #{current_path} &&" +
+                    "mv #{current_path}-clone-cache #{current_path}"
+              run cmd
+            end
+            
+            desc "Deploy"
+            task :default, :roles => :app do
+              update
+              restart
+            end
+            
+            desc "Update the deployed code."
+            task :update_code do # code needs to be updated with all servers
+              puts "updating code to branch #{branch}"
+              cmd = "cd #{current_path} &&" +
+                    "git fetch origin &&" +
+                    "(git checkout -f #{branch} || git checkout -b #{branch} origin/#{branch}) &&" +
+                    "git pull;" +
+                    "git branch"
+              run cmd
+              symlink
+            end
+            
+            desc "Cleans up the git checkout"
+            task :cleanup, :roles => :app do
+              run "cd #{current_path} && git gc --aggressive"
+            end
+            
+            desc "create the symlinks to the shared dirs"
+            task :symlink do
+              set :user, 'deploy'
+              run "rm -rf #{current_path}/log;   ln -sf #{shared_path}/log   #{current_path}/log"
+              run "rm -rf #{current_path}/index; ln -sf #{shared_path}/index #{current_path}/index"
+              # link database-config files
+              run "ln -sf #{shared_path}/config/base.yml #{current_path}/config/db/base.yml"
+              run "ln -sf #{shared_path}/config/source.yml #{current_path}/config/db/source.yml"
+              # link unicorn.ru
+              run "ln -sf #{shared_path}/config/unicorn.ru #{current_path}/config/unicorn.ru" # TODO change path
+            end
+            
+            namespace :rollback do
+              desc "Rollback to last release."
+              task :default, :roles => :app do
+                set :branch, branches[-2]
+                puts "rolling back to branch #{branch}"
+                deploy.update_code
+              end
+              
+              task :code, :roles => :app do
+                # implicit
+              end
+            end
+          end
+          
+        end
+      end
+      
+    end
+    
+    module Caching
+      
+      def self.extended cap_config
+        cap_config.instance_eval do
+          namespace :cache do
+            desc "check the index files if they are ready to be used"
+            task :check, :roles => :cache do
+              execute_rake_task 'cache:check'
+            end
+          end
+          namespace :cache do
+            namespace :structure do
+              desc "create the index cache structure"
+              task :create, :roles => :app do
+                execute_rake_task 'cache:structure:create'
+              end
+            end
+          end
+          namespace :solr do
+            desc "create the index cache structure"
+            task :index, :roles => :cache do
+              execute_rake_task 'solr:index'
+            end
+            %w|start stop restart|.collect(&:to_sym).each do |action|
+              desc "#{action} the solr server"
+              task action, :roles => :app do
+                execute_rake_task 'solr:start'
+              end
+            end
+          end
+        end
+      end
+      
+    end
+    
+    module Statistics
+      
+      def self.extended cap_config
+        namespace :statistics do
+          desc 'Start the statistics server'
+          task :start, :roles => :statistics do
+            set :user, 'root'
+            run "daemonize -c #{current_path} -u deploy -v #{current_path}/script/statistics/start production"
+          end
+          desc 'Stop the statistics server'
+          task :stop, :roles => :statistics do
+            run "#{current_path}/script/statistics/stop production"
+          end
+          desc 'Restart the statistics server'
+          task :restart, :roles => :statistics do
+            stop
+            sleep 2
+            start
+          end
+        end
+      end
+      
+    end
+    
+  end
+end

data/lib/picky/application.rb

@@ -0,0 +1,40 @@
+class Application
+  
+  # An application simply delegates to the routing to handle a request.
+  #
+  def self.routing
+    @routing ||= Routing.new
+  end
+  def self.call env
+    routing.call env
+  end
+  
+  # TODO Multiple indexes?
+  #
+  def self.indexes &block
+    indexes_configuration.instance_eval &block
+    # TODO Uglyyyyyy.
+    ::Indexes.configuration = indexes_configuration
+    ::Indexes.setup # TODO Think about setup/reload.
+  end
+  def self.indexes_configuration
+    @indexes || reset_indexes
+  end
+  def self.reset_indexes
+    @indexes = Configuration::Indexes.new # Is instance a problem?
+  end
+  
+  # TODO Multiple Queries?
+  #
+  def self.queries &block
+    queries_configuration.instance_eval &block
+    routing.freeze
+  end
+  def self.queries_configuration
+    @queries || reset_queries
+  end
+  def self.reset_queries
+    @queries = Configuration::Queries.new routing # Is instance a problem?
+  end
+  
+end

data/lib/picky/cacher/convenience.rb

@@ -0,0 +1,3 @@
+Partial    = Cacher::Partial
+Similarity = Cacher::Similarity
+Weights    = Cacher::Weights

data/lib/picky/cacher/generator.rb

@@ -0,0 +1,17 @@
+module Cacher
+
+  # A cache generator holds an index type.
+  #
+  # TODO Rename to index_type.
+  #
+  class Generator
+
+    attr_reader :index
+
+    def initialize index
+      @index = index
+    end
+
+  end
+
+end

data/lib/picky/cacher/partial/default.rb

@@ -0,0 +1,7 @@
+module Cacher
+  module Partial
+    # Default is Subtoken, down to 1.
+    #
+    Default = Subtoken.new :down_to => 1
+  end
+end

data/lib/picky/cacher/partial/none.rb

@@ -0,0 +1,19 @@
+module Cacher
+
+  module Partial
+
+    # Does not generate a partial index.
+    #
+    class None < Strategy
+
+      # Returns an empty index.
+      #
+      def generate_from index
+        {}
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/cacher/partial/strategy.rb

@@ -0,0 +1,7 @@
+module Cacher
+  module Partial
+    # Superclass for partial strategies.
+    #
+    class Strategy; end
+  end
+end

data/lib/picky/cacher/partial/subtoken.rb

@@ -0,0 +1,91 @@
+module Cacher
+
+  module Partial
+    
+    # The subtoken partial strategy.
+    #
+    # If given
+    # "florian"
+    # will index
+    # "floria"
+    # "flori"
+    # "flor"
+    # "flo"
+    # "fl"
+    # "f"
+    # Depending on what the given down_to value is. (Example with down_to == 1)
+    #
+    class Subtoken < Strategy
+      
+      attr_reader :down_to, :starting_at
+      
+      # Down to is how far it will go down in generating the subtokens.
+      #
+      # Examples:
+      # With :hello, and starting_at 0
+      # * down to == 1: [:hello, :hell, :hel, :he, :h]
+      # * down to == 4: [:hello, :hell]
+      #
+      # With :hello, and starting_at -1
+      # * down to == 1: [:hell, :hel, :he, :h]
+      # * down to == 4: [:hell]
+      #
+      def initialize options = {}
+        @down_to     = options[:down_to] || 1
+        starting_at  = options[:starting_at] || 0
+        @starting_at = starting_at.zero? ? 0 : starting_at - 1
+      end
+      
+      # Generates a partial index from the given index.
+      #
+      def generate_from index
+        result = {}
+        
+        # Generate for each key token the subtokens.
+        #
+        i = 5000
+        index.each_key do |token|
+          i -= 1
+          if i == 0
+            puts "#{Time.now}: Generating partial tokens for token #{token}. This appears every 5000 tokens."
+            i = 5000
+          end
+          generate_for token, index, result
+        end
+        
+        # Remove duplicate ids.
+        #
+        # TODO If it is unique for a subtoken, it is
+        #      unique for all derived longer tokens.
+        #
+        result.each_value &:uniq! # Removed because of the set combination operation below
+        
+        result
+      end
+      
+      private
+        
+        # To each shortened token of :test
+        # :test, :tes, :te, :t
+        # add all ids of :test
+        #
+        # "token" here means just text.
+        #
+        # TODO Could be improved by appending the aforegoing ids?
+        #
+        def generate_for token, index, result
+          clipped_token = starting_at.zero? ? token : token[0..starting_at].to_sym
+          clipped_token.subtokens(down_to).each do |subtoken|
+            if result[subtoken]
+              result[subtoken] += index[token] # unique
+            else
+              result[subtoken] = index[token].dup
+            end
+          end
+        end
+
+    end
+
+  end
+
+end

data/lib/picky/cacher/partial_generator.rb

@@ -0,0 +1,15 @@
+module Cacher
+
+  # The partial generator uses a subtoken(downto:1) generator as default.
+  #
+  class PartialGenerator < Generator
+
+    # Generate a similarity index based on the given index.
+    #
+    def generate strategy = Partial::Subtoken.new(:down_to => 1)
+      strategy.generate_from self.index
+    end
+
+  end
+
+end

data/lib/picky/cacher/similarity/default.rb

@@ -0,0 +1,7 @@
+module Cacher
+  module Similarity
+    # Default is no similarity.
+    #
+    Default = None.new
+  end
+end

data/lib/picky/cacher/similarity/double_levenshtone.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+#
+module Cacher
+
+  module Similarity
+
+    # DoubleLevensthone means that it's a combination of
+    # * DoubleMetaphone
+    # and
+    # * Levenshtein
+    # :)
+    #
+    class DoubleLevenshtone < Strategy
+
+      attr_reader :amount
+
+      #
+      #
+      def initialize amount = 10
+        @amount = amount
+      end
+
+      # Encodes the given symbol.
+      #
+      # Returns a symbol.
+      #
+      def encoded sym
+        codes = Text::Metaphone.double_metaphone sym.to_s
+        codes.first.to_sym unless codes.empty?
+      end
+
+      # Generates an index for the given index (in full index style).
+      #
+      # In the following form:
+      # [:meier, :mueller, :peter, :pater] => { :MR => [:meier], :MLR => [:mueller], :PTR => [:peter, :pater] }
+      #
+      def generate_from index
+        hash = hashify index.keys
+        sort hash
+      end
+
+      private
+
+        # Sorts the index values in place.
+        #
+        def sort index
+          index.each_pair.each do |code, ary|
+            ary.sort_by_levenshtein! code
+            ary.slice! amount, ary.size # size is not perfectly correct, but anyway
+          end
+          index
+        end
+
+        # Hashifies a list of symbols.
+        #
+        # Where:
+        # { encoded_sym => [syms] }
+        #
+        def hashify list
+          list.inject({}) do |total, element|
+            if code = encoded(element)
+              total[code] ||= []
+              total[code] << element
+            end
+            total
+          end
+        end
+
+    end
+
+  end
+
+end

data/lib/picky/cacher/similarity/none.rb

@@ -0,0 +1,25 @@
+module Cacher
+
+  module Similarity
+
+    # Similarity strategy that does nothing.
+    #
+    class None < Strategy
+
+      # Does not encode text. Just returns nil.
+      #
+      def encoded text
+        nil
+      end
+
+      # Returns an empty index.
+      #
+      def generate_from index
+        {}
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/cacher/similarity/strategy.rb

@@ -0,0 +1,7 @@
+module Cacher
+  module Similarity
+    # Base class for all similarity strategies.
+    #
+    class Strategy; end
+  end
+end

data/lib/picky/cacher/similarity_generator.rb

@@ -0,0 +1,15 @@
+module Cacher
+
+  # Uses no similarity as default.
+  #
+  class SimilarityGenerator < Generator
+
+    # Generate a similarity index based on the given index.
+    #
+    def generate strategy = Similarity::None.new
+      strategy.generate_from self.index
+    end
+
+  end
+
+end

data/lib/picky/cacher/weights/default.rb

@@ -0,0 +1,7 @@
+module Cacher
+  module Weights
+    # Default is Logarithmic.
+    #
+    Default = Logarithmic.new
+  end
+end

data/lib/picky/cacher/weights/logarithmic.rb

@@ -0,0 +1,39 @@
+module Cacher
+
+  module Weights
+
+    # Uses a logarithmic weight.
+    # If for a key k we have x ids, the weight is:
+    # w(x): log(x)
+    # Special case: If x < 1, then we use 0.
+    #
+    class Logarithmic < Strategy
+
+      # Generates a partial index from the given index.
+      #
+      def generate_from index
+        index.inject({}) do |hash, text_ids|
+          text, ids = *text_ids
+          weight = weight_for ids.size
+          hash[text] ||= weight.round(2) if weight
+          hash
+        end
+      end
+
+      # Sets the weight value.
+      #
+      # If the size is 0 or one, we would get -Infinity or 0.0.
+      # Thus we do not set a value if there is just one. The default, dynamically, is 0.
+      #
+      # BUT: We need the value, even if 0. To designate that there is a weight!
+      #
+      def weight_for amount
+        return 0 if amount < 1
+        Math.log amount
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/cacher/weights/strategy.rb

@@ -0,0 +1,7 @@
+module Cacher
+  module Weights
+    # Superclass for weighing strategies.
+    #
+    class Strategy; end
+  end
+end

data/lib/picky/cacher/weights_generator.rb

@@ -0,0 +1,15 @@
+module Cacher
+
+  # Uses a logarithmic algorithm as default.
+  #
+  class WeightsGenerator < Generator
+
+    # Generate a weights index based on the given index.
+    #
+    def generate strategy = Weights::Logarithmic.new
+      strategy.generate_from self.index
+    end
+
+  end
+
+end

data/lib/picky/configuration/configuration.rb

@@ -0,0 +1,13 @@
+# module Configuration
+#   
+#   def self.indexes *types
+#     Indexes.new(*types).save
+#   end
+#   def self.type name, *fields
+#     Type.new name, *fields
+#   end
+#   def self.field name, options = {}
+#     Field.new name, options
+#   end
+#   
+# end