picky 1.0.0 → 1.1.0

data/bin/picky

@@ -2,13 +2,13 @@
 #
 
 begin
-  require 'picky/generator'
+  require 'picky/cli'
 rescue LoadError => e
   require 'rubygems'
   picky_path = File.expand_path '../../lib', __FILE__
   $:.unshift(picky_path) if File.directory?(picky_path) && !$:.include?(picky_path)
-  require 'picky/generator'
+  require 'picky/cli'
 end
 
-generator = Picky::Generator.new
-generator.generate ARGV
+cli = Picky::CLI.new
+cli.execute *ARGV

data/lib/picky/application.rb

@@ -9,7 +9,7 @@
 #     # Your configuration goes here.
 #   end
 # The generator
-#   $ picky project project_name
+#   $ picky generate unicorn_server project_name
 # will generate an example <tt>project_name/app/application.rb</tt> file for you
 # with some example code inside.
 #

data/lib/picky/cli.rb

@@ -0,0 +1,58 @@
+module Picky
+  
+  # A very simple CLI selector.
+  #
+  class CLI # :nodoc:all
+    
+    def self.mapping
+      @@mapping
+    end
+    
+    # Execute a command.
+    #
+    # Note: By default, help is displayed. I.e. when no command is given.
+    #
+    def execute selector = nil, *args
+      executor_class, *params = selector && @@mapping[selector.to_sym] || Help
+      executor = executor_class.new
+      executor.execute selector, args, params
+    end
+    
+    class Base
+      def usage name, params
+        puts "Usage\n  picky #{name} #{params_to_s(params)}"
+      end
+      def params_to_s params
+        params.map { |param| "<#{param}>" }.join(' ') if params
+      end
+    end
+    class Generate < Base
+      def execute name, args, params
+        system "picky-generate #{args.join(' ')}"
+      end
+    end
+    class Help < Base
+      # Displays usage information.
+      #
+      def execute name, args, params
+        commands = Picky::CLI.mapping.map do |command, object_and_params|
+          _, *params = object_and_params
+          "  picky #{command} #{params_to_s(params)}"
+        end.join(?\n)
+        
+        puts "Possible commands:\n" + commands
+      end
+    end
+    
+    # Maps commands to the other gem's command.
+    #
+    # TODO Add optional params.
+    #
+    @@mapping = {
+      :generate => [Generate, 'thing_to_generate: e.g. "unicorn_server"', :parameters],
+      :help     => [Help]
+    }
+    
+  end
+  
+end

data/lib/picky/extensions/array.rb

@@ -35,6 +35,8 @@ class Array # :nodoc:all
   def sort_by_levenshtein! from
     from = from.to_s
     sort! do |this, that|
+      # TODO Cache for speed?
+      #
       Text::Levenshtein.distance(this.to_s, from) <=> Text::Levenshtein.distance(that.to_s, from)
     end
   end

data/lib/picky/loader.rb

@@ -260,10 +260,6 @@ module Loader # :nodoc:all
     #
     # load_relative 'solr/schema_generator'
     load_relative 'cores'
-    
-    # Load generation.
-    #
-    load_relative 'generator'
   end
 
 end

data/lib/picky/query/weigher.rb

@@ -47,6 +47,8 @@ module Query
         #  [c,e]
         # ]
         #
+        # TODO Use transpose?
+        #
         expanded_combinations = expanded_combinations.shift.zip *expanded_combinations
         
         # Wrap into a real combination.

data/lib/picky/query/weights.rb

@@ -38,7 +38,7 @@ module Query
     # Note: Cache this if more complicated weighings become necessary.
     #
     def score combinations
-      # TODO Beautify?
+      # TODO Beautify? Use categories for weights?
       #
       # weight_for combinations.map(&:category).clustered_uniq_fast.map!(&:name)
       

data/lib/picky/sources/db.rb

@@ -107,22 +107,23 @@ module Sources
     
     # Harvests the data to index in chunks.
     #
-    def harvest type, category
+    def harvest type, category, &block
       connect_backend
       
       (0..count(type)).step(chunksize) do |offset|
-        get_data(type, category, offset).each do |indexed_id, text|
-          next unless text
-          text.force_encoding 'utf-8' # TODO Still needed?
-          yield indexed_id, text
-        end
+        get_data type, category, offset, &block
       end
     end
     
     # Gets the data from the backend.
     #
-    def get_data type, category, offset # :nodoc:
-      database.connection.execute harvest_statement_with_offset(type, category, offset)
+    def get_data type, category, offset, &block # :nodoc:
+      select_statement = harvest_statement_with_offset(type, category, offset)
+      database.connection.execute(select_statement).each do |indexed_id, text|
+        next unless text
+        text.force_encoding 'utf-8' # TODO Still needed? Or move to backend?
+        yield indexed_id, text
+      end
     end
     
     # Builds a harvest statement for getting data to index.

data/lib/picky/tokenizers/base.rb

@@ -4,8 +4,9 @@ module Tokenizers # :nodoc:all
   #
   class Base
     
-    # TODO use frozen EMPTY_STRING for ''
+    # TODO Move EMPTY_STRING top level.
     #
+    EMPTY_STRING = ''.freeze
     
     # Stopwords.
     #
@@ -13,7 +14,7 @@ module Tokenizers # :nodoc:all
       @remove_stopwords_regexp = regexp
     end
     def remove_stopwords text
-      text.gsub! @remove_stopwords_regexp, '' if @remove_stopwords_regexp
+      text.gsub! @remove_stopwords_regexp, EMPTY_STRING if @remove_stopwords_regexp
       text
     end
     @@non_single_stopword_regexp = /^\b[\w:]+?\b[\.\*\~]?\s?$/
@@ -30,7 +31,7 @@ module Tokenizers # :nodoc:all
       @removes_characters_regexp = regexp
     end
     def remove_illegals text
-      text.gsub! @removes_characters_regexp, '' if @removes_characters_regexp
+      text.gsub! @removes_characters_regexp, EMPTY_STRING if @removes_characters_regexp
       text
     end
     
@@ -66,7 +67,7 @@ module Tokenizers # :nodoc:all
       @removes_characters_after_splitting_regexp = regexp
     end
     def remove_after_normalizing_illegals text
-      text.gsub! @removes_characters_after_splitting_regexp, '' if @removes_characters_after_splitting_regexp
+      text.gsub! @removes_characters_after_splitting_regexp, EMPTY_STRING if @removes_characters_after_splitting_regexp
     end
     
     # Substitute Characters with this substituter.

data/lib/picky/tokenizers/index.rb

@@ -1,4 +1,5 @@
 module Tokenizers
+  
   # The base indexing tokenizer.
   #
   # Override in indexing subclasses and define in configuration.
@@ -15,11 +16,10 @@ module Tokenizers
     # Default indexing preprocessing hook.
     #
     # Does:
-    #   1. Umlaut substitution.
-    #   2. Downcasing.
-    #   3. Remove illegal expressions.
-    #   4. Contraction.
-    #   5. Remove non-single stopwords. (Stopwords that occur with other words)
+    # 1. Character substitution.
+    # 2. Downcasing.
+    # 3. Remove illegal expressions.
+    # 4. Remove non-single stopwords. (Stopwords that occur with other words)
     #
     def preprocess text
       text = substitute_characters text
@@ -27,7 +27,7 @@ module Tokenizers
       remove_illegals text
       # we do not remove single stopwords for an entirely different
       # reason than in the query tokenizer.
-      # An indexed thing with just name "UND" (a stopword) should not lose its name.
+      # An indexed thing with just name "UND" (a possible stopword) should not lose its name.
       #
       remove_non_single_stopwords text
       text
@@ -36,8 +36,8 @@ module Tokenizers
     # Default indexing pretokenizing hook.
     #
     # Does:
-    #   1. Split the text into words.
-    #   2. Normalize each word.
+    # 1. Split the text into words.
+    # 2. Normalize each word.
     #
     # TODO Rename into wordize? Or somesuch?
     #
@@ -60,7 +60,7 @@ module Tokenizers
     #
     # Override in subclasses to redefine behaviour.
     #
-    # TODO Make parametrizable! reject { |token| }
+    # TODO TODO TODO Make parametrizable! reject { |token| }
     #
     def reject tokens
       tokens.reject! &:blank?

data/lib/tasks/server.rake

@@ -14,7 +14,6 @@ namespace :server do
   # desc "Start the unicorns. (Wehee!)"
   task :start => :framework do
     chdir_to_root
-    # Rake::Task[:"solr:start"].invoke # TODO Move to better place.
     daemonize = PICKY_ENVIRONMENT == 'production' ? '-D' : ''
     command = "export PICKY_ENV=#{PICKY_ENVIRONMENT}; unicorn -c unicorn.ru #{daemonize}".strip
     puts "Running \`#{command}\`."
@@ -24,7 +23,6 @@ namespace :server do
   # desc "Stop the unicorns. (Blam!)"
   task :stop => :framework do
     `kill -QUIT #{current_pid}` if current_pid
-    # Rake::Task[:"solr:stop"].invoke # TODO Move to better place.
   end
   
   # desc "Restart the unicorns."

data/spec/lib/rack/harakiri_spec.rb

@@ -4,6 +4,7 @@ require 'spec_helper'
 describe Rack::Harakiri do
   before(:each) do
     @app = stub :app
+    Process.stub! :kill # not taking any chances
   end
   context "defaults" do
     before(:each) do

data/spec/lib/sources/db_spec.rb

@@ -17,13 +17,33 @@ describe Sources::DB do
   end
   
   describe "get_data" do
-    it "delegates" do
-      type     = stub :type,     :name => :some_type
-      category = stub :category, :from => :some_category
-      
-      @connection.should_receive(:execute).once.with 'SELECT indexed_id, some_category FROM some_type_type_index st WHERE st.id > some_offset LIMIT 25000'
-      
-      @source.get_data type, category, :some_offset
+    before(:each) do
+      @type     = stub :type,     :name => :some_type
+      @category = stub :category, :from => :some_category
+    end
+    context 'no data' do
+      it "delegates to the connection" do
+
+        @connection.should_receive(:execute).
+                    once.
+                    with('SELECT indexed_id, some_category FROM some_type_type_index st WHERE st.id > some_offset LIMIT 25000').
+                    and_return []
+
+        @source.get_data @type, @category, :some_offset
+      end
+    end
+    context 'with data' do
+      it 'yields to the caller' do
+        @connection.should_receive(:execute).
+                    any_number_of_times.
+                    with('SELECT indexed_id, some_category FROM some_type_type_index st WHERE st.id > some_offset LIMIT 25000').
+                    and_return [[1, 'text']]
+
+        @source.get_data @type, @category, :some_offset do |id, text|
+          id.should == 1
+          text.should == 'text'
+        end
+      end
     end
   end
   
@@ -67,48 +87,22 @@ describe Sources::DB do
     end
   end
   
-  # TODO Redo.
-  #
-  # describe "harvest" do
-  #   before(:each) do
-  #     @source.stub! :harvest_statement_with_offset
-  #   end
-  #   context 'expectations' do
-  #     before(:each) do
-  #       @connection.stub! :execute => []
-  #       @connection.stub! :select_value
-  #     end
-  #     after(:each) do
-  #       @source.harvest :type_name, :some_field
-  #     end
-  #     context "with WHERE" do
-  #       before(:each) do
-  #         @source.stub! :select_statement => 'bla WHERE blu'
-  #       end
-  #       it "should connect" do
-  #         @source.should_receive(:connect_backend).once.with
-  #       end
-  #       it "should call the harvest statement with an offset" do
-  #         @source.should_receive(:harvest_statement_with_offset).once.with :some_type, :some_field, :some_offset
-  #       end
-  #     end
-  #     context "without WHERE" do
-  #       it "should connect" do
-  #         @adapter.should_receive(:connect).once.with
-  #       end
-  #       it "should call the harvest statement with an offset" do
-  #         @source.should_receive(:harvest_statement_with_offset).once.with :some_type, :some_field, :some_offset
-  #       end
-  #     end
-  #   end
-  #   context 'returns' do
-  #     it "should return whatever the execute statement returns" do
-  #       @connection.stub! :execute => :some_result
-  #       
-  #       @source.harvest(:some_type, :some_field).should == :some_result
-  #     end
-  #   end
-  # end
+  describe 'harvest' do
+    before(:each) do
+      @type     = stub :type,     :name => :some_type
+      @category = stub :category, :name => :some_category
+
+      @source.should_receive(:get_data).any_number_of_times.and_return [[:some_id, 'some_text']].cycle
+      @source.stub! :count => 17
+    end
+    it 'calls connect_backend' do
+      @source.should_receive(:connect_backend).once.with()
+      
+      @source.harvest @type, @category do |id, text|
+        p [id, text]
+      end
+    end
+  end
 
   describe "harvest_statement_with_offset" do
     before(:each) do

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 1
+  - 1
   - 0
-  - 0
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors: 
 - Florian Hanke
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-09 00:00:00 +01:00
+date: 2010-12-12 00:00:00 +01:00
 default_executable: picky
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -63,6 +63,7 @@ files:
 - lib/picky/cacher/weights_generator.rb
 - lib/picky/calculations/location.rb
 - lib/picky/character_substituters/west_european.rb
+- lib/picky/cli.rb
 - lib/picky/configuration/index.rb
 - lib/picky/cores.rb
 - lib/picky/ext/maybe_compile.rb
@@ -72,7 +73,6 @@ files:
 - lib/picky/extensions/module.rb
 - lib/picky/extensions/object.rb
 - lib/picky/extensions/symbol.rb
-- lib/picky/generator.rb
 - lib/picky/helpers/cache.rb
 - lib/picky/helpers/gc.rb
 - lib/picky/helpers/measuring.rb
@@ -147,19 +147,6 @@ files:
 - lib/tasks/statistics.rake
 - lib/tasks/try.rake
 - lib/picky/ext/ruby19/performant.c
-- project_prototype/app/application.rb
-- project_prototype/app/db.yml
-- project_prototype/app/library.csv
-- project_prototype/app/logging.rb
-- project_prototype/app/README
-- project_prototype/config.ru
-- project_prototype/Gemfile
-- project_prototype/log/README
-- project_prototype/Rakefile
-- project_prototype/script/console
-- project_prototype/tmp/pids/README
-- project_prototype/tmp/README
-- project_prototype/unicorn.ru
 - spec/ext/performant_spec.rb
 - spec/lib/application_spec.rb
 - spec/lib/cacher/cacher_strategy_spec.rb

data/lib/picky/generator.rb

@@ -1,198 +0,0 @@
-# encoding: utf-8
-#
-require 'fileutils'
-
-module Picky
-  
-  # Thrown when no generator for the command
-  #   picky <command> <options>
-  # is found.
-  #
-  class NoGeneratorError < StandardError # :nodoc:all
-    
-    def initialize generator
-      super usage + possible_commands(generator.types)
-    end
-    
-    def usage
-      "\nUsage:\n" +
-      "picky <command> <params>\n" +
-      ?\n
-    end
-    
-    def possible_commands types
-      "Possible commands:\n" +
-      types.map do |name, klass_params|
-        result = "picky #{name}"
-        _, params = *klass_params
-        result << ' ' << [*params].map { |param| "<#{param}>" }.join(' ') if params
-        result
-      end.join(?\n) + ?\n
-    end
-    
-  end
-  
-  # This is a very simple project generator.
-  # Not at all like Padrino's or Rails'.
-  # (No diss, just by way of a faster explanation)
-  #
-  # Basically copies a prototype project into a newly generated directory.
-  #
-  class Generator # :nodoc:all
-    
-    attr_reader :types
-    
-    def initialize
-      @types = {
-        project: [Project, :project_name]
-      }
-    end
-    
-    # Run the generators with this command.
-    #
-    # This will "route" the commands to the right specific generator.
-    #
-    def generate args
-      generator = generator_for *args
-      generator.generate
-    end
-    
-    #
-    #
-    def generator_for identifier, *args
-      generator_info = types[identifier.to_sym]
-      raise NoGeneratorError.new(self) unless generator_info
-      generator_class = generator_info.first
-      generator_for_class generator_class, identifier, *args
-    end
-    
-    #
-    #
-    def generator_for_class klass, *args
-      klass.new *args
-    end
-    
-    # Generates a new Picky project.
-    #
-    # Example:
-    #   > picky project my_lovely_project
-    #
-    class Project
-      
-      attr_reader :name, :project_prototype_basedir
-      
-      def initialize identifier, name, *args
-        @name = name
-        @project_prototype_basedir = File.expand_path '../../../project_prototype', __FILE__
-      end
-      
-      #
-      #
-      def generate
-        exclaim "Setting up Picky project \"#{name}\"."
-        create_target_directory
-        copy_all_files
-        exclaim "\"#{name}\" is a great project name! Have fun :)\n"
-        exclaim ""
-        exclaim "Next steps:"
-        exclaim "1. cd #{name}"
-        exclaim "2. bundle install"
-        exclaim "3. rake index"
-        exclaim "4. rake start"
-        exclaim "5. rake           # (optional) shows you where Picky needs input from you"
-        exclaim "                  #            if you want to define your own search."
-      end
-      
-      #
-      #
-      def create_target_directory
-        if File.exists?(target_directory)
-          exists target_directory
-        else
-          FileUtils.mkdir target_directory
-          created target_directory
-        end
-      end
-      
-      #
-      #
-      def copy_all_files
-        all_prototype_files.each do |filename|
-          next if filename.match(/\.textile$/)
-          copy_single_file filename
-        end
-      end
-      
-      #
-      #
-      def target_filename_for filename
-        filename.gsub(%r{#{project_prototype_basedir}}, target_directory)
-      end
-      #
-      #
-      def copy_single_file filename
-        target = target_filename_for filename
-        if File.exists? target
-          exists target
-        else
-          smart_copy filename, target
-        end
-      end
-      
-      # Well, "smart" ;)
-      #
-      def smart_copy filename, target
-        # p "Trying to copy #{filename} -> #{target}"
-        FileUtils.copy_file filename, target
-        created target
-      rescue Errno::EISDIR
-        # p "EISDIR #{filename} -> #{target}"
-        FileUtils.rm target
-        FileUtils.mkdir_p target unless Dir.exists?(target)
-        created target
-      rescue Errno::EEXIST
-        # p "EEXIST #{filename} -> #{target}"
-        exists target
-      rescue Errno::ENOTDIR
-        # p "ENOTDIR #{filename} -> #{target}"
-        FileUtils.mkdir_p File.dirname(target) rescue nil
-        retry
-      rescue Errno::ENOENT => e
-        # p "ENOENT #{filename} -> #{target}"
-        if File.exists? filename
-          FileUtils.mkdir_p File.dirname(target)
-          retry
-        else
-          raise e
-        end
-      end
-      
-      #
-      #
-      def all_prototype_files
-        Dir[File.join(project_prototype_basedir, '**', '*')]
-      end
-      
-      #
-      #
-      def target_directory
-        File.expand_path name, Dir.pwd
-      end
-      
-      def created entry
-        exclaim "#{entry} \x1b[32mcreated\x1b[m."
-      end
-      
-      def exists entry
-        exclaim "#{entry} \x1b[31mexists\x1b[m, skipping."
-      end
-      
-      def exclaim something
-        puts something
-      end
-      
-    end
-    
-  end
-  
-end

data/project_prototype/Gemfile

@@ -1,30 +0,0 @@
-source :gemcutter
-
-# Gems required by Picky.
-#
-gem 'picky',            '~> 1.0.0'
-gem 'rake'
-gem 'bundler'
-gem 'rack',             '~> 1.2.1'
-gem 'rack-mount',       '~> 0.6.9'
-gem 'text',             '~> 0.2.0'
-gem 'yajl-ruby',        '~> 0.7.8', :require => 'yajl'
-
-# Should be optional, but isn't yet. Sorry.
-#
-gem 'activerecord',     '~> 2.3.8', :require => 'active_record'
-
-
-
-
-# Optional. Makes rack faster.
-#
-gem 'rack_fast_escape', '2009.06.24'
-
-# Optional. Use your preferred web server.
-#
-gem 'unicorn'
-
-# Optional. Use your preferred database adapter.
-#
-# gem 'mysql'

data/project_prototype/Rakefile

@@ -1,11 +0,0 @@
-require 'picky-tasks'
-
-desc "Finds where Picky still needs input from you."
-task :todo do
-  if system "grep -e 'TO#{}DO.*' -n --color=always -R *"
-    puts "Picky needs a bit of input from you there. Thanks."
-  else
-    puts "Picky seems to be fine (no TO#{}DOs found)."
-  end
-end
-task :default => :todo

data/project_prototype/app/README

@@ -1,5 +0,0 @@
-All your application definitions go here.
-
-/app needs a few files defined by picky:
-  * application.rb - how to handle indexing, queries, routing.
-  * logging.rb