picky 0.2.3 → 0.2.4

data/lib/picky/cacher/partial/subtoken.rb

@@ -74,7 +74,7 @@ module Cacher
         index.each_key do |token|
           i += 1
           if i == 5000
-            puts "#{Time.now}: Generating partial tokens for token #{token}. This appears every 5000 tokens."
+            timed_exclaim "Generating partial tokens for token #{token}. This appears every 5000 tokens."
             i = 0
           end
           generate_for token, index, result

data/lib/picky/configuration/field.rb

@@ -7,7 +7,7 @@ module Configuration
     attr_reader :name, :indexed_name, :virtual
     attr_accessor :type # convenience
     def initialize name, options = {}
-      @name            = name
+      @name            = name.to_sym
       
       # TODO Dup the options?
       # TODO add source as option
@@ -18,7 +18,7 @@ module Configuration
       @indexed_name   = options.delete(:indexed_field) || name # TODO Rename to indexed_as?
       @virtual        = options.delete(:virtual)       || false
       
-      qualifiers = options[:qualifiers]
+      qualifiers = generate_qualifiers_from options
       Query::Qualifiers.add(name, qualifiers) if qualifiers
       
       # @remove          = options[:remove]        || false
@@ -26,6 +26,9 @@ module Configuration
       
       @options = options
     end
+    def generate_qualifiers_from options
+      options[:qualifiers] || options[:qualifier] && [options[:qualifier]] || [name]
+    end
     def source
       @source || type.source
     end

data/lib/picky/extensions/hash.rb

@@ -7,7 +7,9 @@ class Hash
   # TODO Still used? If yes, spec!
   #
   def dump_to path
-    File.open(path, 'w:binary') { |out_file| Marshal.dump self, out_file }
+    File.open(path, 'w:binary') do |out_file|
+      Yajl::Encoder.encode self, out_file
+    end
   end
   
   # Use yajl's encoding.

data/lib/picky/extensions/object.rb

@@ -0,0 +1,12 @@
+module Kernel
+  
+  
+  def timed_exclaim text
+    exclaim "#{Time.now.strftime("%H:%M:%S")}: #{text}"
+  end
+  
+  def exclaim text
+    puts text
+  end
+  
+end

data/lib/picky/generator.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+#
 require 'fileutils'
 
 module Picky
@@ -67,7 +69,13 @@ module Picky
         exclaim "Setting up Picky project \"#{name}\"."
         create_target_directory
         copy_all_files
-        exclaim "\"#{name}\" is a great project name! Have fun :)"
+        exclaim "\"#{name}\" is a great project name! Have fun :)\n"
+        exclaim "Next steps:"
+        exclaim ""
+        exclaim "cd #{name}"
+        exclaim "cat Gemfile # <- Do you need the mysql gem, for example?"
+        exclaim "bundle install"
+        exclaim "rake        # <- shows you where Picky needs input from you."
       end
       
       #
@@ -154,6 +162,8 @@ module Picky
         exclaim "#{entry} \x1b[31mexists\x1b[m, skipping."
       end
       
+      # TODO Remove?
+      #
       def exclaim something
         puts something
       end

data/lib/picky/index/bundle.rb

@@ -50,7 +50,7 @@ module Index
     # Identifier for this bundle.
     #
     def identifier
-      "#{name}:#{type.name}:#{category.name}"
+      "#{name}: #{type.name} #{category.name}"
     end
     
     # Point to category.
@@ -152,7 +152,7 @@ module Index
     # Generates a cache path.
     #
     def cache_path text
-      File.join cache_directory, "#{name}_#{text}.dump"
+      File.join cache_directory, "#{name}_#{text}.json"
     end
     def index_cache_path
       cache_path "#{category.name}_index"
@@ -172,18 +172,18 @@ module Index
       load_weights
     end
     def load_the index_method_name, path
-      self.send "#{index_method_name}=", Marshal.load(File.open(path, "r:binary")) if File.exists? path
+      self.send "#{index_method_name}=", Yajl::Parser.parse(File.open(path, 'r'), :symbolize_keys => true)
     end
     def load_index
-      puts "#{Time.now}: Loading the index for #{identifier} from the cache."
+      timed_exclaim "Loading the index for #{identifier} from the cache."
       load_the :index, index_cache_path
     end
     def load_similarity
-      puts "#{Time.now}: Loading the similarity for #{identifier} from the cache."
+      timed_exclaim "Loading the similarity for #{identifier} from the cache."
       load_the :similarity, similarity_cache_path
     end
     def load_weights
-      puts "#{Time.now}: Loading the weights for #{identifier} from the cache."
+      timed_exclaim "Loading the weights for #{identifier} from the cache."
       load_the :weights, weights_cache_path
     end
     
@@ -195,13 +195,15 @@ module Index
     # * generates derived indexes
     # * dumps all the indexes into files
     #
+    # TODO Rename to Source!!!
+    #
     def generate_caches_from_db
       cache_from_db_generation_message
       load_from_index_file
       generate_caches_from_memory
     end
     def cache_from_db_generation_message
-      puts "#{Time.now}: Generating caches from db for #{identifier}."
+      timed_exclaim "CACHE FROM SOURCE #{identifier}."
     end
     # Generates derived indexes from the index and dumps.
     #
@@ -212,7 +214,7 @@ module Index
       generate_derived
     end
     def cache_from_memory_generation_message
-      puts "#{Time.now}: Generating derived caches from memory for #{identifier}."
+      timed_exclaim "CACHE FROM MEMORY #{identifier}."
     end
     
     # Generates the weights and similarity from the main index.

data/lib/picky/index/category.rb

@@ -34,19 +34,19 @@ module Index
     end
     
     def identifier
-      "#{type.name}:#{name}"
+      "#{type.name} #{name}"
     end
     
     # Generates all caches for this category.
     #
     def generate_caches
-      timed_exclaim "Loading data from db for #{identifier}."
+      timed_exclaim "LOAD #{identifier}."
       generate_caches_from_db
-      timed_exclaim "Generating partial for #{identifier}."
+      timed_exclaim "PARTIAL #{identifier}."
       generate_partial
-      timed_exclaim "Generating caches from memory for #{identifier}."
+      timed_exclaim "CACHE #{identifier}."
       generate_caches_from_memory
-      timed_exclaim "Dumping all caches for #{identifier}."
+      timed_exclaim "DUMP #{identifier}."
       dump_caches
     end
     def generate_caches_from_db
@@ -62,16 +62,6 @@ module Index
       exact.dump
       partial.dump
     end
-    # TODO move to Kernel?
-    #
-    def timed_exclaim text
-      exclaim "#{Time.now}: #{text}"
-    end
-    # TODO move to Kernel?
-    #
-    def exclaim text
-      puts text
-    end
     
     # Used for testing.
     #

data/lib/picky/indexers/base.rb

@@ -35,7 +35,7 @@ module Indexers
       @field.source || raise_no_source
     end
     def raise_no_source
-      raise NoSourceSpecifiedException.new "No source given for #{@type.name}:#{@field.name}." # TODO field.identifier
+      raise NoSourceSpecifiedException.new "No source given for index:#{@type.name}, field:#{@field.name}." # TODO field.identifier
     end
     
     # Selects the original id (indexed id) and a column to process. The column data is called "token".
@@ -61,7 +61,7 @@ module Indexers
     end
     
     def indexing_message
-      puts "#{Time.now}: Indexing #{@type.name}:#{@field.name}:#{@field.indexed_name}." # TODO field.identifier
+      timed_exclaim "INDEX #{@type.name}, #{@field.name}" #:#{@field.indexed_name}." # TODO field.identifier
     end
     
   end

data/lib/picky/indexers/solr.rb

@@ -21,7 +21,7 @@ module Indexers
     # TODO Rewrite such that it works in batches.
     #
     def index
-      puts "#{Time.now}: Indexing solr for #{type.name}:#{fields.join(', ')}"
+      timed_exclaim "Indexing solr for #{type.name}:#{fields.join(', ')}"
       statement = "SELECT indexed_id, #{fields.join(',')} FROM #{type.snapshot_table_name}"
       
       # TODO Rewrite.

data/lib/picky/indexes.rb

@@ -16,7 +16,7 @@ module Indexes
 
     # Run in parallel.
     #
-    puts "Indexing using #{Cores.max_processors} processors."
+    timed_exclaim "Indexing using #{Cores.max_processors} processors."
     Cores.forked self.fields, :randomly => true do |field|
       # Reestablish DB connection.
       #

data/lib/picky/loader.rb

@@ -90,6 +90,7 @@ module Loader
     
     # Load extensions.
     #
+    load_relative 'extensions/object'
     load_relative 'extensions/array'
     load_relative 'extensions/symbol'
     load_relative 'extensions/module'

data/lib/picky/query/combination.rb

@@ -58,10 +58,11 @@ module Query
       [identifier, *@token.to_result]
     end
     
-    # exact/title:Flarbl:flarbl
+    # Example:
+    #  "exact title:Peter*:peter"
     #
     def to_s
-      "#{bundle.name}/#{to_result.join(':')}"
+      "#{bundle.name} #{to_result.join(':')}"
     end
     
   end

data/lib/picky/query/qualifiers.rb

@@ -13,7 +13,7 @@ module Query
     #
     def initialize normalized_qualifier, codes
       @normalized_qualifier = normalized_qualifier
-      @codes                = codes
+      @codes                = codes.map &:to_sym
     end
 
     # Will overwrite if the key is present in the hash.

data/prototype_project/Gemfile

@@ -9,15 +9,21 @@ gem 'rack-mount',       '~> 0.6.9'
 gem 'text',             '~> 0.2.0'
 gem 'yajl-ruby',        '~> 0.7.8', :require => 'yajl'
 
-# Should be optional, but isn't yet.
+# Should be optional, but isn't yet. Sorry.
 #
 gem 'activerecord',     '~> 2.3.8', :require => 'active_record'
 
-# Optional.
+
+
+
+# Optional. Makes rack faster.
 #
 gem 'rack_fast_escape', '2009.06.24'
 
-# Required by your project.
+# Optional. Use your preferred web server.
 #
 gem 'unicorn'
+
+# Optional. Use your preferred database adapter.
+#
 gem 'mysql'

data/prototype_project/Gemfile.lock

@@ -0,0 +1,33 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activerecord (2.3.10)
+      activesupport (= 2.3.10)
+    activesupport (2.3.10)
+    mysql (2.8.1)
+    picky (0.2.3)
+    rack (1.2.1)
+    rack-mount (0.6.13)
+      rack (>= 1.0.0)
+    rack_fast_escape (2009.06.24)
+      url_escape
+    text (0.2.0)
+    unicorn (1.1.4)
+      rack
+    url_escape (2009.06.24)
+    yajl-ruby (0.7.8)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord (~> 2.3.8)
+  bundler (>= 0.9.26)
+  mysql
+  picky (~> 0.2.0)
+  rack (~> 1.2.1)
+  rack-mount (~> 0.6.9)
+  rack_fast_escape (= 2009.06.24)
+  text (~> 0.2.0)
+  unicorn
+  yajl-ruby (~> 0.7.8)

data/prototype_project/Rakefile

@@ -1 +1,11 @@
-require 'picky-tasks'
+require 'picky-tasks'
+
+desc "Finds where Picky still needs input from you."
+task :todo do
+  if system "grep -e 'TO#{}DO.*' -n --color=always -R *"
+    puts "Picky needs a bit of input from you there. Thanks."
+  else
+    puts "Picky seems to be fine (no TO#{}DOs found)."
+  end
+end
+task :default => :todo

data/prototype_project/app/application.rb

@@ -1,14 +1,11 @@
 # encoding: utf-8
 #
-# This is your application.
-#
-# Have fun with Picky!
-#
-class PickySearch < Application # The App Constant needs to be identical in config.ru.
+class PickySearch < Application
   
-  # This is an example with books that you can adapt.
+  # TODO Adapt the generated example
+  #      (a library books finder) to what you need.
   #
-  # Note: Much more is possible, but let's start out super easy.
+  # Check the Wiki http://github.com/floere/picky/wiki for more options.
   #
   # Ask me if you have questions or specific requests.
   #

data/prototype_project/app/db.yml

@@ -2,6 +2,9 @@
 #
 # Note: ActiveRecord is used. Sorry.
 #
+# TODO Configure the database adapter
+#      or ignore and use a csv source.
+#
 adapter: mysql
 host: localhost
 username: root

data/prototype_project/config.ru

@@ -16,18 +16,19 @@ require 'picky'
 #
 Loader.load_application
 
-# Load the data. This loads data from cache files e.g. "some_index/*_index.dump" into Indexes[:some_index]
+# Load the indexes into the memory.
 #
 Indexes.load_from_cache
 
-# Use Harakiri middleware to kill unicorn child after X requests.
+# Use Harakiri middleware to kill worker child after X requests.
 #
-# See http://vimeo.com/12614970 for more info.
+# Works only with web servers that fork worker children and which
+# fork new children, like for example Unicorn.
 #
-    Rack::Harakiri.after = 50
+Rack::Harakiri.after = 50
 use Rack::Harakiri
 
-# Finalize the application and start accepting requests.
+# Start accepting requests.
 #
 # Note: Needs to be the same constant name as in app/application.rb.
 #

data/prototype_project/unicorn.ru

@@ -6,6 +6,10 @@ stdout_path       'log/unicorn.stdout.log'
 timeout           10
 worker_processes  2
 
+# After forking, the GC is disabled, because we
+# kill off the workers after x requests and fork
+# new ones – so the GC doesn't run.
+#
 after_fork do |_, _|
   GC.disable
 end

data/spec/lib/cacher/partial/subtoken_spec.rb

@@ -106,7 +106,7 @@ describe Cacher::Partial::Subtoken do
           end
         end
         it "should be fast" do
-          performance_of { @cacher.generate_from(@index) }.should < 0.005
+          performance_of { @cacher.generate_from(@index) }.should < 0.008
         end
       end
       describe "a bigger example with almost identical symbols" do
@@ -118,7 +118,7 @@ describe Cacher::Partial::Subtoken do
           end
         end
         it "should be fast" do
-          performance_of { @cacher.generate_from(@index) }.should < 0.003
+          performance_of { @cacher.generate_from(@index) }.should < 0.0045
         end
       end
     end

data/spec/lib/configuration/field_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+describe "Configuration::Field" do
+  
+  context "unit specs" do
+    context "name symbol" do
+      before(:each) do
+        @field = Configuration::Field.new :some_name
+      end
+      describe "generate_qualifiers_from" do
+        context "with qualifiers" do
+          it "uses the qualifiers" do
+            @field.generate_qualifiers_from(:qualifiers => :some_qualifiers).should == :some_qualifiers
+          end
+        end
+        context "without qualifiers" do
+          context "with qualifier" do
+            it "uses the [qualifier]" do
+              @field.generate_qualifiers_from(:qualifier => :some_qualifier).should == [:some_qualifier]
+            end
+          end
+          context "without qualifier" do
+            context "with name" do
+              it "uses the [name]" do
+                @field.generate_qualifiers_from(:nork => :blark).should == [:some_name]
+              end
+            end
+          end
+        end
+      end
+    end
+    context "name string" do
+      before(:each) do
+        @field = Configuration::Field.new 'some_name'
+      end
+      describe "generate_qualifiers_from" do
+        context "without qualifiers" do
+          context "without qualifier" do
+            context "with name" do
+              it "uses the [name]" do
+                @field.generate_qualifiers_from(:nork => :blark).should == [:some_name]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+  
+end

data/spec/lib/extensions/object_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Object do
+  
+  before(:each) do
+    @object = Object.new
+  end
+  
+  describe "exclaim" do
+    it "delegates to puts" do
+      @object.should_receive(:puts).once.with :bla
+      
+      @object.exclaim :bla
+    end
+  end
+  
+  describe "timed_exclaim" do
+    it "should exclaim right" do
+      Time.stub! :now => Time.parse('07-03-1977 12:34:56')
+      @object.should_receive(:exclaim).once.with "12:34:56: bla"
+      
+      @object.timed_exclaim 'bla'
+    end
+  end
+  
+end

data/spec/lib/extensions/symbol_spec.rb

@@ -17,7 +17,7 @@ describe Symbol do
     it "should be fast" do
       timed do
         @token.each_subtoken do |subtoken| end
-      end.should < 0.0004
+      end.should < 0.0005
     end
   end
   

data/spec/lib/index/bundle_spec.rb

@@ -14,7 +14,7 @@ describe Index::Bundle do
 
   describe 'identifier' do
     it 'should return a specific identifier' do
-      @index.identifier.should == 'some_name:some_type:some_category'
+      @index.identifier.should == 'some_name: some_type some_category'
     end
   end
 
@@ -141,7 +141,7 @@ describe Index::Bundle do
       it 'should raise' do
         lambda do
           @index.raise_unless_cache_exists
-        end.should raise_error("weights cache for some_name:some_type:some_category missing.")
+        end.should raise_error("weights cache for some_name: some_type some_category missing.")
       end
     end
     context 'similarity cache missing' do
@@ -153,7 +153,7 @@ describe Index::Bundle do
       it 'should raise' do
         lambda do
           @index.raise_unless_cache_exists
-        end.should raise_error("similarity cache for some_name:some_type:some_category missing.")
+        end.should raise_error("similarity cache for some_name: some_type some_category missing.")
       end
     end
     context 'index cache missing' do
@@ -165,7 +165,7 @@ describe Index::Bundle do
       it 'should raise' do
         lambda do
           @index.raise_unless_cache_exists
-        end.should raise_error("index cache for some_name:some_type:some_category missing.")
+        end.should raise_error("index cache for some_name: some_type some_category missing.")
       end
     end
     context 'all ok' do
@@ -210,17 +210,17 @@ describe Index::Bundle do
 
   describe 'weights_cache_path' do
     it 'should return the correct file name' do
-      @index.weights_cache_path.should == 'some/search/root/index/test/some_type/some_name_some_category_weights.dump'
+      @index.weights_cache_path.should == 'some/search/root/index/test/some_type/some_name_some_category_weights.json'
     end
   end
   describe 'similarity_cache_path' do
     it 'should return the correct file name' do
-      @index.similarity_cache_path.should == 'some/search/root/index/test/some_type/some_name_some_category_similarity.dump'
+      @index.similarity_cache_path.should == 'some/search/root/index/test/some_type/some_name_some_category_similarity.json'
     end
   end
   describe 'index_cache_path' do
     it 'should return the correct file name' do
-      @index.index_cache_path.should == 'some/search/root/index/test/some_type/some_name_some_category_index.dump'
+      @index.index_cache_path.should == 'some/search/root/index/test/some_type/some_name_some_category_index.json'
     end
   end
 

data/spec/lib/query/qualifiers_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Query::Qualifiers do
   
   before(:each) do
-    Query::Qualifiers.instance << Query::Qualifier.new(:test1, [:t1, :tt1, :ttt1])
+    Query::Qualifiers.instance << Query::Qualifier.new(:test1, ['t1', 'tt1', 'ttt1'])
     Query::Qualifiers.instance << Query::Qualifier.new(:test2, [:t2, :tt2, :ttt2])
     Query::Qualifiers.instance << Query::Qualifier.new(:test3, [:t3, :tt3, :ttt3])
     Query::Qualifiers.instance.prepare

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 3
-  version: 0.2.3
+  - 4
+  version: 0.2.4
 platform: ruby
 authors: 
 - Florian Hanke
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-15 00:00:00 +02:00
+date: 2010-10-24 00:00:00 +02:00
 default_executable: picky
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -68,6 +68,7 @@ files:
 - lib/picky/extensions/array.rb
 - lib/picky/extensions/hash.rb
 - lib/picky/extensions/module.rb
+- lib/picky/extensions/object.rb
 - lib/picky/extensions/symbol.rb
 - lib/picky/generator.rb
 - lib/picky/helpers/cache.rb
@@ -134,6 +135,7 @@ files:
 - prototype_project/app/README
 - prototype_project/config.ru
 - prototype_project/Gemfile
+- prototype_project/Gemfile.lock
 - prototype_project/log/README
 - prototype_project/Rakefile
 - prototype_project/script/console
@@ -150,11 +152,13 @@ files:
 - spec/lib/cacher/similarity_generator_spec.rb
 - spec/lib/cacher/weights/logarithmic_spec.rb
 - spec/lib/cacher/weights_generator_spec.rb
+- spec/lib/configuration/field_spec.rb
 - spec/lib/configuration/type_spec.rb
 - spec/lib/cores_spec.rb
 - spec/lib/extensions/array_spec.rb
 - spec/lib/extensions/hash_spec.rb
 - spec/lib/extensions/module_spec.rb
+- spec/lib/extensions/object_spec.rb
 - spec/lib/extensions/symbol_spec.rb
 - spec/lib/generator_spec.rb
 - spec/lib/helpers/cache_spec.rb
@@ -235,11 +239,13 @@ test_files:
 - spec/lib/cacher/similarity_generator_spec.rb
 - spec/lib/cacher/weights/logarithmic_spec.rb
 - spec/lib/cacher/weights_generator_spec.rb
+- spec/lib/configuration/field_spec.rb
 - spec/lib/configuration/type_spec.rb
 - spec/lib/cores_spec.rb
 - spec/lib/extensions/array_spec.rb
 - spec/lib/extensions/hash_spec.rb
 - spec/lib/extensions/module_spec.rb
+- spec/lib/extensions/object_spec.rb
 - spec/lib/extensions/symbol_spec.rb
 - spec/lib/generator_spec.rb
 - spec/lib/helpers/cache_spec.rb