picky 0.0.0 → 0.0.2

data/lib/picky/solr/schema_generator.rb

@@ -0,0 +1,73 @@
+module Solr
+  class SchemaGenerator
+
+    attr_reader :types
+
+    # Takes an array of index type configs.
+    #
+    def initialize configuration
+      @types = configuration.types
+    end
+
+    #
+    #
+    def generate
+      generate_schema_for bound_field_names
+    end
+
+    # Returns a binding with the values needed for the schema xml.
+    #
+    def bound_field_names
+      field_names = combine_field_names
+      binding
+    end
+
+    # TODO
+    #
+    def combine_field_names
+      field_names = []
+      types.each do |type|
+        field_names += type.solr_fields.map(&:name)
+      end
+      field_names.uniq!
+      field_names
+    end
+
+    #
+    #
+    def generate_schema_for binding
+      template_text = read_template
+      result = evaluate_erb template_text, binding
+      write result
+    end
+
+    #
+    #
+    def evaluate_erb text, binding
+      require 'erb'
+      template = ERB.new text
+      template.result binding
+    end
+
+    #
+    #
+    def read_template
+      template_path = File.join SEARCH_ROOT, 'solr', 'conf', 'schema.xml.erb'
+      schema = ''
+      File.open(template_path, 'r') do |f|
+        schema = f.read
+      end
+      schema
+    end
+
+    #
+    #
+    def write result
+      schema_path = File.join SEARCH_ROOT, 'solr', 'conf', 'schema.xml'
+      File.open(schema_path, 'w') do |f|
+        f << result
+      end
+    end
+
+  end
+end

data/lib/picky/sources/base.rb

@@ -0,0 +1,19 @@
+module Sources
+  
+  class Base
+    
+    def take_snapshot type
+      
+    end
+    
+    def count type
+      
+    end
+    
+    def harvest offset
+      
+    end
+    
+  end
+  
+end

data/lib/picky/sources/csv.rb

@@ -0,0 +1,30 @@
+module Sources
+  
+  class CSV < Base
+    
+    attr_reader :file_name
+    
+    def initialize file_name, *field_names
+      @file_name = file_name
+      @field_names
+    end
+    
+    # Counts all the entries that are used for the index.
+    #
+    def count type
+      `wc -l #{file_name}`
+    end
+    
+    # Harvests the data to index, chunked.
+    #
+    # Subclasses should override harvest_statement to define how their data is found.
+    # Example:
+    #   "SELECT indexed_id, value FROM bla_table st WHERE kind = 'bla'"
+    #
+    def harvest offset
+      File.open file_name, 'r'
+    end
+    
+  end
+  
+end

data/lib/picky/sources/db.rb

@@ -0,0 +1,77 @@
+module Sources
+  
+  class DB < Base
+    
+    attr_reader :select_statement, :database
+    
+    def initialize select_statement, database_adapter
+      @select_statement = select_statement
+      @database         = database_adapter
+    end
+    
+    # Take the snapshot.
+    #
+    def take_snapshot type
+      database.connect
+      
+      origin = snapshot_table_name type
+
+      database.connection.execute "DROP TABLE IF EXISTS #{origin}"
+      database.connection.execute "CREATE TABLE #{origin} AS #{select_statement}"
+      database.connection.execute "ALTER TABLE #{origin} CHANGE COLUMN id indexed_id INTEGER"
+      database.connection.execute "ALTER TABLE #{origin} ADD COLUMN id INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT"
+
+      # Execute any special queries this type needs executed.
+      #
+      database.connection.execute type.after_indexing if type.after_indexing
+    end
+    
+    # Counts all the entries that are used for the index.
+    #
+    def count type
+      database.connection.select_value("SELECT COUNT(id) FROM #{snapshot_table_name(type)}").to_i
+    end
+    
+    # Ok here?
+    #
+    def snapshot_table_name type
+      "#{type.name}_type_index"
+    end
+    
+    # Harvests the data to index, chunked.
+    #
+    # Subclasses should override harvest_statement to define how their data is found.
+    # Example:
+    #   "SELECT indexed_id, value FROM bla_table st WHERE kind = 'bla'"
+    #
+    def harvest type, field, offset, chunksize
+      database.connect
+      
+      database.connection.execute harvest_statement_with_offset(type, field, offset, chunksize)
+    end
+    
+    # Base harvest statement for dbs.
+    #
+    def harvest_statement type, field
+      "SELECT indexed_id, #{field.name} FROM #{snapshot_table_name(type)} st"
+    end
+    
+    # Builds a harvest statement for getting data to index.
+    #
+    # TODO Use the adapter for this.
+    #
+    def harvest_statement_with_offset type, field, offset, chunksize
+      statement = harvest_statement type, field
+      
+      if statement.include? 'WHERE'
+        statement += ' AND'
+      else
+        statement += ' WHERE'
+      end
+      
+      "#{statement} st.id > #{offset} LIMIT #{chunksize}"
+    end
+    
+  end
+  
+end

data/lib/picky/tokenizers/base.rb

@@ -0,0 +1,130 @@
+module Tokenizers
+  # The Tokenizers
+  #
+  class Base # TODO Rename Strategy.
+
+    # Stopwords.
+    #
+    def self.stopwords regexp
+      define_method :remove_stopwords do |text|
+        text.gsub! regexp, ''
+      end
+      # Use this method if you don't want to remove
+      # stopwords if it is just one word.
+      #
+      @@non_single_stopword_regexp = /^\b[\w:]+?\b[\.\*\~]?\s?$/
+      define_method :remove_non_single_stopwords do |text|
+        return text if text.match @@non_single_stopword_regexp
+        remove_stopwords text
+      end
+    end
+    def remove_stopwords text; end
+
+    # Contraction.
+    #
+    def self.contract_expressions what, to_what
+      define_method :contract do |text|
+        text.gsub! what, to_what
+      end
+    end
+    def contract text; end
+
+    # Illegals.
+    #
+    # TODO Should there be a legal?
+    #
+    def self.illegal_characters regexp
+      define_method :remove_illegals do |text|
+        text.gsub! regexp, ''
+      end
+    end
+    def remove_illegals text; end
+
+    # Splitting.
+    #
+    def self.split_text_on regexp
+      define_method :split do |text|
+        text.split regexp
+      end
+    end
+    def split text; end
+
+    # Normalizing.
+    #
+    def self.normalize_words regexp_replaces
+      define_method :normalize_with_patterns do |text|
+        regexp_replaces.each do |regex, replace|
+          # This should be sufficient
+          #
+          text.gsub!(regex, replace) and break
+        end
+        remove_after_normalizing_illegals text
+        text
+      end
+    end
+    def normalize_with_patterns text; end
+
+    # Illegal after normalizing.
+    #
+    # TODO Rename illegal_after_tokenizing?
+    #
+    def self.illegal_characters_after regexp
+      define_method :remove_after_normalizing_illegals do |text|
+        text.gsub! regexp, ''
+      end
+    end
+    def remove_after_normalizing_illegals text; end
+
+    # Returns a number of tokens, generated from the given text.
+    #
+    # Note:
+    #  * preprocess, pretokenize are hooks
+    #
+    def tokenize text
+      text   = preprocess text  # processing the text
+      return empty_tokens if text.blank?
+      words  = pretokenize text # splitting and preparations for tokenizing
+      return empty_tokens if words.empty?
+      tokens = tokens_for words # creating tokens / strings
+               process tokens   # processing tokens / strings
+    end
+
+    # Hooks.
+    #
+
+    # Preprocessing.
+    #
+    def preprocess text; end
+    # Pretokenizing.
+    #
+    def pretokenize text; end
+    # Postprocessing.
+    #
+    def process tokens
+      reject tokens    # Reject any tokens that don't meet criteria
+      tokens
+    end
+
+    # Rejects blank tokens.
+    #
+    def reject tokens
+      tokens.reject! &:blank?
+    end
+    # Converts words into real tokens.
+    #
+    def tokens_for words
+      ::Query::Tokens.new words.collect! { |word| token_for word }
+    end
+    # Turns non-blank text into symbols.
+    #
+    def symbolize text
+      text.blank? ? nil : text.to_sym
+    end
+    # Returns a tokens object.
+    #
+    def empty_tokens
+      ::Query::Tokens.new
+    end
+
+  end
+end

data/lib/picky/tokenizers/default.rb

@@ -0,0 +1,3 @@
+module Tokenizers
+  Default = Index
+end

data/lib/picky/tokenizers/index.rb

@@ -0,0 +1,73 @@
+module Tokenizers
+  # The base indexing tokenizer.
+  #
+  # Override in indexing subclasses and define in configuration.
+  #
+  class Index < Base
+    
+    include UmlautSubstituter
+    
+    # Default handling definitions. Override in config.
+    #
+    illegal_characters(//)
+    stopwords(//)
+    contract_expressions(//, '')
+    split_text_on(/\s/)
+    normalize_words([])
+    illegal_characters_after(//)
+    
+    # Default indexing preprocessing hook.
+    #
+    # Does:
+    #   1. Umlaut substitution.
+    #   2. Downcasing.
+    #   3. Remove illegal expressions.
+    #   4. Contraction.
+    #   5. Remove non-single stopwords. (Stopwords that occur with other words)
+    #
+    def preprocess text
+      text = substitute_umlauts text
+      text.downcase!
+      remove_illegals text
+      contract text
+      # we do not remove single stopwords for an entirely different
+      # reason than in the query tokenizer.
+      # An indexed thing with just name "UND" (a stopword) should not lose its name.
+      #
+      remove_non_single_stopwords text
+      text
+    end
+    
+    # Default indexing pretokenizing hook.
+    #
+    # Does:
+    #   1. Split the text into words.
+    #   2. Normalize each word.
+    #
+    # TODO Rename into wordize? Or somesuch?
+    #
+    def pretokenize text
+      words = split text
+      words.collect! do |word|
+        normalize_with_patterns word
+        word
+      end
+    end
+    
+    # Does not actually return a token, but a
+    # symbol "token".
+    #
+    def token_for text
+      symbolize text
+    end
+    
+    # Rejects tokens if they are too short (or blank).
+    #
+    # Override in subclasses to redefine behaviour.
+    #
+    def reject tokens
+      tokens.reject! { |token| token.to_s.size < 2 }
+    end
+    
+  end
+end

data/lib/picky/tokenizers/query.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+#
+module Tokenizers
+  
+  # There are a few class methods that you can use to configure how a query works.
+  #
+  # illegal_characters regexp
+  # illegal_after_normalizing regexp
+  # stopwords regexp
+  # contract_expressions regexp, to_string
+  # split_text_on regexp
+  # normalize_words [[/regexp1/, 'replacement1'], [/regexp2/, 'replacement2']]
+  #
+  class Query < Base
+    
+    include UmlautSubstituter
+    
+    # Default query tokenizer behaviour. Override in config.
+    #
+    illegal_characters(//)
+    stopwords(//)
+    contract_expressions(//, '')
+    split_text_on(/\s/)
+    normalize_words([])
+    illegal_characters_after(//)
+    
+    def preprocess text
+      remove_illegals text             # Remove illegal characters
+      remove_non_single_stopwords text # remove stop words
+      contract text                    # contract st sankt etc
+      text
+    end
+    
+    # Split the text and put some back together.
+    #
+    def pretokenize text
+      split text
+    end
+    
+    # Let each token process itself.
+    # Reject, limit, and partialize tokens.
+    #
+    def process tokens
+      tokens.tokenize_with self
+      tokens.reject          # Reject any tokens that don't meet criteria
+      tokens.cap             # Cut off superfluous tokens
+      tokens.partialize_last # Set certain tokens as partial
+      tokens
+    end
+    
+    # Called by the token.
+    #
+    # TODO Perhaps move to Normalizer?
+    #
+    def normalize text
+      text = substitute_umlauts text # Substitute special characters TODO Move to subclass
+      text.downcase!                 # Downcase all text
+      normalize_with_patterns text   # normalize
+      text.to_sym                    # symbolize
+    end
+    
+    # Returns a token for a word.
+    # The basic query tokenizer uses new tokens.
+    #
+    def token_for word
+      ::Query::Token.processed word
+    end
+    
+  end
+end

data/lib/picky/umlaut_substituter.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+#
+module UmlautSubstituter
+  def substitute_umlauts text
+    trans = ActiveSupport::Multibyte.proxy_class.new(text).normalize(:kd)
+
+    # substitute special cases
+    #
+    trans.gsub!('ß', 'ss')
+
+    # substitute umlauts (of A,O,U,a,o,u)
+    #
+    trans.gsub!(/([AOUaou])\314\210/u, '\1e')
+
+    # get rid of ecutes, graves and …
+    #
+    trans.unpack('U*').select { |cp|
+      cp < 0x0300 || cp > 0x035F
+    }.pack('U*')
+  end
+end

data/lib/picky-tasks.rb

@@ -0,0 +1,6 @@
+all_rake_files = File.expand_path File.join(File.dirname(__FILE__), 'tasks', '*.rake')
+
+Dir[all_rake_files].each do |rakefile|
+  next if rakefile =~ /spec\.rake$/
+  load rakefile
+end

data/lib/picky.rb

@@ -0,0 +1,18 @@
+# Require the constants.
+#
+# TODO Move to app?
+#
+require File.expand_path(File.join(File.dirname(__FILE__), 'constants'))
+
+# Library bundling.
+#
+require File.expand_path(File.join(File.dirname(__FILE__), 'bundling'))
+
+# Loader which handles framework and app loading.
+#
+require File.expand_path(File.join(File.dirname(__FILE__), 'picky', 'loader'))
+
+# Load the framework
+#
+Loader.load_framework
+puts "Loaded picky with environment '#{SEARCH_ENVIRONMENT}' in #{SEARCH_ROOT} on Ruby #{RUBY_VERSION}."

data/lib/tasks/application.rake

@@ -0,0 +1,5 @@
+desc "Loads the application, including its configuration."
+task :application => :framework do
+  puts "Running rake task 'application'."
+  Loader.load_application
+end

data/lib/tasks/cache.rake

@@ -0,0 +1,53 @@
+namespace :cache do
+  
+  namespace :structure do
+    
+    desc 'create the directory structure for the cache indexes'
+    task :create => :application do
+      Indexes.create_directory_structure
+      puts "Directory structure generated."
+    end
+    
+  end
+  
+  desc "Generates the index cache files."
+  task :generate => :application do
+    Indexes.generate_caches
+    puts "Caches generated."
+  end
+
+  desc "Generates a specific index cache file like field=books:title. Note: Index tables need to be there. Will generate just the cache."
+  task :only => :application do
+    type_and_field = ENV['FIELD'] || ENV['field']
+    type, field = type_and_field.split ':'
+    Indexes.generate_cache_only type.to_sym, field.to_sym
+  end
+  
+  
+  desc 'Checks the index cache files'
+  task :check => :application do
+    Indexes.check_caches
+    puts "All caches look ok."
+  end
+  
+  
+  desc "Removes the index cache files."
+  task :clear => :application do
+    Indexes.clear_caches
+    puts "All index cache files removed."
+  end
+  
+  
+  desc 'Backup the index cache files'
+  task :backup => :application do
+    Indexes.backup_caches
+    puts "Index cache files moved to the backup directory"
+  end
+  
+  desc 'Restore the index cache files'
+  task :restore => :application do
+    Indexes.restore_caches
+    puts "Index cache files restored from the backup directory"
+  end
+  
+end

data/lib/tasks/framework.rake

@@ -0,0 +1,4 @@
+desc "Loads the framework."
+task :framework do
+  require File.expand_path(File.join(File.dirname(__FILE__), '..', 'picky'))
+end

data/lib/tasks/index.rake

@@ -0,0 +1,29 @@
+# Global.
+#
+namespace :index do
+  
+  desc "Generates the temp tables: Copies the data from the public entries."
+  task :generate_temp_tables => :application do
+    Indexes.take_snapshot
+  end
+  
+  desc "rake index:tables:update"
+  task :prepare => :application do
+    Indexes.take_snapshot
+    Indexes.configuration.index
+  end
+  
+  desc "E.g. Generates a specific index table. Note: temp tables need to be there. Will generate just the index table."
+  task :only, [:type, :field] => :application do |_, options|
+    type, field = options.type, options.field
+    Indexes.generate_index_only type.to_sym, field.to_sym
+  end
+  
+  desc "E.g. Generates a specific index cache file. Note: temp tables need to be there. Will generate index table and cache."
+  task :specific, [:type, :field] => :application do |_, options|
+    type, field = options.type, options.field
+    Indexes.generate_index_only type.to_sym, field.to_sym
+    Indexes.generate_cache_only type.to_sym, field.to_sym
+  end
+  
+end

data/lib/tasks/server.rake

@@ -0,0 +1,48 @@
+# TODO This file needs some love.
+#
+namespace :server do
+  def chdir_to_root
+    Dir.chdir SEARCH_ROOT
+  end
+  def current_pid
+    pid = `cat #{File.join(SEARCH_ROOT, 'tmp/pids/unicorn.pid')}`
+    pid.blank? ? nil : pid.chomp
+  end
+  desc "Start the unicorns. Weheee!"
+  task :start => :application do
+    chdir_to_root
+    # Rake::Task[:"solr:start"].invoke # TODO Move to better place.
+    config = {}
+    config['production'] = {
+      :port      => 6000,
+      :daemonize => true
+    }
+    config['development'] = {
+      :port      => 4000,
+      :daemonize => false
+    }
+    # TODO Move port!
+    port = SEARCH_ENVIRONMENT == 'production' ? 6000 : 4000
+    `export SEARCH_ENV=#{SEARCH_ENVIRONMENT}; unicorn -p #{config[SEARCH_ENVIRONMENT][:port]} -c #{File.join(SEARCH_ROOT, 'app/unicorn.ru')} #{config[SEARCH_ENVIRONMENT][:daemonize] ? '-D' : ''} #{File.join(SEARCH_ROOT, 'app/application.ru')}`
+  end
+  desc "Restart the unicorns!"
+  task :restart do
+    Rake::Task[:"server:stop"].invoke
+    sleep 15
+    Rake::Task[:"server:start"].invoke
+  end
+  desc "Stop the unicorns. Blam!"
+  task :stop => :application do
+    chdir_to_root
+    `kill -QUIT #{current_pid}` if current_pid
+    # Rake::Task[:"solr:stop"].invoke # TODO Move to better place.
+  end
+
+  # TODO
+  #
+  desc 'send the USR1 signal to the thin server'
+  task :usr1 => :ruby_version do
+    puts "Sending USR1 signal to the thin server."
+    `pidof thin#{RUBY_VERSION_APPENDIX}`.split.each { |pid| Process.kill('USR1', pid.to_i) }
+  end
+end

data/lib/tasks/shortcuts.rake

@@ -0,0 +1,13 @@
+desc "Shortcut for indexing and caching."
+task :index => :application do
+  Indexes.index
+end
+
+desc "shortcut for server:start"
+task :start do
+  Rake::Task[:'server:start'].invoke
+end
+desc "shortcut for server:stop"
+task :stop do
+  Rake::Task[:'server:stop'].invoke
+end