picky 0.0.0 → 0.0.2

data/lib/picky/index/category.rb

@@ -0,0 +1,109 @@
+module Index
+  
+  # An index category holds a full and a partial index for a given field.
+  #
+  # For example an index category for names holds a full and
+  # a partial index bundle for names.
+  #
+  class Category
+    
+    attr_reader :name, :type, :full, :partial
+    
+    #
+    #
+    def initialize name, type, options = {}
+      @name = name
+      @type = type
+      
+      partial    = options[:partial]    || Cacher::Partial::Default
+      weights    = options[:weights]    || Cacher::Weights::Default
+      similarity = options[:similarity] || Cacher::Similarity::Default
+      
+      @full    = options[:full_bundle]    || Bundle.new(:full,    self, type, Cacher::Partial::None.new, weights, similarity)
+      @partial = options[:partial_bundle] || Bundle.new(:partial, self, type, partial, weights, Cacher::Similarity::None.new)
+      
+      @full    = options[:full_lambda].call(@full, @partial)    if options[:full_lambda]
+      @partial = options[:partial_lambda].call(@full, @partial) if options[:partial_lambda]
+    end
+    
+    # Loads the index from cache.
+    #
+    def load_from_cache
+      full.load
+      partial.load
+    end
+    
+    def identifier
+      "#{type.name}:#{name}"
+    end
+    
+    # Generates all caches for this category.
+    #
+    def generate_caches
+      exclaim "#{Time.now}: Loading data from db for #{identifier}."
+      generate_caches_from_db
+      exclaim "#{Time.now}: Generating partial for #{identifier}."
+      generate_partial
+      exclaim "#{Time.now}: Generating caches from memory for #{identifier}."
+      generate_caches_from_memory
+      exclaim "#{Time.now}: Dumping all caches for #{identifier}."
+      dump_caches
+    end
+    def generate_caches_from_db
+      full.generate_caches_from_db
+    end
+    def generate_partial
+      partial.generate_partial_from full.index
+    end
+    def generate_caches_from_memory
+      partial.generate_caches_from_memory
+    end
+    def dump_caches
+      full.dump
+      partial.dump
+    end
+    def exclaim text
+      puts text
+    end
+
+    # Used for testing.
+    #
+    def generate_indexes_from_full_index
+      generate_derived_full
+      generate_partial
+      generate_derived_partial
+    end
+    def generate_derived_full
+      full.generate_derived
+    end
+    def generate_derived_partial
+      partial.generate_derived
+    end
+
+    # Gets the weight for this token's text.
+    #
+    def weight token
+      bundle_for(token).weight token.text
+    end
+
+    # Gets the ids for this token's text.
+    #
+    def ids token
+      bundle_for(token).ids token.text
+    end
+
+    # Returns the right index bundle for this token.
+    #
+    def bundle_for token
+      token.partial? ? partial : full
+    end
+
+    #
+    #
+    def combination_for token
+      weight(token) && ::Query::Combination.new(token, self)
+    end
+
+  end
+
+end

data/lib/picky/index/combined.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+#
+module Index
+
+  # 
+  #
+  class Combined < Bundle
+    
+    delegate :similar,
+             :identifier,
+             :name,
+             :to => :@full
+    delegate :type,
+             :category,
+             :weight,
+             :generate_partial_from,
+             :generate_caches_from_memory,
+             :generate_derived,
+             :dump,
+             :load,
+             :to => :@partial
+    
+    def initialize full, partial
+      @full    = full
+      @partial = partial
+    end
+    
+    def ids text
+      @full.ids(text) + @partial.ids(text)
+    end
+    
+    def weight text
+      [@full.weight(text) || 0, @partial.weight(text) || 0].max
+    end
+    
+  end
+  
+end

data/lib/picky/index/type.rb

@@ -0,0 +1,30 @@
+module Index
+  
+  # This class is for multiple types.
+  #
+  # For example, you could have types books, isbn.
+  #
+  class Type
+    
+    attr_reader :name, :result_type, :categories, :combinator
+    
+    each_delegate :generate_caches, :load_from_cache, :to => :categories
+    
+    # TODO Use config
+    #
+    def initialize name, result_type, ignore_unassigned_tokens, *categories
+      @name        = name
+      @result_type = result_type # TODO Move.
+      @categories  = categories # for each_delegate
+      @combinator  = Query::Combinator.new @categories, :ignore_unassigned_tokens => ignore_unassigned_tokens # TODO pass this in?
+    end
+    
+    #
+    #
+    def possible_combinations token
+      @combinator.possible_combinations_for token
+    end
+    
+  end
+  
+end

data/lib/picky/indexers/base.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+module Indexers
+  # Indexer.
+  #
+  # 1. Gets data from the original table and copies it into a "snapshot table".
+  # 3. Processes the data. I.e. takes the snapshot table data words and tokenizes etc. them. Writes the result into a txt file.
+  #
+  class Base
+    
+    def initialize type, field
+      @type       = type
+      @field      = field
+    end
+    
+    # Convenience method for getting the right Tokenizer.
+    #
+    def tokenizer
+      @field.tokenizer
+    end
+    # Convenience methods for user subclasses.
+    #
+    def search_index_file_name
+      @field.search_index_file_name
+    end
+    
+    # Executes the specific strategy.
+    #
+    def index
+      process
+    end
+    
+    # Get the source where the data is taken from.
+    #
+    def source
+      @field.source || raise_no_source
+    end
+    def raise_no_source
+      raise NoSourceSpecifiedException.new "No source given for #{@type.name}:#{@field.name}."
+    end
+    
+    # Selects the original id (indexed id) and a column to process. The column data is called "token".
+    #
+    def process
+      comma   = ?,
+      newline = ?\n
+      
+      File.open(search_index_file_name, 'w:binary') do |file|
+        chunked do |indexed_id, text|
+          tokenizer.tokenize(text).each do |token_text|
+            file.write indexed_id
+            file.write comma
+            file.write token_text
+            file.write newline
+          end
+        end
+      end
+    end
+    # Split original data into chunks.
+    #
+    def chunked
+      (0..source.count(@type)).step(chunksize) do |offset|
+        indexing_message offset
+        data = source.harvest @type, @field, offset, chunksize
+        data.each do |indexed_id, text|
+          next unless text
+          text.force_encoding 'utf-8' # TODO Still needed?
+          yield indexed_id, text
+        end
+      end
+    end
+    
+    def indexing_message offset
+      puts "#{Time.now}: Indexing #{@type.name}:#{@field.name}:#{@field.indexed_name} beginning at #{offset}."
+    end
+    
+  end
+end

data/lib/picky/indexers/default.rb

@@ -0,0 +1,3 @@
+module Indexers
+  Default = Field
+end

data/lib/picky/indexers/field.rb

@@ -0,0 +1,13 @@
+module Indexers
+  # Base indexer for fields.
+  #
+  class Field < Base
+
+    # Override in subclasses.
+    #
+    def chunksize
+      25_000
+    end
+    
+  end
+end

data/lib/picky/indexers/no_source_specified_error.rb

@@ -0,0 +1,5 @@
+module Indexers
+  
+  class NoSourceSpecifiedException < StandardError; end
+  
+end

data/lib/picky/indexers/solr.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+#
+require 'rsolr'
+module Indexers
+  # This is an indexer in its own right.
+  #
+  # TODO Perhaps merge with the existing indexer.
+  #
+  class Solr
+
+    attr_reader :type, :fields, :solr
+
+    # Takes a Configuration::Type.
+    #
+    def initialize type
+      @type   = type
+      @fields = type.solr_fields.map(&:name).map(&:to_sym)
+      @solr   = RSolr.connect
+    end
+
+    # TODO Rewrite such that it works in batches.
+    #
+    def index
+      puts "#{Time.now}: Indexing solr for #{type.name}:#{fields.join(', ')}"
+      statement = "SELECT indexed_id, #{fields.join(',')} FROM #{type.snapshot_table_name}"
+      
+      # TODO Rewrite.
+      #
+      DB.connect
+      results   = DB.connection.execute statement
+
+      return unless results # TODO check
+
+      type_name = @type.name.to_s
+
+      solr.delete_by_query "type:#{type_name}"
+      solr.commit
+
+      documents = []
+
+      results.each do |indexed_id, *values|
+        values.each &:downcase!
+        documents << hashed(values).merge(:id => indexed_id, :type => type_name)
+      end
+
+      solr.add documents
+      solr.commit
+      solr.optimize
+    end
+
+    def hashed values
+      result = {}
+      fields.zip(values).each do |field, value|
+        result[field] = value
+      end
+      result
+    end
+
+  end
+end

data/lib/picky/indexes.rb

@@ -0,0 +1,180 @@
+module Indexes
+
+  mattr_accessor :configuration, :types, :type_mapping
+
+  # Takes a snapshot of the originating table.
+  #
+  def self.take_snapshot *args
+    configuration.take_snapshot *args
+  end
+  # Runs the indexers in parallel (index + cache).
+  #
+  def self.index
+    Indexes.take_snapshot
+
+    # Run in parallel.
+    #
+    puts "Indexing using #{Cores.max_processors} processors."
+    Cores.forked self.fields, :randomly => true do |field|
+      # Reestablish DB connection.
+      DB.connect # TODO Rewrite!
+      field.index
+      field.cache
+    end
+  end
+  def self.index_solr
+    configuration.index_solr
+  end
+  
+  # Returns an array of fields.
+  #
+  # TODO Rewrite.
+  #
+  def self.fields
+    result = []
+    configuration.types.each do |type|
+      type.fields.inject(result) do |total, field|
+        total << field
+      end
+    end
+    result
+  end
+  
+  #
+  #
+  # TODO Rewrite.
+  #
+  def self.find type_name, field_name
+    type_name  = type_name.to_sym
+    field_name = field_name.to_sym
+    configuration.types.each do |type|
+      next unless type.name == type_name
+      type.fields.each do |field|
+        next unless field.name == field_name
+        return field
+      end
+    end
+  end
+  
+  # Backup / Restore.
+  #
+  def self.backup_caches
+    each_category do |category|
+      category.full.backup
+      category.partial.backup
+    end
+  end
+  def self.restore_caches
+    each_category do |category|
+      category.full.restore
+      category.partial.restore
+    end
+  end
+  
+  # Use these if you need to index / cache a single index.
+  #
+  def self.generate_index_only type_name, field_name
+    Indexes.configuration.types.each do |type|
+      next unless type_name == type.name
+      type.fields.each do |field|
+        field.index if field_name == field.name
+      end
+    end
+  end
+  def self.generate_cache_only type_name, category_name
+    Indexes.each do |type|
+      next unless type_name == type.name
+      type.categories.each do |category|
+        category.generate_caches if category_name == category.name
+      end
+    end
+  end
+  
+  # Runs the index cache generation.
+  #
+  # TODO Needed?
+  #
+  def self.generate_caches
+    each &:generate_caches
+  end
+  # Loads all indexes from the caches.
+  #
+  def self.load_from_cache
+    each &:load_from_cache
+  end
+  def self.reload
+    load_from_cache
+  end
+  
+  # Checks if the caches are there.
+  #
+  def self.check_caches
+    each do |type|
+      type.categories.each do |category|
+        category.full.raise_unless_cache_exists
+        category.partial.raise_unless_cache_exists
+      end
+    end
+  end
+  
+  # Removes the cache files.
+  #
+  def self.clear_caches
+    each do |type|
+      type.categories.each do |category|
+        category.full.delete_all
+        category.partial.delete_all
+      end
+    end
+  end
+
+  # Creates the directory structure for the indexes.
+  #
+  # TODO Should be on type?
+  #
+  def self.create_directory_structure
+    each_bundle do |full, partial|
+      full.create_directory
+      partial.create_directory
+    end
+  end
+
+  # 
+  #
+  def self.clear
+    self.types = []
+  end
+  
+  
+  def self.each &block
+    types.each &block
+  end
+  def self.each_category &block
+    each do |type|
+      type.categories.each &block
+    end
+  end
+  def self.each_bundle
+    each_category do |category|
+      yield category.full, category.partial
+    end
+  end
+  
+  # Loads all index definitions.
+  #
+  def self.setup
+    self.types        ||= []
+    self.type_mapping ||= {}
+    configuration.types.each do |type|
+      add type.generate
+    end
+  end
+  def self.add type
+    self.type_mapping[type.name] = type
+    self.types << type
+  end
+  def self.[] type_name
+    self.type_mapping[type_name]
+  end
+  
+end

data/lib/picky/initializers/ext.rb

@@ -0,0 +1,6 @@
+# TODO What to do with this?
+#
+Dir.chdir File.join(File.dirname(__FILE__), '../ext/ruby19') do
+  %x{ ruby extconf.rb && make }
+end
+require File.expand_path(File.join(File.dirname(__FILE__), '../ext/ruby19/performant'))

data/lib/picky/initializers/mysql.rb

@@ -0,0 +1,22 @@
+# Wrapper for the mysql adapter method execute
+# to handle the 8 hours disconnect problem.
+# (http://www.mysql.fr/search/?q=autoreconnect)
+#
+ActiveRecord::ConnectionAdapters::MysqlAdapter.module_eval do
+  def execute_with_retry_once(sql, name = nil)
+    retried = false
+    begin
+      execute_without_retry_once(sql, name)
+    rescue ActiveRecord::StatementInvalid => statement_invalid_exception
+      # Our database connection has gone away, reconnect and retry this method
+      #
+      reconnect!
+      unless retried
+        retried = true
+        retry
+      end
+    end
+  end
+
+  alias_method_chain :execute, :retry_once
+end