picky 0.0.0 → 0.0.2

data/lib/picky/configuration/field.rb

@@ -0,0 +1,68 @@
+module Configuration
+
+  class Field
+    attr_reader :name, :indexed_name, :virtual
+    attr_accessor :type # convenience
+    def initialize name, options = {}
+      @name            = name
+      
+      # TODO Dup the options?
+      # TODO add source as option
+      
+      @indexer_class   = options.delete(:indexer)   || Indexers::Default
+      @tokenizer_class = options.delete(:tokenizer) || Tokenizers::Index # Default
+      
+      @indexed_name   = options.delete(:indexed_field) || name # TODO Rename to indexed_as?
+      @virtual        = options.delete(:virtual)       || false
+      
+      Query::Qualifiers.add(name, options[:qualifiers]) if options[:qualifiers]
+      
+      # @remove          = options[:remove]        || false
+      # @filter          = options[:filter]        || true
+      
+      @options = options
+    end
+    def source
+      @source || type.source
+    end
+    def generate
+      Index::Category.new self.name, type, @options
+    end
+    # TODO Duplicate code in bundle. Move to application.
+    #
+    # TODO Move to type, and use in bundle from there.
+    #
+    def search_index_root
+      File.join SEARCH_ROOT, 'index'
+    end
+    # TODO Move to config. Duplicate Code in field.rb.
+    #
+    def cache_directory
+      File.join search_index_root, SEARCH_ENVIRONMENT, type.name.to_s
+    end
+    def search_index_file_name
+      File.join cache_directory, "#{type.name}_#{name}_index.txt"
+    end
+    def index
+      prepare_cache_directory
+      indexer.index
+    end
+    def prepare_cache_directory
+      FileUtils.mkdir_p cache_directory
+    end
+    def cache
+      prepare_cache_directory
+      generate.generate_caches
+    end
+    def indexer
+      @indexer || @indexer = @indexer_class.new(type, self)
+    end
+    def tokenizer
+      @tokenizer || @tokenizer = @tokenizer_class.new # TODO Make instances.
+    end
+    def virtual?
+      !!virtual
+    end
+  end
+
+end

data/lib/picky/configuration/indexes.rb

@@ -0,0 +1,60 @@
+module Configuration
+
+  class Indexes
+    
+    attr_reader :types
+    
+    def initialize *types
+      @types = types
+    end
+    
+    #
+    #
+    def default_index
+      Tokenizers::Index
+    end
+    
+    # Delegates
+    #
+    delegate :illegal_characters, :contract_expressions, :stopwords, :split_text_on, :normalize_words, :illegal_characters_after, :to => :default_index
+    
+    #
+    #
+    def type name, *fields
+      types << Type.new(name, *fields)
+    end
+    def field name, options = {}
+      Field.new name, options
+    end
+    
+    #
+    #
+    def take_snapshot *type_names
+      only_if_included_in type_names do |type|
+        type.take_snapshot
+      end
+    end
+    def index *type_names
+      only_if_included_in type_names do |type|
+        type.index
+      end
+    end
+    def index_solr *type_names
+      only_if_included_in type_names do |type|
+        type.index_solr
+      end
+    end
+    
+    #
+    #
+    def only_if_included_in type_names = []
+      type_names = types.map(&:name) if type_names.empty?
+      types.each do |type|
+        next unless type_names.include?(type.name)
+        yield type
+      end
+    end
+    
+  end
+
+end

data/lib/picky/configuration/queries.rb

@@ -0,0 +1,32 @@
+module Configuration
+  
+  class Queries
+    
+    attr_reader :routing
+    
+    #
+    #
+    def initialize routing
+      @routing = routing
+    end
+    
+    #
+    #
+    def default_index
+      Tokenizers::Query
+    end
+    
+    # Routes.
+    #
+    delegate :defaults, :route, :live, :full, :root, :default, :to => :routing
+    
+    # Delegates.
+    #
+    def maximum_tokens amount
+      Query::Tokens.maximum = amount
+    end
+    delegate :illegal_characters, :contract_expressions, :stopwords, :split_text_on, :normalize_words, :illegal_characters_after, :to => :default_index
+    
+  end
+  
+end

data/lib/picky/configuration/type.rb

@@ -0,0 +1,52 @@
+module Configuration
+  class Type
+    attr_reader :name,
+                :source,
+                :fields,
+                :after_indexing,
+                :result_type,
+                :ignore_unassigned_tokens,
+                :solr
+    def initialize name, source, *fields, options
+      if Configuration::Field === options
+        fields << options
+        options = {}
+      end
+
+      @name                     = name
+      @source                   = source
+                                  # dup, if field is reused. TODO Rewrite.
+      @fields                   = fields.map { |field| field = field.dup; field.type = self; field }
+
+      @after_indexing           = options[:after_indexing]
+      @result_type              = options[:result_type] || name
+      @ignore_unassigned_tokens = options[:ignore_unassigned_tokens] || false       # TODO Move to query?
+      @solr                     = options[:solr] || nil
+    end
+    def generate
+      categories = fields.map { |field| field.generate }
+      Index::Type.new name, result_type, ignore_unassigned_tokens, *categories
+    end
+    def table_name
+      self # FIXME UGH, Remove anyway
+    end
+    def take_snapshot
+      source.take_snapshot self
+    end
+    def index
+      fields.each do |field|
+        field.index
+      end
+    end
+    def solr_fields
+      solr ? fields.select { |field| !field.virtual? } : []
+    end
+    # TODO Delegate to Solr handler.
+    #
+    def index_solr
+      return unless solr
+      @indexer = Indexers::Solr.new self
+      @indexer.index
+    end
+  end
+end

data/lib/picky/cores.rb

@@ -0,0 +1,101 @@
+Infinity = 1.0/0
+
+# Handles processing over multiple cores.
+#
+class Cores
+  
+  # Pass it an ary or generator.
+  #
+  #   generator = (1..10).each
+  #   forked generator, :max => 5 do |element|
+  #     
+  #   end
+  #
+  # Options include:
+  #  * max: Maximum # of processors to use. Default is all it can get.
+  #
+  def self.forked ary_or_generator, options = {}
+    ary_or_generator = ary_or_generator.sort_by { rand } if options[:randomly]
+    generator        = ary_or_generator.each
+    
+    # Get the maximum number of processors.
+    #
+    max                  = max_processors options
+    currently_processing = 0
+    
+    # 
+    #
+    while generator
+      
+      # Ramp it up to num processors.
+      #
+      while currently_processing < max
+        
+        currently_processing = currently_processing + 1
+        
+        element = nil
+        begin
+          element = generator.next
+        rescue StopIteration => si
+          break
+        end
+        break unless element
+        
+        Process.fork do
+          yield element
+        end
+        
+      end
+      
+      begin
+        Process.wait 0 # Block and wait for any child to finish.
+      rescue Errno::ECHILD => e
+        break
+      ensure
+        currently_processing = currently_processing - 1
+      end
+    end
+  end
+  
+  # Return the number of maximum usable processors.
+  #
+  def self.max_processors options = {}
+    options[:amount] || [number_of_cores, (options[:max] || Infinity)].min
+  end
+  
+  # Gets the number of cores depending on OS.
+  #
+  def self.number_of_cores
+    extract_cores_for actual_platform
+  end
+  # Extracts the platform os from the platform.
+  #
+  # Note: Could also use 'rbconfig'.
+  #
+  def self.actual_platform
+    matched = platform.match(/-\b([a-z]*)/)
+    matched && matched[1]
+  end
+  # Returns a mapping
+  #   os_name => lambda_which_returns_a_number_of_cores
+  #
+  @@number_of_cores = {
+    'darwin' => lambda { `system_profiler SPHardwareDataType | grep 'Total Number Of Cores'`.gsub(/[^\d]/, '') },
+    'linux'  => lambda { `grep -ci ^processor /proc/cpuinfo` }
+  }
+  def self.os_to_core_mapping
+    @@number_of_cores
+  end
+  # Extracts the number of cores for the given os name.
+  #
+  # Note: Default is 1.
+  #
+  def self.extract_cores_for os
+    code_to_execute = os_to_core_mapping[os]
+    code_to_execute && code_to_execute.call.to_i || 1
+  end
+  def self.platform
+    RUBY_PLATFORM
+  end
+  
+end

data/lib/picky/db/configuration.rb

@@ -0,0 +1,23 @@
+class DB < ActiveRecord::Base
+  
+  self.abstract_class = true
+  
+  #
+  #
+  def self.configure options = {}
+    @connection_options = if filename = options[:file]
+      File.open(File.join(SEARCH_ROOT, filename)) { |f| YAML::load(f) }
+    else
+      options
+    end
+    self
+  end
+  
+  #
+  #
+  def self.connect
+    return if SEARCH_ENVIRONMENT.to_s == 'test'
+    establish_connection @connection_options
+  end
+  
+end

data/lib/picky/ext/ruby19/extconf.rb

@@ -0,0 +1,7 @@
+puts 'Compiling with Ruby 1.9'
+require 'mkmf'
+
+abort 'need ruby.h' unless have_header("ruby.h")
+
+dir_config('performant')
+create_makefile('performant')

data/lib/picky/ext/ruby19/performant.c

@@ -0,0 +1,339 @@
+// Note: This is the Ruby 1.9 version.
+//
+#include "ruby.h"
+
+// Copying internal ruby methods.
+//
+static inline VALUE rb_ary_elt(ary, offset)
+    VALUE ary;
+    long offset;
+{
+    if (RARRAY_LEN(ary) == 0) return Qnil;
+    if (offset < 0 || RARRAY_LEN(ary) <= offset) {
+        return Qnil;
+    }
+    return RARRAY_PTR(ary)[offset];
+}
+VALUE rb_ary_make_hash(VALUE, VALUE);
+static VALUE ary_make_hash(ary1, ary2)
+    VALUE ary1, ary2;
+{
+    VALUE hash = rb_hash_new();
+    long i;
+
+    for (i=0; i<RARRAY_LEN(ary1); i++) {
+        rb_hash_aset(hash, RARRAY_PTR(ary1)[i], Qtrue);
+    }
+    if (ary2) {
+        for (i=0; i<RARRAY_LEN(ary2); i++) {
+            rb_hash_aset(hash, RARRAY_PTR(ary2)[i], Qtrue);
+        }
+    }
+    return hash;
+}
+
+// Comparison functions.
+//
+inline int intvaluecmp(VALUE a, VALUE b) {
+  return FIX2INT(a) - FIX2INT(b);
+}
+inline int intcmp(const int * a, const int * b) {
+  return (*a - *b);
+}
+inline long longcmp(const void * a, const void * b) {
+  return (*(long*) a - *(long*) b);
+}
+
+// This version just calls the & consecutively for all arrays.
+//
+inline VALUE memory_efficient_intersect(VALUE self, VALUE length_sorted_array_of_arrays) {
+  // counters
+  long i, j;
+
+  // structs
+  struct RArray *rb_array_of_arrays;
+  struct RArray *smallest_array;
+  struct RArray *current_array;
+  VALUE hash;
+
+  // temps
+  VALUE v, vv;
+
+  // conversions
+  rb_array_of_arrays = RARRAY(length_sorted_array_of_arrays);
+  smallest_array     = RARRAY(rb_ary_dup(RARRAY_PTR(rb_array_of_arrays)[0]));
+
+  // iterate through all arrays
+  for (i = 1; i < RARRAY_LEN(rb_array_of_arrays); i++) {
+    // Break if the smallest array is empty
+    if (RARRAY_LEN(smallest_array) == 0) {
+      break;
+    }
+
+    // make a hash from the currently smallest version
+    hash = ary_make_hash(smallest_array, 0);
+    // clear for use as temp array
+    rb_ary_clear(smallest_array);
+
+    current_array = RARRAY_PTR(rb_array_of_arrays)[i];
+    // iterate through all array elements
+    for (j = 0; j < RARRAY_LEN(current_array); j++) {
+      v = vv = rb_ary_elt(current_array, j);
+      if (st_delete(RHASH_TBL(hash), (unsigned long*)&vv, 0)) {
+        rb_ary_push(smallest_array, v);
+      }
+    }
+  }
+
+  return smallest_array;
+}
+
+// Brute force algorithm to find the intersection of an array of length sorted, unsorted arrays.
+// This algorithm can be faster than others for small arrays.
+//
+// inline VALUE brute_force_intersect(VALUE self, VALUE length_sorted_array_of_arrays) {
+//   // counters
+//   long i, j, k;
+//
+//   // structs
+//   struct RArray *rb_array_of_arrays;
+//   struct RArray *candidate_answer_set;
+//   struct RArray *current_set;
+//
+//   // conversions
+//   rb_array_of_arrays = RARRAY(length_sorted_array_of_arrays);
+//
+//   // temps
+//   VALUE e;
+//   unsigned char found;
+//
+//   // Let the smallest set s[0] be the candidate answer set
+//   // Note: Need a duplicate
+//   candidate_answer_set = RARRAY(rb_ary_dup(rb_array_of_arrays->ptr[0]));
+//
+//   // For each entry in candidate anser set
+//   // Get current value
+//   for(i = 0; i < candidate_answer_set->len; i++) {
+//     e = candidate_answer_set->ptr[i];
+//
+//     // Find the current value in other arrays
+//     // if not found, break
+//     for(j = 1; j < rb_array_of_arrays->len; j++) {
+//       current_set = RARRAY(rb_array_of_arrays->ptr[j]);
+//       found = 0;
+//
+//       // Find with a linear search
+//       for(k = 0; k < current_set->len; k++) {
+//         if (e == current_set->ptr[k]) {
+//           found = 1;
+//           break;
+//         }
+//       }
+//
+//       // break if not found
+//       if (!found) {
+//         break;
+//       }
+//     }
+//
+//     // remove from candidate answer set if not found
+//     if (!found) {
+//       candidate_answer_set->ptr[i] = Qnil;
+//     }
+//   }
+//
+//   // compact the candidate answer set
+//   // rb_ary_compact_bang(candidate_answer_set);
+//   rb_funcall(candidate_answer_set, rb_intern("compact!"), 0);
+//
+//   return candidate_answer_set;
+// }
+
+// inline VALUE intersect_unique(VALUE self, VALUE length_sorted_array_of_arrays) {
+//   // VALUE length_sorted_array_of_arrays = (_length_sorted_array_of_arrays);
+//
+//   // structs
+//   struct RArray *result;
+//   struct RArray *rb_array_of_arrays;
+//
+//   // conversions
+//   rb_array_of_arrays = RARRAY(length_sorted_array_of_arrays);
+//
+//   // TODO
+//
+//   return result;
+// }
+
+// Generates the intersection of multiple
+//
+// inline VALUE sorting_intersect_multiple(VALUE self, VALUE length_sorted_array_of_arrays) {
+//   // TODO
+// }
+
+// Generates the intersection of multiple length sorted, sorted arrays
+//
+// inline VALUE intersect_multiple_sorted(VALUE self, VALUE _length_sorted_array_of_arrays) {
+//   VALUE length_sorted_array_of_arrays = (_length_sorted_array_of_arrays);
+//
+//   // counters
+//   long i, j;
+//   long current_set_position, current_answer_set_position;
+//
+//   // structs
+//   struct RArray *rb_array_of_arrays;
+//   struct RArray *candidate_answer_set;
+//   struct RArray *current_set;
+//
+//   // temps
+//   long e;
+//
+//   // conversions
+//   rb_array_of_arrays = RARRAY(length_sorted_array_of_arrays);
+//
+//   // Let the smallest set s[0] be the candidate answer set
+//   // Note: Need a duplicate
+//   candidate_answer_set = RARRAY(rb_ary_dup(rb_array_of_arrays->ptr[0]));
+//
+//   // For each set s[i], i = 1 .. k do
+//   for(i = 1; i < rb_array_of_arrays->len; i++) {
+//     current_set = RARRAY(rb_array_of_arrays->ptr[i]);
+//     current_set_position = 0;
+//
+//     // for each element e in the candidate answer set
+//     for(j = 0; j < candidate_answer_set->len; j++) {
+//       e = candidate_answer_set->ptr[j];
+//
+//       // search for e in the range l[i] to size(s[i])
+//       // and update l[i] to the last position probed in the previous step
+//       // if e was not found then
+//       if (bsearch(
+//         &e,
+//         &current_set->ptr[current_set_position],
+//         (current_set->len - current_set_position),
+//         sizeof(VALUE), //sizeof(current_set->ptr[0]),
+//         intcmp //longcmp
+//       ) == NULL) {
+//
+//         // remove e from the candidate answer set
+//         // and advance e to the next element in the answer set
+//         // rb_ary_delete_at(candidate_answer_set, j);
+//         candidate_answer_set->ptr[j] = Qnil;
+//       }
+//       current_set_position = j - 1;
+//     }
+//
+//     // compact the candidate answer set
+//     // rb_ary_compact_bang(candidate_answer_set);
+//     rb_funcall(candidate_answer_set, rb_intern("compact!"), 0);
+//   }
+//
+//   return candidate_answer_set;
+// }
+
+// Trying to make a custom version of Matz' ary &
+//
+// Differences:
+//  * Multiple arrays
+//  * No to_ary
+//  * Smallest array is used to make hash
+// Note: Assumes that whatever is given in as array of arrays is sorted by array sizes.
+//
+// static VALUE rb_ary_and(ary1, ary2) VALUE ary1, ary2; {
+// static VALUE intersect_multiple_with_hash(VALUE self, VALUE _length_sorted_array_of_arrays) {
+//   //    VALUE hash, ary3, v, vv;
+//   //    long i;
+//   //
+//   //    ary2 = to_ary(ary2);
+//   //    ary3 = rb_ary_new2(RARRAY(ary1)->len < RARRAY(ary2)->len ?
+//   //            RARRAY(ary1)->len : RARRAY(ary2)->len);
+//   //    hash = ary_make_hash(ary2, 0);
+//   //
+//   //    for (i=0; i<RARRAY(ary1)->len; i++) {
+//   //        v = vv = rb_ary_elt(ary1, i);
+//   //        if (st_delete(RHASH(hash)->tbl, (st_data_t*)&vv, 0)) {
+//   //            rb_ary_push(ary3, v);
+//   //        }
+//   //    }
+//   //
+//   //    return ary3;
+//   VALUE length_sorted_array_of_arrays = (_length_sorted_array_of_arrays);
+//
+//   // structs
+//   struct RArray *candidate_answer_set;
+//   struct RArray *current_set;
+//
+//   // temps
+//   VALUE hash, v, vv;
+//   long i, j, k;
+//
+//   // Get smallest array size
+//   candidate_answer_set = rb_ary_new2((RARRAY(rb_array_of_arrays->ptr[0])->len);
+//
+//   hash = ary_make_hash(RARRAY(rb_array_of_arrays->ptr[0]), 0);
+//
+//   // For each entry in candidate answer set
+//   // Get current value
+//   for(i = 0; i < candidate_answer_set->len; i++) {
+//     // e = candidate_answer_set->ptr[i];
+//     v = vv = rb_ary_elt(candidate_answer_set, i);
+//
+//     // Find the current value in other arrays
+//     // if not found, break
+//     for(j = 1; j < rb_array_of_arrays->len; j++) {
+//       current_set = RARRAY(rb_array_of_arrays->ptr[j]);
+//       found = 0;
+//
+//       // Find with a linear search
+//       for(k = 0; k < current_set->len; k++) {
+//         // if (e == current_set->ptr[k]) {
+//         if (st_delete(RHASH(hash)->tbl, (unsigned long*)&vv, 0))
+//           found = 1;
+//           break;
+//         }
+//       }
+//
+//       // break if not found
+//       if (!found) {
+//         break;
+//       }
+//     }
+//
+//     // remove from candidate answer set if not found
+//     if (!found) {
+//       rb_ary_push(result, v);
+//       // candidate_answer_set->ptr[i] = Qnil;
+//     }
+//   }
+//
+//   // compact the candidate answer set
+//   // rb_ary_compact_bang(candidate_answer_set);
+//   rb_funcall(candidate_answer_set, rb_intern("compact!"), 0);
+//
+//   return candidate_answer_set;
+// }
+
+// VALUE rb_ary_clear_bang(ary) VALUE ary; {
+//     rb_ary_modify(ary);
+//     ARY_SET_LEN(ary, 0);
+//     // capa stays the same
+//     // if (ARY_DEFAULT_SIZE * 2 < RARRAY(ary)->aux.capa) {
+//     //    REALLOC_N(RARRAY(ary)->ptr, VALUE, ARY_DEFAULT_SIZE * 2);
+//     //    RARRAY(ary)->aux.capa = ARY_DEFAULT_SIZE * 2;
+//     // }
+//     return ary;
+// }
+
+VALUE p_mPerformant, p_cArray;
+
+void Init_performant() {
+  p_mPerformant = rb_define_module("Performant");
+  p_cArray = rb_define_class_under(p_mPerformant, "Array", rb_cObject);
+  // p_cArray = rb_define_module_under(p_mPerformant, "Array");
+
+  // rb_define_method(rb_cArray, "clear!", rb_ary_clear_bang, 0);
+
+  rb_define_singleton_method(p_cArray, "memory_efficient_intersect", memory_efficient_intersect, 1);
+  // rb_define_singleton_method(p_cArray, "brute_force_intersect", brute_force_intersect, 1);
+  // rb_define_singleton_method(p_cArray, "intersect_multiple_sorted", intersect_multiple_sorted, 1);
+  // rb_define_singleton_method(p_cArray, "intersect_multiple_with_hash", intersect_multiple_sorted_with_hash, 1);
+}