active_blur 0.1.0

data/active_blur.gemspec

@@ -0,0 +1,25 @@
+# encoding: utf-8
+$:.push File.expand_path("../lib", __FILE__)
+require 'active_blur/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'active_blur'
+  s.version       = ActiveBlur::VERSION
+  s.platform      = Gem::Platform::RUBY
+  s.authors       = ["Andrew Avenoso"]
+  s.homepage      = "https://github.com/nearinfinity/active_blur"
+  s.summary       = "Search ActiveRecord models using Blur"
+  s.description   = "Active Blur is an Active Record extension library to enable creating indexes, updating indexes and searching indexes in Blur. The Active Record model objects that are configured to use this gem will automatically create tables in Blur, re-index upon updates in Blur, and allow searching (fielded and full text) in Blur."
+  s.license       = "Apache-2.0"
+
+  s.files         = `git ls-files`.split("\n").reject { |path| path =~ /^(Gemfile|.gitignore|Rakefile)/ }
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency('rails', '~> 3.0.0')
+  s.add_dependency('thrift_client', '~> 0.7.0')
+
+  s.add_development_dependency('rspec', '2.7.0')
+  s.add_development_dependency('rdoc', '3.12')
+end

data/lib/active_blur.rb

@@ -0,0 +1,2 @@
+require 'active_blur/active_blur'
+require 'active_blur/railtie' if defined? Rails

data/lib/active_blur/active_blur.rb

@@ -0,0 +1,256 @@
+require 'thrift_client'
+require 'active_blur/thrift/blur'
+require 'active_blur/search_results'
+
+module ActiveBlur
+  @searchable_classes = {}
+  @blur_client = nil
+  @tables = []
+  @create_mode=false
+  @global_options = {:full_text_index=>true,
+                     :default_analyzer=>:standard,
+                     :full_text_analyzer=>:standard
+  }
+
+  class <<self
+
+    attr_accessor :create_mode
+
+    def setup_connection(connection_str, options={}, thrift_options={})
+      @global_options = @global_options.merge(options)
+      thrift_defaults = {:retries=>5}
+      thrift_options = thrift_defaults.merge(thrift_options)
+      if connection_str.class == String
+        urls = connection_str.split(/,/)
+      else
+        urls = Array(connection_str)
+      end
+      unless thrift_options[:server_retry_period]
+        thrift_options[:server_retry_period] = urls.size < 2 ? -1 : urls.size
+      end
+      @blur_client = ThriftClient.new Blur::Blur::Client, urls, thrift_options
+      puts "ActiveBlur connection setup"
+      @tables = client.tableList
+      puts "Found tables #{@tables}"
+    end
+
+    def setup_class(clazz, options={})
+      defaults = {:table_name=>clazz.table_name, :column_family=>clazz.table_name.singularize, :shard_count=>3}
+      options = defaults.merge(@global_options).merge(options)
+      options[:exclude] = options[:exclude].collect{|col| col.to_s} if options[:exclude]
+      raise "options[:cluster] is required" unless options[:cluster]
+      puts "setup options for #{clazz} are #{options.inspect}"
+      unless create_mode
+        ensure_table_exists! options[:table_name]
+        ensure_table_enabled! options[:table_name]
+        ensure_schemas_match! clazz, options
+      end
+      @searchable_classes[clazz] = options
+    end
+
+    def searchable_classes
+      @searchable_classes.keys
+    end
+
+    def create_table(clazz)
+      options = options_for clazz
+      raise "#{clazz.name} has not been configured to be searchable" unless options
+      raise "table #{options[:table_name]} already exists" if @tables.include? options[:table_name]
+      db_columns = columns_for_class(clazz, options)
+      table_descriptor = Blur::TableDescriptor.new
+      table_descriptor.shardCount = options[:shard_count]
+      table_descriptor.tableUri = calculate_table_uri(options)
+      table_descriptor.cluster = options[:cluster]
+      table_descriptor.name = options[:table_name]
+      analyzer = Blur::AnalyzerDefinition.new
+      analyzer.defaultDefinition = new_column_definition(options[:default_analyzer], options[:full_text_index])
+      analyzer.fullTextAnalyzerClassName = analyzer(options[:full_text_analyzer])
+
+      col_fam_definition = Blur::ColumnFamilyDefinition.new
+      col_fam_definition.defaultDefinition = new_column_definition(options[:default_analyzer], options[:full_text_index])
+      col_fam_definition.columnDefinitions = {}
+      columns_for_class(clazz, options).each do |col|
+        col_fam_definition.columnDefinitions[col] = new_column_definition(options[:default_analyzer], options[:full_text_index])
+      end
+      analyzer.columnFamilyDefinitions={options[:column_family]=>col_fam_definition}
+      table_descriptor.analyzerDefinition = analyzer
+      puts "about to create table #{table_descriptor.inspect}"
+      client.createTable(table_descriptor)
+    end
+
+    def search(clazz, query, options={})
+      model_options = options_for clazz
+      blur_query = Blur::BlurQuery.new
+      #query.facets = ???
+      simple_query = Blur::SimpleQuery.new
+      simple_query.queryStr = query
+      simple_query.superQueryOn = false
+      simple_query.type = score_type(options[:score_type]) unless options[:score_type].nil?
+      simple_query.postSuperFilter = options[:post_super_filter] unless options[:post_super_filter].nil?
+      simple_query.preSuperFilter = options[:pre_super_filter] unless options[:pre_super_filter].nil?
+      blur_query.simpleQuery = simple_query
+      blur_query.allowStaleData = options[:allow_stale_data] unless options[:allow_stale_data].nil?
+      blur_query.useCacheIfPresent = options[:use_cache_if_present] unless options[:use_cache_if_present].nil?
+      blur_query.start = options[:start] unless options[:start].nil?
+      blur_query.fetch = options[:fetch] unless options[:fetch].nil?
+      blur_query.minimumNumberOfResults = options[:minimum_number_of_results] unless options[:minimum_number_of_results].nil?
+      blur_query.maxQueryTime = options[:max_query_time] unless options[:max_query_time].nil?
+      blur_query.uuid = options[:uuid] unless options[:uuid].nil?
+      blur_query.userContext = options[:user_context] unless options[:user_context].nil?
+      blur_query.cacheResult = options[:cache_result] unless options[:cache_result].nil?
+      blur_query.modifyFileCaches = options[:modify_file_caches] unless options[:modify_file_caches].nil?
+      selector = Blur::Selector.new(:recordOnly=>true, :columnFamiliesToFetch=>['none_col_fam'])
+      blur_query.selector = selector
+      results = client.query(model_options[:table_name], blur_query)
+      SearchResults.new(results, clazz)
+    end
+
+    def reindex(clazz)
+      all = clazz.all
+      all.each do |obj|
+        index(obj)
+      end
+    end
+
+    def index(object)
+      puts "indexing #{object.class} with id #{object.id}"
+      options = options_for object.class
+      mutation = Blur::RowMutation.new
+      mutation.table = options[:table_name]
+      mutation.rowId = "#{object.class.name}-#{object.id}"
+      mutation.rowMutationType = Blur::RowMutationType::REPLACE_ROW
+      record_mutation = Blur::RecordMutation.new
+      record_mutation.recordMutationType = Blur::RecordMutationType::REPLACE_ENTIRE_RECORD
+      record = Blur::Record.new
+      record.recordId = object.id.to_s
+      record.family = options[:column_family]
+      record.columns = []
+      columns_for_class(object.class, options).each do |col|
+        column = Blur::Column.new
+        column.name = col
+        val = object.send(col)
+        if val
+          column.value = val.to_s
+          record.columns << column
+        end
+      end
+      record_mutation.record = record
+      mutation.recordMutations = [record_mutation]
+      client.mutate(mutation)
+    end
+
+    def remove(object)
+      remove_by_id(object.class, object.id)
+    end
+
+    def remove_by_id(clazz, id)
+      puts "removing #{clazz} with id #{id}"
+      options = options_for clazz
+      mutation = Blur::RowMutation.new
+      mutation.table = options[:table_name]
+      mutation.rowId = "#{clazz.name}-#{id}"
+      mutation.rowMutationType = Blur::RowMutationType::DELETE_ROW
+      client.mutate(mutation)
+    end
+
+    private
+
+    def new_column_definition(analyzer, full_text)
+      Blur::ColumnDefinition.new(:analyzerClassName=>analyzer(analyzer), :fullTextIndex=>full_text)
+    end
+
+    def analyzer(sym_or_class)
+      if sym_or_class.is_a? Symbol
+        case sym_or_class
+          when :standard
+            "org.apache.lucene.analysis.standard.StandardAnalyzer"
+          when :keyword
+            "org.apache.lucene.analysis.KeywordAnalyzer"
+          when :simple
+            "org.apache.lucene.analysis.SimpleAnalyzer"
+          when :stop
+            "org.apache.lucene.analysis.StopAnalyzer"
+          when :whitespace
+            "org.apache.lucene.analysis.WhitespaceAnalyzer"
+          else
+            raise "invalid analzyer #{sym_or_class}"
+        end
+      elsif sym_or_class.is_a? String
+        sym_or_class
+      else
+        raise "invalid analzyer #{sym_or_class}"
+      end
+    end
+
+    def score_type(sym)
+      case sym
+        when :best
+          Blur::ScoreType::BEST
+        when :super
+          Blur::ScoreType::SUPER
+        when :aggregate
+          Blur::ScoreType::AGGREGATE
+        when :constant
+          Blur::ScoreType::CONSTANT
+        else
+          raise "invalid score type #{sym}"
+      end
+    end
+
+    def client
+      raise "Active blur not configured" unless @blur_client
+      @blur_client
+    end
+
+    def options_for(clazz)
+      @searchable_classes[clazz]
+    end
+
+    def ensure_table_enabled!(table_name)
+      raise "#{table_name} is not enabled" unless client.describe(table_name).isEnabled
+    end
+
+    def ensure_table_exists!(table_name)
+      raise "table #{table_name} doesn't exist, run rake blur:create_table" unless @tables.include? table_name
+    end
+
+    def ensure_schemas_match!(clazz, options)
+      definition = client.describe(options[:table_name]).analyzerDefinition
+      if definition.defaultDefinition.nil? || definition.defaultDefinition.analyzerClassName != analyzer(options[:default_analyzer]) || definition.defaultDefinition.fullTextIndex != options[:full_text_index]
+        raise "default analyzer does not match, run rake blur:create_table"
+      end
+      if definition.fullTextAnalyzerClassName != analyzer(options[:full_text_analyzer])
+        raise "full text analyzer does not match, run rake blur:create_table"
+      end
+      if definition.columnFamilyDefinitions.nil? || definition.columnFamilyDefinitions[options[:column_family].nil?]
+        raise "#{options[:column_family]} column family not found in #{options[:table_name]} table"
+      end
+
+      blur_columns = definition.columnFamilyDefinitions[options[:column_family]].columnDefinitions
+      db_columns = columns_for_class(clazz, options)
+      if blur_columns.size != db_columns.size
+        raise "blur columns do not match, expected #{db_columns}, but was #{blur_columns}, run rake blur:create_table"
+      end
+      db_columns.each do |col|
+        if blur_columns[col].nil?
+          raise "blur columns do not match, #{col} was not found in blur, run rake blur:create_table"
+        end
+      end
+    end
+
+    def columns_for_class(clazz, options)
+      ignore = options[:ignore]
+      clazz.column_names.reject {|col| col == 'id' || (ignore && ignore.include?(col))}
+    end
+
+    def calculate_table_uri(options)
+      return options[:table_uri] if options[:table_uri]
+      raise ":table_uri or :table_uri_base must be set" unless options[:table_uri_base]
+      uri = options[:table_uri_base]
+      uri += '/' unless uri.last == '/'
+      uri += options[:table_name] + '/'
+      uri
+    end
+
+  end
+end

data/lib/active_blur/config.rb

@@ -0,0 +1,25 @@
+module ActiveBlur
+  class Config
+    attr_accessor :connection_str, :thrift_options, :options
+
+    def initialize
+      self.thrift_options={}
+      self.options={}
+    end
+
+    def valid?
+      !invalid?
+    end
+
+    def invalid?
+      connection_str.nil?
+    end
+
+    def errors
+      return nil if valid?
+      errors = []
+      errors << "connection_str cannot be nil" if connection_str.nil?
+      errors.join("\n")
+    end
+  end
+end

data/lib/active_blur/railtie.rb

@@ -0,0 +1,25 @@
+require 'active_blur/searchable'
+require 'active_blur/config'
+module ActiveBlur
+  class Railtie < Rails::Railtie
+
+    config.active_blur = ActiveBlur::Config.new
+
+    initializer "active_blur.initialize" do |app|
+      abc = app.config.active_blur
+      raise ArgumentError.new(abc.errors) if abc.invalid?
+      ActiveBlur.setup_connection(abc.connection_str, abc.options, abc.thrift_options)
+    end
+
+    initializer "active_blur.initialize_active_record" do
+      ActiveSupport.on_load(:active_record) do
+        include ActiveBlur::Searchable
+      end
+    end
+
+    rake_tasks do
+      load "tasks/create_table.rake"
+      load "tasks/reindex.rake"
+    end
+  end
+end

data/lib/active_blur/search_results.rb

@@ -0,0 +1,53 @@
+module ActiveBlur
+  class ScoredResult
+    include ActiveModel::Serialization
+    attr_accessor :score, :result
+    def initialize(score, result)
+      @score=score
+      @result=result
+    end
+  end
+  
+  class SearchResults
+    include ActiveModel::Serialization
+    attr_accessor :total_results, :ids, :records, :clazz
+
+    def initialize(blur_results, clazz, should_fetch_records=false)
+      @total_results=blur_results.totalResults
+      @clazz = clazz
+      @ids=[]
+      #@records=[] if fetch_records
+      blur_results.results.each do |blur_result|
+        score = blur_result.score
+        fetch_result = blur_result.fetchResult
+        if fetch_result && fetch_result.exists && !fetch_result.deleted && fetch_result.recordResult && fetch_result.recordResult.record
+          record_id = fetch_result.recordResult.record.recordId
+          @ids << ScoredResult.new(score, record_id)
+          #@records << ScoredResult.new(score, clazz.find(record_id)) if fetch_records==true
+        end
+      end
+      fetch_records if should_fetch_records
+    end
+
+    def records
+      fetch_records unless @records
+      @records
+    end
+
+    private
+
+    def fetch_records()
+      @records=[]
+      ids.each do |scored_result|
+        begin
+          obj = clazz.find(scored_result.result)
+          obj.instance_variable_set(:@blur_score, scored_result.score)
+          @records << obj
+        rescue ActiveRecord::RecordNotFound
+        end
+      end
+      self
+    end
+
+  end
+end

data/lib/active_blur/searchable.rb

@@ -0,0 +1,119 @@
+module ActiveBlur
+  module Searchable
+    class <<self
+      def included(base) #:nodoc:
+        base.module_eval do
+          extend(ActsAsMethods)
+        end
+      end
+    end
+
+    module ActsAsMethods
+      #
+      # This method is defined on all ActiveRecord::Base subclasses. It
+      # is false for classes on which #searchable has not been called, and
+      # true for classes on which #searchable has been called.
+      #
+      # ==== Returns
+      #
+      # +false+
+      #
+      def searchable?
+        false
+      end
+
+      private
+      
+      def searchable(options = {})
+        ActiveBlur.setup_class(self, options)
+        if searchable?
+          active_blur_options[:include].concat(Util::Array(options[:include]))
+        else
+          extend ClassMethods
+          include InstanceMethods
+
+          class_attribute :active_blur_options
+
+          unless options[:auto_index] == false
+            before_save :mark_for_auto_indexing
+            after_save :index_if_needed
+          end
+
+          unless options[:auto_remove] == false
+            after_destroy do |searchable|
+              searchable.blur_remove_from_index
+            end
+          end
+          #options[:include] = Util::Array(options[:include])
+
+          self.active_blur_options = options
+        end
+      end
+    end
+
+    module ClassMethods
+      def self.extended(base) #:nodoc:
+        class <<base
+          alias_method :search, :blur_search unless method_defined? :search
+          alias_method :reindex, :blur_reindex unless method_defined? :reindex
+        end
+      end
+
+      def blur_search(query,options = {})
+        ActiveBlur.search(self, query, options)
+      end
+
+      def blur_reindex()
+        ActiveBlur.reindex(self)
+      end
+      
+      #
+      # Classes that have been defined as searchable return +true+ for this
+      # method.
+      #
+      # ==== Returns
+      #
+      # +true+
+      #
+      def searchable?
+        true
+      end
+    end
+
+    module InstanceMethods
+      def self.included(base) #:nodoc:
+        base.module_eval do
+          alias_method :index, :blur_index unless method_defined? :index
+          alias_method :remove_from_index, :blur_remove_from_index unless method_defined? :remove_from_index
+          alias_method :score, :blur_score unless method_defined? :score
+        end
+      end
+
+      def blur_index
+        ActiveBlur.index(self)
+      end
+
+      def blur_remove_from_index
+        ActiveBlur.remove(self)
+      end
+
+      def blur_score
+        @blur_score
+      end
+
+      private
+
+      def mark_for_auto_indexing
+        @marked_for_auto_indexing = new_record? || changed?
+        true
+      end
+
+      def index_if_needed
+        if @marked_for_auto_indexing
+          blur_index
+          remove_instance_variable(:@marked_for_auto_indexing)
+        end
+      end
+    end
+  end
+end