fulltext_searchable 0.1.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,12 @@
+.bundle/
+Gemfile.lock
+log/*.log
+pkg/
+spec/dummy/db/*
+spec/dummy/config/database.yml
+spec/dummy/log/*.log
+spec/dummy/tmp/
+coverage*
+rdoc/
+*.swp
+

data/.rspec

@@ -0,0 +1,2 @@
+--color
+

data/Gemfile

@@ -0,0 +1,20 @@
+source "http://rubygems.org"
+
+if ENV['RAILS_VER'] == '3.0'
+  gem "rails", "~> 3.0.5"
+  gem "mysql2", "~> 0.2.6"
+  gem 'will_paginate', :git => 'git://github.com/mislav/will_paginate.git', :branch => 'rails3'
+else
+  gem "rails", "~> 3.0"
+  gem "mysql2"
+  gem 'will_paginate', "~> 3.0.4"
+end
+
+if ENV['PARANOID'] == 'original'
+  gem 'acts_as_paranoid'
+else
+  gem 'rails3_acts_as_paranoid', :git => 'git://github.com/mshibuya/rails3_acts_as_paranoid.git'
+end
+gem "debugger"
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Mitsuhiro Shibuya
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,4 @@
+FulltextSearchable
+===
+
+This project rocks and uses MIT-LICENSE.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/app/models/fulltext_index.rb

@@ -0,0 +1,163 @@
+# coding: utf-8
+
+require 'digest/md5'
+
+##
+# == 概要
+# 全文検索インデックスとして機能するモデル。
+#
+class FulltextIndex < ActiveRecord::Base
+  self.table_name = FulltextSearchable::TABLE_NAME
+  self.primary_key = :_id
+  after_create :set_grn_insert_id
+
+  ## :nodoc:
+  # Association
+  belongs_to :item, :polymorphic => true
+
+  class << self
+    cattr_accessor :target_models
+
+    ##
+    # 全文検索で絞り込む。
+    #
+    # ==== phrase
+    # 検索する単語。スペース区切りで複数指定。
+    #
+    # ==== options
+    # ===== model
+    # 検索対象にするモデルを指定。
+    # 例: FulltextIndex.match('天気', :model => User)
+    # ===== with
+    # 検索対象にしたいレコード(AR object)を指定する。
+    # 例: FulltextIndex.match('天気', :with => @user)
+    #
+    def match(phrase, options={})
+      options = options.symbolize_keys
+      if phrase.is_a? String
+        phrase = phrase.split(/[\s　]/)
+      end
+
+      # モデルで絞り込む
+      model_keywords = []
+      if options[:model]
+        Array.wrap(options.delete(:model)).each do |t|
+          if t.is_a?(Class) && indexed?(t)
+            model_keywords.push(FulltextSearchable.to_model_keyword(t))
+          end
+        end
+      end
+      # レコードで絞り込む
+      item_keywords = []
+      if options[:with]
+        Array.wrap(options.delete(:with)).each do |t|
+          if indexed?(t.class)
+            item_keywords.push(FulltextSearchable.to_item_keyword(t))
+          end
+        end
+      end
+      [model_keywords, item_keywords].each do |keywords|
+        case keywords.count
+        when 0
+        when 1
+          phrase.unshift(keywords.first)
+        else
+          phrase.unshift("(#{keywords.join(' ')})")
+        end
+      end
+      phrase.map!{|i| '+' + i }
+
+      if connection.mroonga_match_against?
+        where("MATCH(`text`) AGAINST(? IN BOOLEAN MODE)",phrase.join(' ')).
+          order(sanitize_sql_array(["MATCH(`text`) AGAINST(? IN BOOLEAN MODE)",phrase.join(' ')]))
+      else
+        where("MATCH(`text`) AGAINST(? IN BOOLEAN MODE)",phrase.join(' ')).
+          order('`_score` DESC')
+      end
+    end
+
+    ##
+    # 結果を取得し、インデックス元モデルの配列に変換。
+    #
+    def items(*args)
+      begin
+        # ActiveRecordの参照テーブル判定処理がおかしいので、SQLクエリ内にピリオドが現れる際に
+        # ピリオドの前の文字列を参照テーブル名と誤認し、eager loadをjoinの方針に切り替えて
+        # polymorphic associationをjoinでeager loadできず例外を吐く。
+        # なのでrescueしてeager loadなしで再度クエリを試みておく。
+        select('`_id`, `item_type`, `item_id`').
+          includes(:item).all(*args).map{|i| i.item }
+      rescue ActiveRecord::EagerLoadPolymorphicError
+        # eager loadなしで試みる
+        select('`_id`, `item_type`, `item_id`').all(*args).map{|i| i.item }
+      end
+    end
+    ##
+    # 特定レコードの更新をインデックスに非同期で反映する。
+    #
+    def update(item)
+      l = lambda do
+        self.match(FulltextSearchable.to_item_keyword(item)).
+          includes(:item).all.each do |record|
+          next unless record.item
+          record.text = record.item.fulltext_keywords
+          record.save
+          end
+      end
+      if FulltextSearchable::Engine.config.respond_to?(:async) &&
+        FulltextSearchable::Engine.config.async
+        Thread.new{ l.call }
+      else
+        l.call
+      end
+    end
+    ##
+    # 全文検索インデックスを再構築する。
+    #
+    def rebuild_all
+      if connection.tables.include?(FulltextSearchable::TABLE_NAME)
+        connection.drop_table FulltextSearchable::TABLE_NAME
+      end
+      connection.create_fulltext_index_table
+
+      ActiveRecord::Base.descendants.each do |model|
+        next unless model.table_exists? &&
+          model.fulltext_searchable?
+        n = 0
+        depends = model.fulltext_dependent_models
+        begin
+          rows = model.unscoped.includes(depends).offset(n).
+            limit(FulltextSearchable::PROCESS_UNIT).order(:id).all
+          rows.each do |r|
+            index = self.find_or_initialize_by_key(create_key(r))
+            index.update_attributes :text => r.fulltext_keywords, :item => r
+          end
+          n += rows.count
+        end while rows.count >= FulltextSearchable::PROCESS_UNIT
+      end
+    end
+    ##
+    # key用文字列を生成する。
+    #
+    def create_key(item)
+      "#{item.class.name}_#{'% 10d' % item.id}"
+    end
+
+    protected
+
+    ##
+    # 全文検索対応モデルかどうかを判定し、返す。
+    #
+    def indexed?(klass)
+      klass.respond_to?(:fulltext_searchable?) && klass.fulltext_searchable?
+    end
+  end
+
+  protected
+  ##
+  # 互換のためレコード作成時に主キーをセット。
+  #
+  def set_grn_insert_id
+    self.id = connection.execute('SELECT last_insert_grn_id();').to_a.first.first
+  end
+end

data/fulltext_searchable.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fulltext_searchable/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fulltext_searchable"
+  spec.version       = FulltextSearchable::VERSION
+  spec.authors       = ["M.Shibuya"]
+  spec.email         = ["mit.shibuya@gmail.com"]
+  spec.summary       = %q{Rails engine that provides fulltext-search capability using mroonga}
+  spec.description   = %q{Rails engine that provides fulltext-search capability using mroonga(formerly. groonga storage engine). Requires Rails ~> 3.0}
+  spec.homepage      = "https://github.com/greenbell/fulltext_searchable"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activerecord", "~> 3.0"
+  spec.add_dependency "mysql2"
+  spec.add_dependency "htmlentities"
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "cover_me"
+  spec.add_development_dependency "database_cleaner"
+  spec.add_development_dependency "factory_girl"
+  spec.add_development_dependency "faker"
+  spec.add_development_dependency "jeweler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rdoc"
+  spec.add_development_dependency "rspec-rails"
+end

data/lib/fulltext_searchable/active_record.rb

@@ -0,0 +1,208 @@
+# coding: utf-8
+require 'htmlentities'
+
+module FulltextSearchable
+  ##
+  # == 概要
+  # ActiveRecord::Baseを拡張するモジュール
+  #
+  module ActiveRecord # :nodoc:
+    def self.included(base) # :nodoc:
+      base.extend(ClassMethods)
+    end
+    #
+    # ActiveRecord::Baseにextendされるモジュール
+    #
+    module ClassMethods 
+      ##
+      # 全文検索機能を有効にする。
+      #
+      # ==== columns
+      # 全文検索の対象とするカラムを指定。
+      #
+      # ==== 例:
+      #   fulltext_searchable :title, :body
+      #
+      def fulltext_searchable(columns=[], options={}, &block)
+        options = options.symbolize_keys
+        cattr_accessor  :fulltext_columns,
+          :fulltext_keyword_proc, :fulltext_referenced_columns
+
+        referenced = options.delete(:referenced)
+        self.fulltext_columns = Array.wrap(columns)
+        self.fulltext_referenced_columns = Array.wrap(referenced) if referenced
+        self.fulltext_keyword_proc = block
+
+        class_eval <<-EOV
+        has_one :fulltext_index, {
+          :as => :item,
+          :conditions => proc{ {:key => FulltextIndex.create_key(self) } }
+        }
+
+        include FulltextSearchable::ActiveRecord::Behaviors
+
+        after_commit       :save_fulltext_index
+        after_destroy      :destroy_fulltext_index
+        EOV
+        if self.fulltext_referenced_columns
+          class_eval <<-EOV
+            before_save       :check_fulltext_changes
+          EOV
+        end
+      end
+      ##
+      # 全文検索対応モデルかどうかを返す。
+      #
+      def fulltext_searchable?
+        self.ancestors.include?(
+          ::FulltextSearchable::ActiveRecord::Behaviors)
+      end
+    end
+    #
+    # 各モデルにincludeされるモジュール
+    #
+    module Behaviors
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        ##
+        # 各モデルに対し全文検索を行う。
+        #
+        def fulltext_match(phrase)
+          FulltextIndex.match(phrase, :model => self)
+        end
+
+        ##
+        # eager loadのために全文検索対応モデルが依存する他のモデルを返す。
+        #
+        def fulltext_dependent_models(columns=nil)
+          columns ||= fulltext_columns
+          if columns.is_a? Hash
+            columns = Array.wrap(columns)
+          end
+          if columns.is_a? Array
+            result = []
+            columns.flatten!
+            columns.each do |i|
+              if i.is_a?(Hash)
+                i.each do |k,v|
+                  if v.is_a?(Hash) || v.is_a?(Array)
+                    r = fulltext_dependent_models(v)
+                    if r
+                      result.push({k=>r})
+                    else
+                      result.push(k)
+                    end
+                  elsif v.to_s.downcase != 'html'
+                    result.push(k)
+                  end
+                end
+              end
+            end
+            case result.count
+            when 0
+              nil
+            when 1
+              result.first
+            else
+              result
+            end
+          else
+            nil
+          end
+        end
+      end
+
+      ##
+      # レコードの内容を全文検索インデックス用に変換
+      #
+      def fulltext_keywords
+        # 論理削除されていたら空に
+        return '' if self.respond_to?(:deleted?) && self.deleted?
+        [
+          FulltextSearchable.to_model_keyword(self.class.name),
+          FulltextSearchable.to_item_keyword(self),
+        ].
+        tap{|a| a.push(fulltext_keyword_proc.call) if fulltext_keyword_proc }.
+          concat(collect_fulltext_keywords(self, fulltext_columns)).
+          flatten.join(' ')
+      end
+
+      private
+
+      ##
+      # before_saveにフック。全文検索対象カラムが変更されているかどうか調べる。
+      #
+      def check_fulltext_changes
+        @fulltext_change = fulltext_referenced_columns &&
+            fulltext_referenced_columns.any?{|i| changes[i]}
+        true
+      end
+      ##
+      # after_commitにフック。
+      #
+      def save_fulltext_index
+        if self.fulltext_index
+          if !fulltext_index.text.empty? && (!defined?(@fulltext_change) || @fulltext_change)
+            FulltextIndex.update(self)
+          else
+            self.fulltext_index.text = fulltext_keywords
+            self.fulltext_index.save
+          end
+        else
+          self.create_fulltext_index(
+            :key => FulltextIndex.create_key(self),
+            :text => fulltext_keywords
+          )
+        end
+      end
+      ##
+      # after_destroyにフック。全文検索インデックスを削除
+      #
+      def destroy_fulltext_index
+        return unless fulltext_index
+        unless persisted?
+          fulltext_index.destroy
+        else
+          fulltext_index.update_attributes(:text => '')
+        end
+      end
+
+      def collect_fulltext_keywords(target, columns)
+        result = []
+        return result unless target
+        if columns.is_a? Hash
+          columns = Array.wrap(columns)
+        end
+        unless columns.is_a? Array
+          return result.push(target.send(columns).to_s)
+        end
+        columns.flatten!
+        columns.each do |column|
+          if column.is_a? Hash
+            column.each do |k,v|
+              if v.to_s.downcase == 'html'
+                result.push(
+                  HTMLEntities.new(:xhtml1).decode(
+                    target.send(k.to_s).to_s.gsub(/<[^>]*>/ui,'')
+                  ).gsub(/[ \s]+/u, ' ') # contains &nbsp;
+                )
+              else
+                Array.wrap(target.send(k)).each do |t|
+                  result.concat([
+                    FulltextSearchable.to_item_keyword(t),
+                    collect_fulltext_keywords(t, v)
+                  ])
+                end
+              end
+            end
+          else
+            result.push(collect_fulltext_keywords(target, column))
+          end
+        end
+        result.flatten
+      end
+    end
+  end
+end
+

data/lib/fulltext_searchable/engine.rb

@@ -0,0 +1,4 @@
+module FulltextSearchable
+  class Engine < Rails::Engine
+  end
+end

data/lib/fulltext_searchable/mysql2_adapter.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+
+module FulltextSearchable
+  module Mysql2Adapter
+    def create_fulltext_index_table
+      execute( <<SQL
+CREATE TABLE IF NOT EXISTS `#{::FulltextSearchable::TABLE_NAME}` (
+  `_id` INT(11),
+  `key` VARCHAR(32),
+  `item_type` VARCHAR(255),
+  `item_id` INT(11),
+  `text` TEXT,
+  #{"`_score` FLOAT," unless mroonga_match_against?}
+  PRIMARY KEY(`key`),
+  #{"UNIQUE " if mroonga_unique_hash_index_safe?}INDEX(`_id`) USING HASH,
+  FULLTEXT INDEX (`text`)
+) ENGINE = #{mroonga_storage_engine_name} COLLATE utf8_unicode_ci;
+SQL
+      )
+    end
+
+    def mroonga_match_against?
+      mroonga_version >= '1.2'
+    end
+
+    def mroonga_version
+      execute("SHOW VARIABLES LIKE '#{mroonga_storage_engine_name}_version';").map(&:last).first ||
+        '0.0' # older than 1.0
+    end
+
+    def mroonga_storage_engine_name
+      @mroonga_storage_engine_name ||=
+        execute('SHOW ENGINES;').map(&:first).find{|name| name =~ /.+roonga/} or
+        raise "mroonga or groonga storage engine is not installed"
+    end
+
+    private
+
+    ##
+    # mroonga on MacOS X fails truncation/deletion of unique hash indexed table.
+    # As a workaround, we actually temporary table with unique hash index and
+    # see if it is safe to truncate it.
+    #
+    def mroonga_unique_hash_index_safe?
+      temporary_table_name = ::FulltextSearchable::TABLE_NAME + '_temp_' + Time.now.to_i.to_s
+      execute( <<SQL
+CREATE TABLE IF NOT EXISTS `#{temporary_table_name}` (
+  `_id` INT(11),
+  UNIQUE INDEX(`_id`) USING HASH
+) ENGINE = #{mroonga_storage_engine_name} COLLATE utf8_unicode_ci;
+SQL
+      )
+      safe = true
+      begin
+        execute("TRUNCATE TABLE `#{temporary_table_name}`;")
+      rescue 
+        safe = false
+      end
+      drop_table temporary_table_name
+      safe
+    end
+  end
+end
+

data/lib/fulltext_searchable/version.rb

@@ -0,0 +1,3 @@
+module FulltextSearchable
+  VERSION = "0.1.0"
+end

data/lib/fulltext_searchable.rb

@@ -0,0 +1,50 @@
+# coding: utf-8
+$:.unshift(File.dirname(__FILE__))
+
+require 'digest/md5'
+##
+# == 概要
+# モデルを全文検索対応にするプラグイン
+#
+module FulltextSearchable
+  require 'fulltext_searchable/engine' if defined?(Rails)
+
+  # 再構築タスク時の一回の処理レコード数
+  PROCESS_UNIT = 1000
+  TABLE_NAME = 'fulltext_indices'
+
+  class << self
+    def to_model_keyword(model)
+      '' + Digest::MD5.hexdigest('FulltextSearchable_'+model.to_s)[0,9] + ''
+    end
+    def to_item_keyword(item)
+      '' + Digest::MD5.hexdigest('FulltextSearchable_'+item.class.to_s)[0,8] + '_' + item.id.to_s + ''
+    end
+  end
+
+  autoload :ActiveRecord,  'fulltext_searchable/active_record'
+  autoload :Mysql2Adapter, 'fulltext_searchable/mysql2_adapter'
+end
+
+ActiveSupport.on_load(:active_record) do
+  ActiveRecord::Base.class_eval { include FulltextSearchable::ActiveRecord }
+
+  require 'active_record/connection_adapters/mysql2_adapter'
+  ActiveRecord::ConnectionAdapters::Mysql2Adapter.
+    send(:include, FulltextSearchable::Mysql2Adapter)
+
+  ActiveRecord::SchemaDumper.class_eval do
+    def table_with_grn(table, stream)
+      if table.to_s == ::FulltextSearchable::TABLE_NAME
+        tbl = StringIO.new
+        tbl.puts "  create_fulltext_index_table"
+        tbl.puts ""
+        tbl.rewind
+        stream.print tbl.read
+      else
+        table_without_grn(table, stream)
+      end
+    end
+    alias_method_chain :table, :grn
+  end
+end

data/lib/rails/generators/fulltext_searchable/fulltext_searchable_generator.rb

@@ -0,0 +1,46 @@
+require 'rails/generators'
+require 'rails/generators/migration'
+
+class FulltextSearchableGenerator < Rails::Generators::Base
+  include Rails::Generators::Migration
+
+  def self.source_root
+    File.join(File.dirname(__FILE__), 'templates')
+  end
+
+  def self.next_migration_number(dirname) #:nodoc:
+    if ActiveRecord::Base.timestamped_migrations
+      Time.now.utc.strftime("%Y%m%d%H%M%S")
+    else
+      "%.3d" % (current_migration_number(dirname) + 1)
+    end
+  end
+
+
+  # Every method that is declared below will be automatically executed when the generator is run
+
+  def create_migration_file
+    f = File.open File.join(File.dirname(__FILE__), 'templates', 'schema.rb')
+    schema = f.read; f.close
+
+    schema.gsub!(/ActiveRecord::Schema.*\n/, '')
+    schema.gsub!(/^end\n*$/, '')
+
+    f = File.open File.join(File.dirname(__FILE__), 'templates', 'migration.rb')
+    migration = f.read; f.close
+    migration.gsub!(/SCHEMA_AUTO_INSERTED_HERE/, schema)
+
+    tmp = File.open "tmp/~migration_ready.rb", "w"
+    tmp.write migration
+    tmp.close
+
+    migration_template  File.expand_path(tmp.path),
+                        'db/migrate/create_fulltext_indices_table.rb'
+    remove_file 'tmp/~migration_ready.rb'
+  end
+
+  def copy_initializer_file
+    copy_file 'initializer.rb', 'config/initializers/fulltext_searchable.rb'
+  end
+
+end

data/lib/rails/generators/fulltext_searchable/templates/initializer.rb

@@ -0,0 +1,7 @@
+module FulltextSearchable
+  class Engine < Rails::Engine
+
+    # if set to true, updates are processed asynchronously(in another thread)
+    config.async = false
+  end
+end

data/lib/rails/generators/fulltext_searchable/templates/migration.rb

@@ -0,0 +1,9 @@
+class CreateFulltextIndicesTable < ActiveRecord::Migration
+  def self.up
+    SCHEMA_AUTO_INSERTED_HERE
+  end
+
+  def self.down
+    drop_table :fulltext_indices
+  end
+end

data/lib/rails/generators/fulltext_searchable/templates/schema.rb

@@ -0,0 +1,5 @@
+ActiveRecord::Schema.define(:version => 0) do
+
+  create_fulltext_index_table
+
+end