make-text-search 0.1

data/README.rdoc

@@ -0,0 +1,78 @@
+= MakeTextSearch
+
+MakeTextSearch is a tool that let you make full-text search using the engine of your RDBMS easily.
+
+There are a tools like {Sphinx}[http://sphinxsearch.com/] or {Lucene}[http://lucene.apache.org/] very powerful and fast, but they require an extra effort to configure and maintain them, because they are tools outside of the RDBMS. Some RDBMS, like PostgreSQL or MySQL, have their own full-text search engine, so the time in configuration and maintenance is lesser.
+
+In this first version we have implemented support for {PostgreSQL}[http://www.postgresql.org/docs/8.3/static/textsearch.html]. In the near future we will implement support for {MySQL}[http://dev.mysql.com/doc/refman/5.1/en/fulltext-search.html] and more. If the database has no full-text search engine it will use an equivalent using plain SQL.
+
+MakeTextSearch works with Rails 3
+
+== Installation
+
+In the +Gemfile+ file add
+
+  gem "make-text-search"
+
+After the +bundle install+ you have to generate the migration to create the documents table
+
+  rails generate text_search:migration
+
+== Usage
+
+In the models where you want to run full-text searchs, you have to declare the indexed fields using +has_text_search+
+
+  class Post < ActiveRecord::Base
+    has_text_search :title, :content
+  end
+
+The fields added to the index can be virtual.
+
+  class Post < ActiveRecord::Base
+    belongs_to :user
+
+    # Add the user name to the index
+    def user_name
+      user.try :name
+    end
+
+    has_text_search :title, :content, :user_name
+  end
+
+=== Filters
+
+The content added to the index can be filtered. Right now there are two filters: +:substrings+ y +:strip_html+
+
+  class Post < ActiveRecord::Base
+    has_text_search :title, :filter => :substrings
+    has_text_search :content, :intro :filter => [:strip_html, :substrings]
+  end
+
+You can use several filters using an array. The order is important. If you use both +:substrings+ and +:strip_html+, +:strip_html+ should be the first.
+
++:substrings+ let you search inside the words. For example, the word +knowledge+ can be found with +owled+ if you filter the content with +:substrings+.
+
++:strip_html+ elimina las etiquetas HTML y traduce las entities a su versión en UTF-8. Por ejemplo
++:strip_html+ removes the HTML tags and it translates HTML entities to its equivalent in UTF-8:
+
+  Ir a <a href="http://www.google.es">Google Espa&ntilde;a</a>
+
+will be
+
+  Ir a Google España
+
+
+== Search
+
+Para realizar las búsquedas hay que usar el scope #search_text
+To perform search you have to use the scope #search_text
+
+  Post.search_text("foo")
+
+  Post.published.search_text("foo & bar").paginate(:page => params[:page])
+
+== TODO
+
+* Query builder. Add & and | operators
+* Option :language in #search_text
+* RDoc-ize methods

data/Rakefile

@@ -0,0 +1,36 @@
+
+APPLICATION_NAME = "temp_make_text_search_tests"
+
+task :default => "test:pg"
+
+namespace :test do
+
+  def run_tests(adapter)
+    Process.waitpid(fork do
+      ENV["MTS_GEM_PATH"] = File.dirname(__FILE__)
+
+      Dir.chdir "/tmp"
+      system "rm", "-fr", APPLICATION_NAME if File.directory?(APPLICATION_NAME)
+      system "rails", "new", APPLICATION_NAME, "-d", adapter, "-G", "-J", "-m", "#{ENV["MTS_GEM_PATH"]}/test/app_template.rb"
+
+      Dir.chdir "/tmp/#{APPLICATION_NAME}"
+      exec "rake", "RAILS_ENV=test", "db:create", "db:migrate", "test", "db:drop"
+    end)
+  end
+
+  desc "Run tests againts PostgreSQL"
+  task :pg do
+    run_tests "postgresql"
+  end
+
+  #desc "Run tests againts SQLite3"
+  #task :sqlite3 do
+  #  run_tests "sqlite3"
+  #end
+
+  #desc "Run tests againts MySQL"
+  #task :mysql do
+  #  run_tests "mysql"
+  #end
+
+end

data/lib/generators/migration.rb

@@ -0,0 +1,15 @@
+require 'rails/generators/active_record'
+
+module TextSearch
+  class MigrationGenerator < ActiveRecord::Generators::Base
+    argument :name, :type => :string, :default => "add_text_search_documents_table"
+
+    source_root File.join(File.dirname(__FILE__), "templates")
+
+    def create_migration_file
+      migration_template "migration.rb", "db/migrate/#{file_name}.rb"
+    end
+
+  end
+end
+

data/lib/generators/templates/migration.rb

@@ -0,0 +1,9 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def self.up
+    create_text_search_documents_table <%= Rails.application.config.make_text_search.table_name.inspect  %>
+  end
+
+  def self.down
+    drop_table <%= Rails.application.config.make_text_search.table_name.inspect  %>
+  end
+end

data/lib/make-text-search/adapters/postgresql_ts.rb

@@ -0,0 +1,65 @@
+module MakeTextSearch
+  module Adapters
+    class PostgreSQL
+
+      attr_reader :connection
+
+      def self.is_available?(connection)
+        begin
+          connection.select_one("select to_tsquery('test') as query").has_key?("query")
+        rescue ActiveRecord::StatementInvalid
+          false
+        end
+      end
+
+      def initialize(connection)
+        @connection = connection
+      end
+
+      # Schema actions
+
+      def create_text_search_documents_table(table_name)
+        connection.execute %[CREATE TABLE #{table_name} (record_type varchar(300) NOT NULL, record_id integer NOT NULL, language varchar(20), document tsvector)]
+        connection.execute %[CREATE INDEX #{table_name}_idx ON #{table_name} USING gin(document)]
+      end
+
+
+      # Document actions
+
+      def quote(value)
+        @connection.quote value
+      end
+
+      def update_document(record)
+
+        record_class = record.class
+        return if record_class.text_search_fields.empty?
+
+        table_name = Rails.application.config.make_text_search.table_name
+        record_type = quote record_class.name
+        record_id = quote record.id
+        language = quote record.text_search_language
+
+        document = "to_tsvector(#{language}, #{quote record.text_search_build_document})"
+
+        if connection.select_value("SELECT count(*) FROM #{table_name} WHERE #{_where_record record}").to_i == 0
+          connection.insert(%[INSERT INTO #{table_name}
+                              (record_type, record_id, language, document)
+                             VALUES
+                              (#{record_type}, #{record_id}, #{language}, #{document})])
+        else
+          connection.update(%[UPDATE #{table_name} SET document = #{document} WHERE #{_where_record record}])
+        end
+      end
+
+      def remove_document(record)
+        connection.delete "DELETE FROM #{Rails.application.config.make_text_search.table_name} WHERE #{_where_record record}"
+      end
+
+      def _where_record(record)
+        "record_type = #{quote record.class.name} AND record_id = #{quote record.id}"
+      end
+
+    end
+  end
+end

data/lib/make-text-search/filters.rb

@@ -0,0 +1,39 @@
+module MakeTextSearch
+
+  module SubstringsFilter
+    extend self
+
+    def substrings(word, min_length = 3)
+      results = []
+      for starts in 0..word.size
+        started = word[starts..-1]
+        for ends in min_length..started.size
+          results << word[starts, ends]
+        end
+      end
+
+      results
+    end
+
+    def apply_filter(record, value)
+      value.gsub(/(\S+)/) { substrings($1).join(" ") }
+    end
+  end
+
+  module StripHtmlFilter
+    extend self
+
+    def translate_html_entities!(value)
+      # http://gist.github.com/582351
+      @entities_map ||= File.read("#{File.dirname(__FILE__)}/html_entities.dat").split("\0").inject({}) {|hash, line| line = line.split(" ", 2); hash[line[0]] = line[1]; hash };
+
+      value.gsub!(/&(\w+);/) { @entities_map[$1] || $1 } or value
+    end
+
+    def apply_filter(record, value)
+      # TODO extracts the content for some attributes like alt, title and longdesc
+      translate_html_entities! value.gsub(/<[^>]*>/, "")
+    end
+  end
+
+end

data/lib/make-text-search/models.rb

@@ -0,0 +1,65 @@
+module MakeTextSearch
+
+  module ModelHelpers
+    extend ActiveSupport::Concern
+
+    included do
+      class_inheritable_array :text_search_fields
+      self.text_search_fields = []
+
+      after_save :text_search_update_document
+      after_destroy :text_search_remove_document
+    end
+
+    module ClassMethods
+      def has_text_search(*fields)
+        options = fields.extract_options!
+        options.assert_valid_keys :filter
+
+        if options[:filter]
+          options[:filter] = [options[:filter]].flatten.map! {|filter_name| "make_text_search/#{filter_name}_filter".camelize.constantize }
+        end
+
+        fields.each do |field|
+          self.text_search_fields.push([field, options])
+        end
+      end
+
+      def search_text(query)
+        where MakeTextSearch.build_condition(self, query)
+      end
+    end
+
+    def text_search_update_document
+      if not self.class.text_search_fields.empty?
+        self.class.connection.text_search_adapter.update_document self
+      end
+    end
+
+    def text_search_remove_document
+      if not self.class.text_search_fields.empty?
+        self.class.connection.text_search_adapter.remove_document self
+      end
+    end
+
+    def text_search_language
+      Rails.application.config.make_text_search.default_language
+    end
+
+    def text_search_build_document
+      self.class.text_search_fields.map do |ts_field|
+        field_name, options = ts_field
+
+        if value = send(field_name)
+          value = value.to_s
+
+          if filters = options[:filter]
+            filters.each {|f| value = f.apply_filter(self, value) }
+          end
+
+          value
+        end
+      end.compact.join(" ")
+    end
+  end
+end

data/lib/make-text-search/query.rb

@@ -0,0 +1,8 @@
+class <<MakeTextSearch
+  def build_condition(model, query)
+    # TODO use has_text_search_for_postgresql? to implement all the backends
+
+    db_connection =  model.connection
+    %[#{model.table_name}.id IN (SELECT record_id FROM #{Rails.application.config.make_text_search.table_name} WHERE record_type = #{db_connection.quote model.name} AND document @@ to_tsquery(#{db_connection.quote query}))]
+  end
+end

data/lib/make-text-search/schema.rb

@@ -0,0 +1,42 @@
+
+require "make-text-search/adapters/postgresql_ts"
+
+module MakeTextSearch
+  module ConnectionAdapterHelpers
+
+    # Returns the MakeTextSearch adapter for the current connection
+    def text_search_adapter
+      @text_search_adapter ||=
+        begin
+          # TODO Use a list adapter
+          if Adapters::PostgreSQL.is_available?(self)
+            Adapters::PostgreSQL.new(self)
+          else
+            # TODO Generic implementation (for SQLite3, etc)
+            raise NotImplementedError, "There is no adapter for #{self}"
+          end
+        end
+    end
+
+    # Create the text_search_documents table
+    def create_text_search_documents_table(table_name)
+      text_search_adapter.create_text_search_documents_table table_name
+    end
+  end
+
+  module SchemaDumperHelpers #:nodoc:
+    def self.included(cls)
+      cls.alias_method_chain :table, :make_text_search
+    end
+
+    def table_with_make_text_search(table, stream)
+      if table.to_s == Rails.application.config.make_text_search.table_name.to_s
+        stream.puts "  create_text_search_documents_table #{table.inspect}"
+      else
+        table_without_make_text_search(table, stream)
+      end
+
+      stream
+    end
+  end
+end

data/lib/make-text-search.rb

@@ -0,0 +1,23 @@
+module MakeTextSearch
+
+  ActiveSupport.on_load(:active_record) do
+    require 'make-text-search/models'
+    require 'make-text-search/query'
+    require 'make-text-search/schema'
+    require 'make-text-search/filters'
+
+    include ModelHelpers
+    class ActiveRecord::ConnectionAdapters::AbstractAdapter; include ConnectionAdapterHelpers; end
+    class ActiveRecord::SchemaDumper; include SchemaDumperHelpers; end
+  end
+
+  class Railtie < ::Rails::Railtie
+    config.make_text_search = ActiveSupport::OrderedOptions.new
+    config.make_text_search.table_name = "text_search_documents"
+    config.make_text_search.default_language = "english"
+
+    generators do
+      load "generators/migration.rb"
+    end
+  end
+end

data/test/app_template.rb

@@ -0,0 +1,25 @@
+
+if db_user = ENV["DB_USER"]
+  require 'yaml'
+  database = YAML.load(File.read("config/database.yml"))
+  database.values.each do |connection|
+    connection["username"] = ENV["DB_USER"]
+  end
+  File.open("config/database.yml", "w") {|f| f.write database.to_yaml }
+end
+
+gem "make-text-search", :path => ENV["MTS_GEM_PATH"]
+
+generate "text_search:migration"
+generate "model", "Post", "title:string", "content:text"
+
+File.open("app/models/post.rb", "a") do |f|
+  f.puts "\nPost.has_text_search :title, :filter => :substrings"
+  f.puts "Post.has_text_search :content, :filter => [:substrings, :strip_html]"
+end
+
+# Copy tests from the gem
+Dir["#{ENV["MTS_GEM_PATH"]}/test/*_test.rb"].each do |filename|
+  file "test/unit/#{File.basename filename}", File.read(filename)
+end
+

data/test/basics_test.rb

@@ -0,0 +1,65 @@
+# coding: utf-8
+
+require 'test_helper'
+
+class BasicsTest < ActiveSupport::TestCase
+
+  def query(words, expected_ids)
+    assert_equal expected_ids.sort, Post.search_text(words).map {|post| post.id }.sort, "The query #{words.inspect} expects #{expected_ids.inspect}"
+  end
+
+  test "the filter can be used after create some records" do
+    first = Post.create!(:title => "First post", :content => "111 222").id
+    second = Post.create!(:title => "Second post", :content => "random text").id
+
+    query "post", [first, second]
+    query "First", [first]
+    query "first", [first]
+    query "random", [second]
+    query "0", []
+    query "11", []
+
+    # With substrings. Filter has be to activated in the model (see test/app_template.rb)
+    query "econ", [second]
+    query "andom", [second]
+  end
+
+  test "search_text can be re-scoped" do
+    20.times { Post.create!(:title => "The same post", :content => "somethingwithoutsense") }
+
+    assert_equal 20, Post.search_text("somethingwithoutsense").count
+    assert_equal 2, Post.search_text("somethingwithoutsense").limit(2).count
+    assert_equal "The same post", Post.search_text("somethingwithoutsense").last.title
+  end
+
+  test "the index is updated after change the posts" do
+    colors = {}
+    %w(red green blue black white orange).each {|color| colors[color] = Post.create!(:title => "Post #{color}") }
+
+    query "red", [colors["red"].id]
+    assert_equal Post.search_text("post").count, colors.size
+
+    colors["red"].destroy
+
+    query "red", []
+    assert_equal Post.search_text("post").count, colors.size - 1
+
+    colors["black"].update_attribute :content, "Old red"
+    query "red", [colors["black"].id]
+  end
+
+  test "html entities are translated with filters" do
+
+    begin
+      old_default_language = Rails.application.config.make_text_search.default_language
+      Rails.application.config.make_text_search.default_language = "spanish"
+
+      spain = Post.create!(:content => "Pa&iacute;s Espa&ntilde;a")
+      query "país", [spain.id]
+      query "españa", [spain.id]
+    ensure
+      Rails.application.config.make_text_search.default_language = old_default_language
+    end
+  end
+
+end

data/test/schema_test.rb

@@ -0,0 +1,11 @@
+require 'test_helper'
+
+class SchemaTest < ActiveSupport::TestCase
+  test "schema has to include the call for create_text_search_documents_table" do
+    assert_match /^\s*create_text_search_documents_table/, Rails.root.join("db/schema.rb").read
+  end
+
+  test "documents table has to be created" do
+    assert_operator ActiveRecord::Base.connection.tables, "include?", Rails.application.config.make_text_search.table_name
+  end
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification 
+name: make-text-search
+version: !ruby/object:Gem::Version 
+  hash: 9
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  version: "0.1"
+platform: ruby
+authors: 
+- Ayose Cazorla
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-13 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: Some RDBMS (like PostgreSQL 8.3 and newer) implement full text search directly, so you don't need external tools. This Rails plugin gives that functionality in a generic way.
+email: ayosec@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/make-text-search.rb
+- lib/generators/templates/migration.rb
+- lib/generators/migration.rb
+- lib/make-text-search/query.rb
+- lib/make-text-search/html_entities.dat
+- lib/make-text-search/models.rb
+- lib/make-text-search/filters.rb
+- lib/make-text-search/adapters/postgresql_ts.rb
+- lib/make-text-search/schema.rb
+- test/schema_test.rb
+- test/app_template.rb
+- test/basics_test.rb
+- Rakefile
+- README.rdoc
+has_rdoc: true
+homepage: http://github.com/setepo/make_text_search
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Adapts the native Full Text Search of the RDBMS
+test_files: []
+