dusen 0.2.2 → 0.3.0

data/lib/dusen/description.rb

@@ -1,20 +1,21 @@
 # encoding: utf-8
 
+# This is the DSL to describe a Syntax.
 module Dusen
   class Description
 
     attr_reader :syntax
 
-    def initialize
-      @syntax = Syntax.new
+    def initialize(syntax)
+      @syntax = syntax
     end
 
     def search_by(field, &scoper)
       @syntax.learn_field(field, &scoper)
     end
 
-    def self.read_syntax(&dsl)
-      description = new
+    def self.parse_syntax(syntax, &dsl)
+      description = new(syntax)
       description.instance_eval(&dsl)
       description.syntax
     end

data/lib/dusen/query.rb

@@ -5,20 +5,40 @@ module Dusen
 
     include Enumerable
 
-    def initialize
-      @tokens = []
+    attr_reader :tokens
+
+    def initialize(initial_tokens = [])
+      @tokens = initial_tokens
     end
 
     def <<(token)
-      @tokens << token
+      tokens << token
+    end
+
+    def [](index)
+      tokens[index]
     end
 
+    #def +(tokens)
+    #  tokens.each do |token|
+    #    self << token
+    #  end
+    #end
+
     def to_s
       collect(&:to_s).join(" + ")
     end
 
     def each(&block)
-      @tokens.each(&block)
+      tokens.each(&block)
+    end
+
+    def condensed
+      texts = tokens.select(&:text?).collect(&:value)
+      field_tokens = tokens.reject(&:text?)
+      condensed_tokens = field_tokens
+      condensed_tokens << Token.new(texts) if texts.present?
+      self.class.new(condensed_tokens)
     end
 
   end

data/lib/dusen/railtie.rb

@@ -0,0 +1,9 @@
+#module Dusen
+#  class Railtie < Rails::Railtie
+#    railtie_name :dusen
+#
+#    rake_tasks do
+#      load "dusen/tasks.rb"
+#    end
+#  end
+#end

data/lib/dusen/syntax.rb

@@ -19,6 +19,7 @@ module Dusen
     def search(root_scope, query)
       scope = root_scope
       query = parse(query) if query.is_a?(String)
+      query = query.condensed
       query.each do |token|
         scoper = @scopers[token.field] || unknown_scoper
         scope = scoper.call(scope, token.value)
@@ -26,6 +27,10 @@ module Dusen
       scope
     end
 
+    def fields
+      @scopers
+    end
+
     def parse(query)
       Parser.parse(query)
     end

data/lib/dusen/tasks.rb

@@ -0,0 +1,31 @@
+#namespace :dusen do
+#
+#  desc 'Creates a MyISAM table "search_texts" for fast Dusen searches'
+#  task :create_search_text do
+#    generator = File.exists?('script/generate') ? 'script/generate' : 'rails generate'
+#    output = `#{generator} migration CreateSearchText`
+#    output =~ %r{(db/migrate/.+?\.rb)} or raise "Could not create migration: #{output}"
+#    path = $1
+#    File.open(path, 'w') do |file|
+#      file.write(
+#"
+#class CreateSearchText < ActiveRecord::Migration
+#  def up
+#    create_table :search_texts, :options => 'ENGINE=MyISAM' do |t|
+#      t.integer :model_id
+#      t.string  :model_type
+#      t.text    :words
+#    end
+#    add_index :search_texts, [:model_type, :model_id] # for deletion
+#    execute 'CREATE FULLTEXT INDEX fulltext_index_body ON search_texts (words)' # for search
+#  end
+#  def down
+#    drop_table :search_texts
+#  end
+#end
+#"
+#    )
+#    end
+#  end
+#
+#end

data/lib/dusen/token.rb

@@ -19,5 +19,9 @@ module Dusen
       value
     end
 
+    def text?
+      field == 'text'
+    end
+
   end
 end

data/lib/dusen/util.rb

@@ -8,8 +8,33 @@ module Dusen
       "%#{escape_for_like_query(phrase)}%"
     end
 
+    def escape_with_backslash(phrase, characters)
+      characters << '\\'
+      pattern = /[#{characters.collect(&Regexp.method(:quote)).join('')}]/
+      # debugger
+      phrase.gsub(pattern) do |match|
+        "\\#{match}"
+      end
+    end
+
     def escape_for_like_query(phrase)
-      phrase.gsub("%", "\\%").gsub("_", "\\_")
+      # phrase.gsub("%", "\\%").gsub("_", "\\_")
+      escape_with_backslash(phrase, ['%', '_'])
+    end
+
+    def escape_for_boolean_fulltext_query(phrase)
+      escape_with_backslash(phrase, ['+', '-', '<', '>', '(', ')', '~', '*', '"'])
+    end
+
+    def boolean_fulltext_query(phrases)
+      phrases.collect do |word|
+        escaped_word = Dusen::Util.escape_for_boolean_fulltext_query(word)
+        if escaped_word =~ /\s/
+          %{+"#{escaped_word}"} # no prefixed wildcard possible for phrases
+        else
+          %{+#{escaped_word}*}
+        end
+      end.join(' ')
     end
 
     def qualify_column_name(model, column_name)
@@ -32,5 +57,65 @@ module Dusen
       end
     end
 
+    def select_scope_fields(scope, fields)
+      if scope.respond_to?(:select)
+        # Rails 3
+        scope.select(fields)
+      else
+        # Rails 2
+        scope.scoped(:select => fields)
+      end
+    end
+
+    def drop_all_tables
+      connection = ::ActiveRecord::Base.connection
+      connection.tables.each do |table|
+        connection.drop_table table
+      end
+    end
+
+    def migrate_test_database
+      print "\033[30m" # dark gray text
+      drop_all_tables
+      ::ActiveRecord::Migrator.migrate("#{Rails.root}/db/migrate")
+      print "\033[0m"
+    end
+
+    #def scope_to_sql(scope)
+    #  query = if scope.respond_to?(:to_sql)
+    #    scope.to_sql
+    #  else
+    #    scope.construct_finder_sql({})
+    #  end
+    #end
+
+    def scope_to_sql(options = {})
+      if Rails.version < '3'
+        scope.construct_finder_sql(options)
+      else
+        scope.scoped(options).to_sql
+      end
+    end
+
+    def collect_column(scope, column_name, find_options = {})
+      distinct = find_options.delete(:distinct)
+      qualified_column_name = "`#{scope.table_name}`.`#{column_name}`"
+      select = distinct ? "DISTINCT #{qualified_column_name}" : qualified_column_name
+      query = if Rails.version.to_i < 3
+        scope.construct_finder_sql(find_options.merge(:select => select))
+      else
+        scope.scoped(find_options.merge(:select => select)).to_sql
+      end
+      raw_values = scope.connection.select_values(query)
+      column = scope.columns_hash[column_name.to_s] or raise "Could not retrieve column information: #{column_name}"
+      raw_values.collect { |value| column.type_cast(value) }
+    end
+
+    #def collect_ids(scope)
+    #  scope = select_scope_fields(scope, "`#{scope.table_name}`.`id`")
+    #  query = scope_to_sql(scope)
+    #  ::ActiveRecord::Base.connection.select_values(query).collect(&:to_i)
+    #end
+
   end
 end

data/lib/dusen/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module Dusen
-  VERSION = '0.2.2'
+  VERSION = '0.3.0'
 end

data/lib/dusen.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 
+require 'dusen/version'
 require 'dusen/util'
 require 'dusen/token'
 require 'dusen/description'
@@ -8,5 +9,10 @@ require 'dusen/query'
 require 'dusen/syntax'
 
 if defined?(ActiveRecord)
-  require 'dusen/active_record_ext'
+  require 'dusen/active_record/base_ext'
+  require 'dusen/active_record/search_text'
 end
+
+#if defined?(Rails::Railstie)
+#  require 'dusen/railtie'
+#end

data/spec/rails-2.3/Gemfile

@@ -3,8 +3,10 @@ source :rubygems
 gem 'rails', '~>2.3'
 gem 'rspec', '~>1.3'
 gem 'rspec-rails', '~>1.3'
-gem 'sqlite3'
+gem 'mysql2', '~>0.2.0'
 gem 'ruby-debug', :platforms => :ruby_18
 gem 'test-unit', '~>1.2', :platforms => :ruby_19
+gem 'database_cleaner'
+gem 'andand'
 
 gem 'dusen', :path => '../..'

data/spec/rails-2.3/Rakefile

@@ -7,5 +7,5 @@ task :default => :spec
 desc "Run all specs for a specific rails version"
 Spec::Rake::SpecTask.new() do |t|
   t.spec_opts = ['--options', "\"spec.opts\""]
-  t.spec_files = defined?(SPEC) ? SPEC : FileList['**/*_spec.rb', '../shared/**/*_spec.rb']
+  t.spec_files = defined?(SPEC) ? SPEC : FileList['**/*_spec.rb', '../shared/spec/**/*_spec.rb']
 end

data/spec/rails-2.3/app_root/config/database.yml

@@ -1,21 +1,6 @@
-in_memory:
-  adapter: sqlite3
-  database: ":memory:"
-  verbosity: quiet
-sqlite:
-  adapter: sqlite
-  dbfile: plugin_test.sqlite.db
-sqlite3:
-  adapter: sqlite3
-  dbfile: plugin_test.sqlite3.db
-postgresql:
-  adapter: postgresql
-  username: postgres
-  password: postgres
-  database: plugin_test
-mysql:
-  adapter: mysql
+test:
+  adapter: mysql2
   host: localhost
   username: root
-  password:
-  database: plugin_test
+  password: junior
+  database: dusen_test

data/spec/rails-2.3/spec/spec_helper.rb

@@ -2,25 +2,23 @@
 
 $: << File.join(File.dirname(__FILE__), "/../../lib" )
 
-# Set the default environment to sqlite3's in_memory database
-ENV['RAILS_ENV'] = 'in_memory'
+ENV['RAILS_ENV'] = 'test'
 
 # Load the Rails environment and testing framework
 require "#{File.dirname(__FILE__)}/../app_root/config/environment"
 require 'spec/rails'
-
-# Undo changes to RAILS_ENV
-silence_warnings {RAILS_ENV = ENV['RAILS_ENV']}
+DatabaseCleaner.strategy = :truncation
 
 # Requires supporting files with custom matchers and macros, etc in ./support/ and its subdirectories.
 Dir[File.expand_path(File.join(File.dirname(__FILE__),'support','**','*.rb'))].each {|f| require f}
 
 # Run the migrations
-print "\033[30m" # dark gray text
-ActiveRecord::Migrator.migrate("#{Rails.root}/db/migrate")
-print "\033[0m"
+Dusen::Util.migrate_test_database
 
 Spec::Runner.configure do |config|
-  config.use_transactional_fixtures = true
+  config.use_transactional_fixtures = false
   config.use_instantiated_fixtures  = false
+  config.before(:each) do
+    DatabaseCleaner.clean
+  end
 end

data/spec/rails-3.0/Gemfile

@@ -3,7 +3,9 @@ source :rubygems
 gem 'rails', '~>3.0.17'
 gem 'rspec'
 gem 'rspec-rails'
-gem 'sqlite3'
+gem 'mysql2', '~>0.2.0'
 gem 'ruby-debug', :platforms => :ruby_18
+gem 'database_cleaner'
+gem 'andand'
 
 gem 'dusen', :path => '../..'

data/spec/rails-3.0/Rakefile

@@ -6,6 +6,6 @@ task :default => :spec
 
 desc "Run all specs for a specific rails version"
 RSpec::Core::RakeTask.new(:spec) do |t|
-  t.pattern = defined?(SPEC) ? SPEC : ['**/*_spec.rb', '../shared/**/*_spec.rb']
+  t.pattern = defined?(SPEC) ? SPEC : ['**/*_spec.rb', '../shared/spec/**/*_spec.rb']
   t.verbose = false
 end

data/spec/rails-3.0/app_root/config/database.yml

@@ -1,4 +1,6 @@
 test:
-  adapter: sqlite3
-  database: db/gem_test.db
-  verbosity: quiet
+  adapter: mysql2
+  host: localhost
+  username: root
+  password: junior
+  database: dusen_test

data/spec/rails-3.0/spec/spec_helper.rb

@@ -2,22 +2,21 @@
 
 $: << File.join(File.dirname(__FILE__), "/../../lib" )
 
-# Set the default environment to sqlite3's in_memory database
 ENV['RAILS_ENV'] = 'test'
 ENV['RAILS_ROOT'] = 'app_root'
 
 # Load the Rails environment and testing framework
 require "#{File.dirname(__FILE__)}/../app_root/config/environment"
 require 'rspec/rails'
-
-FileUtils.rm(Dir.glob("#{Rails.root}/db/*.db"), :force => true)
+DatabaseCleaner.strategy = :truncation
 
 # Run the migrations
-print "\033[30m" # dark gray text
-ActiveRecord::Migrator.migrate("#{Rails.root}/db/migrate")
-print "\033[0m"
+Dusen::Util.migrate_test_database
 
 RSpec.configure do |config|
-  config.use_transactional_fixtures = true
+  config.use_transactional_fixtures = false
   config.use_instantiated_fixtures  = false
+  config.before(:each) do
+    DatabaseCleaner.clean
+  end
 end

data/spec/rails-3.2/Gemfile

@@ -3,9 +3,10 @@ source :rubygems
 gem 'rails', '~>3.2.0'
 gem 'rspec'
 gem 'rspec-rails'
-gem 'sqlite3'
+gem 'mysql2'
 gem 'rspec_candy'
 gem 'ruby-debug', :platforms => :ruby_18
 gem 'database_cleaner'
+gem 'andand'
 
 gem 'dusen', :path => '../..'

data/spec/rails-3.2/Rakefile

@@ -6,6 +6,6 @@ task :default => :spec
 
 desc "Run all specs for a specific rails version"
 RSpec::Core::RakeTask.new(:spec) do |t|
-  t.pattern = defined?(SPEC) ? SPEC : ['**/*_spec.rb', '../shared/**/*_spec.rb']
+  t.pattern = defined?(SPEC) ? SPEC : ['**/*_spec.rb', '../shared/spec/**/*_spec.rb']
   t.verbose = false
 end

data/spec/rails-3.2/app_root/config/database.yml

@@ -1,4 +1,6 @@
 test:
-  adapter: sqlite3
-  database: db/gem_test.db
-  verbosity: quiet
+  adapter: mysql2
+  host: localhost
+  username: root
+  password: junior
+  database: dusen_test

data/spec/rails-3.2/spec/spec_helper.rb

@@ -5,19 +5,15 @@ $: << File.join(File.dirname(__FILE__), "/../../lib" )
 ENV['RAILS_ENV'] = 'test'
 ENV['RAILS_ROOT'] = 'app_root'
 
-FileUtils.rm(Dir.glob("#{ENV['RAILS_ROOT']}/db/*.db"), :force => true)
-
 # Load the Rails environment and testing framework
 require "#{File.dirname(__FILE__)}/../app_root/config/environment"
 require 'rspec/rails'
-require 'database_cleaner'
-
 DatabaseCleaner.strategy = :truncation
 
+# Dir["#{File.dirname(__FILE__)}/../../shared/spec/shared_examples"].each {|f| require f}
+
 # Run the migrations
-print "\033[30m" # dark gray text
-ActiveRecord::Migrator.migrate("#{Rails.root}/db/migrate")
-print "\033[0m"
+Dusen::Util.migrate_test_database
 
 RSpec.configure do |config|
   config.use_transactional_fixtures = false

data/spec/shared/app_root/app/models/recipe/category.rb

@@ -0,0 +1,13 @@
+class Recipe::Category < ActiveRecord::Base
+
+  self.table_name = 'recipe_categories'
+
+  validates_presence_of :name
+
+  has_many :recipes, :inverse_of => :category
+
+  part_of_search_text_for do
+    recipes
+  end
+
+end

data/spec/shared/app_root/app/models/recipe/ingredient.rb

@@ -0,0 +1,13 @@
+class Recipe::Ingredient < ActiveRecord::Base
+
+  self.table_name = 'recipe_ingredients'
+
+  validates_presence_of :name
+
+  belongs_to :recipe, :inverse_of => :ingredients
+
+  part_of_search_text_for do
+    recipe
+  end
+
+end

data/spec/shared/app_root/app/models/recipe.rb

@@ -0,0 +1,14 @@
+class Recipe < ActiveRecord::Base
+
+  validates_presence_of :name
+
+  has_many :ingredients, :class_name => 'Recipe::Ingredient', :inverse_of => :recipe
+  belongs_to :category, :class_name => 'Recipe::Category', :inverse_of => :recipes
+
+  search_syntax
+
+  search_text do
+    [name, category.andand.name, ingredients.collect(&:name)]
+  end
+
+end

data/spec/shared/app_root/app/models/user/with_fulltext.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+
+module User
+  class WithFulltext < ActiveRecord::Base
+
+    self.table_name = 'users_with_fulltext'
+
+    search_syntax do
+
+      search_by :city do |scope, city|
+        scope.scoped(:conditions => { :city => city })
+      end
+
+      search_by :email do |scope, email|
+        scope.scoped(:conditions => { :email => email })
+      end
+
+    end
+
+    search_syntax do # multiple search_syntax directives are allowed
+
+      search_by :role do |scope, role|
+        scope.scoped(:conditions => { :role => role })
+      end
+
+    end
+
+    search_text do
+      [name, email, city]
+    end
+
+  end
+
+end
+

data/spec/shared/app_root/app/models/user/without_fulltext.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+module User
+  class WithoutFulltext < ActiveRecord::Base
+
+    self.table_name = 'users_without_fulltext'
+
+    search_syntax do
+
+      search_by :text do |scope, text|
+        scope.where_like([:name, :email, :city] => text)
+      end
+
+      search_by :city do |scope, city|
+        scope.scoped(:conditions => { :city => city })
+      end
+
+      search_by :email do |scope, email|
+        scope.scoped(:conditions => { :email => email })
+      end
+
+    end
+
+    search_syntax do # multiple search_syntax directives are allowed
+
+      search_by :role do |scope, role|
+        scope.scoped(:conditions => { :role => role })
+      end
+
+    end
+
+  end
+
+end

data/spec/shared/app_root/config/database.sample.yml

@@ -0,0 +1,6 @@
+test:
+  adapter: mysql2
+  host: localhost
+  username: root
+  password:
+  database: dusen_test

data/spec/shared/app_root/db/migrate/001_create_search_text.rb

@@ -0,0 +1,19 @@
+class CreateSearchText < ActiveRecord::Migration
+
+  def self.up
+    create_table :search_texts, :options => 'ENGINE=MyISAM' do |t|
+      t.integer :source_id
+      t.string  :source_type
+      t.boolean :stale
+      t.text    :words
+    end
+    add_index :search_texts, [:source_type, :source_id] # for updates
+    add_index :search_texts, [:source_type, :stale] # for refreshs
+    execute 'CREATE FULLTEXT INDEX fulltext_index_words ON search_texts (words)'
+  end
+
+  def self.down
+    drop_table :search_texts
+  end
+
+end

data/spec/shared/app_root/db/migrate/002_create_user_variants.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+
+class CreateUserVariants < ActiveRecord::Migration
+
+  def self.user_tables
+    [:users_with_fulltext, :users_without_fulltext]
+  end
+
+  def self.up
+    user_tables.each do |user_table|
+      create_table user_table do |t|
+        t.string :name
+        t.string :email
+        t.string :city
+      end
+    end
+  end
+
+  def self.down
+    user_tables.each do |user_table|
+      drop_table user_table
+    end
+  end
+
+end

data/spec/shared/app_root/db/migrate/003_create_recipe_models.rb

@@ -0,0 +1,23 @@
+class CreateRecipeModels < ActiveRecord::Migration
+
+  def self.up
+    create_table :recipes do |t|
+      t.string :name
+      t.integer :category_id
+    end
+    create_table :recipe_ingredients do |t|
+      t.string :name
+      t.integer :recipe_id
+    end
+    create_table :recipe_categories do |t|
+      t.string :name
+    end
+  end
+
+  def self.down
+    drop_table :recipes
+    drop_table :recipe_ingredients
+    drop_table :recipe_categories
+  end
+
+end