texticle 1.0.3 → 1.0.4.20101004123327
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.rdoc +13 -0
- data/README.rdoc +24 -1
- data/Rakefile +1 -0
- data/lib/texticle.rb +46 -12
- data/lib/texticle/full_text_index.rb +9 -5
- data/lib/texticle/tasks.rb +50 -19
- data/test/helper.rb +1 -0
- data/test/test_full_text_index.rb +4 -4
- data/test/test_texticle.rb +18 -2
- metadata +8 -7
data/CHANGELOG.rdoc
CHANGED
@@ -1,3 +1,16 @@
|
|
1
|
+
=== 1.0.4 / 2010-08-19
|
2
|
+
|
3
|
+
* 2 major enhancements
|
4
|
+
|
5
|
+
* use Rails.root instead of RAILS_ROOT
|
6
|
+
* refactored tasks to ease maintainance and patchability
|
7
|
+
|
8
|
+
* 3 minor enhancements
|
9
|
+
|
10
|
+
* fix timestamp for migrationfile
|
11
|
+
* fixed deprecation warning for rails3 (dropping rails2-support)
|
12
|
+
* prevented warning about defined constant
|
13
|
+
|
1
14
|
=== 1.0.3 / 2010-07-07
|
2
15
|
|
3
16
|
* 1 major enhancement
|
data/README.rdoc
CHANGED
@@ -18,7 +18,7 @@ named_scope methods making searching easy and fun!
|
|
18
18
|
|
19
19
|
In the project's Gemfile add
|
20
20
|
|
21
|
-
gem 'texticle', '1.0.
|
21
|
+
gem 'texticle', '1.0.4'
|
22
22
|
|
23
23
|
=== Rails 2 Configuration
|
24
24
|
|
@@ -54,6 +54,29 @@ Alternatively, you can create a migration that will build your indexes:
|
|
54
54
|
|
55
55
|
$ rake texticle:migration
|
56
56
|
|
57
|
+
== BETA: FUZZY SEARCH
|
58
|
+
|
59
|
+
We're adding support for Postgres' optional trigram matching module, which allows for fuzzier text searches.
|
60
|
+
|
61
|
+
Since it's an optional module, first step is to install it into the current database:
|
62
|
+
|
63
|
+
$ rake texticle:install_trigram
|
64
|
+
|
65
|
+
If you installed Postgres with its optional 'contrib' modules, you should be safe.
|
66
|
+
|
67
|
+
Declare your indexes as you normally would:
|
68
|
+
|
69
|
+
class Book < ActiveRecord::Base
|
70
|
+
index { title }
|
71
|
+
end
|
72
|
+
|
73
|
+
Finally, say hello to #tsearch:
|
74
|
+
|
75
|
+
Book.create :title => "_why's Poignant Guide to Ruby"
|
76
|
+
Book.tsearch "_whys Poignnt Gd t Rb" # => [#<Book title: "_why's Poignant Guide to Ruby">]
|
77
|
+
|
78
|
+
Trigram searching is still in beta, so expect the API to change and/or improve.
|
79
|
+
|
57
80
|
== REQUIREMENTS:
|
58
81
|
|
59
82
|
* Texticle may be used outside rails, but works best with rails.
|
data/Rakefile
CHANGED
data/lib/texticle.rb
CHANGED
@@ -49,7 +49,7 @@ require 'texticle/railtie' if defined?(Rails) and Rails::VERSION::MAJOR > 2
|
|
49
49
|
# end
|
50
50
|
module Texticle
|
51
51
|
# The version of Texticle you are using.
|
52
|
-
VERSION = '1.0.
|
52
|
+
VERSION = '1.0.4' unless defined?(Texticle::VERSION)
|
53
53
|
|
54
54
|
# A list of full text indexes
|
55
55
|
attr_accessor :full_text_indexes
|
@@ -59,19 +59,53 @@ module Texticle
|
|
59
59
|
def index name = nil, dictionary = 'english', &block
|
60
60
|
search_name = ['search', name].compact.join('_')
|
61
61
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
62
|
+
scope_lamba = lambda { |term|
|
63
|
+
# Let's extract the individual terms to allow for quoted and wildcard terms.
|
64
|
+
term = term.scan(/"([^"]+)"|(\S+)/).flatten.compact.map do |lex|
|
65
|
+
lex =~ /(.+)\*\s*$/ ? "'#{$1}':*" : "'#{lex}'"
|
66
|
+
end.join(' & ')
|
67
|
+
|
68
|
+
{
|
69
|
+
:select => "#{table_name}.*, ts_rank_cd((#{full_text_indexes.first.to_s}),
|
70
|
+
to_tsquery(#{connection.quote(term)})) as rank",
|
71
|
+
:conditions =>
|
72
|
+
["#{full_text_indexes.first.to_s} @@ to_tsquery(?)", term],
|
73
|
+
:order => 'rank DESC'
|
74
|
+
}
|
75
|
+
}
|
76
|
+
|
77
|
+
# tsearch, i.e. trigram search
|
78
|
+
trigram_scope_lambda = lambda { |term|
|
79
|
+
term = "'#{term.gsub("'", "''")}'"
|
80
|
+
|
81
|
+
similarities = full_text_indexes.first.index_columns.values.flatten.inject([]) do |array, index|
|
82
|
+
array << "similarity(#{index}, #{term})"
|
83
|
+
end.join(" + ")
|
84
|
+
|
85
|
+
conditions = full_text_indexes.first.index_columns.values.flatten.inject([]) do |array, index|
|
86
|
+
array << "(#{index} % #{term})"
|
87
|
+
end.join(" OR ")
|
88
|
+
|
89
|
+
{
|
90
|
+
:select => "#{table_name}.*, #{similarities} as rank",
|
91
|
+
:conditions => conditions,
|
92
|
+
:order => 'rank DESC'
|
73
93
|
}
|
94
|
+
}
|
95
|
+
|
96
|
+
class_eval do
|
97
|
+
# Trying to avoid the deprecation warning when using :named_scope
|
98
|
+
# that Rails 3 emits. Can't use #respond_to?(:scope) since scope
|
99
|
+
# is a protected method in Rails 2, and thus still returns true.
|
100
|
+
if self.respond_to?(:scope) and not protected_methods.include?('scope')
|
101
|
+
scope search_name.to_sym, scope_lamba
|
102
|
+
scope ('t' + search_name).to_sym, trigram_scope_lambda
|
103
|
+
elsif self.respond_to? :named_scope
|
104
|
+
named_scope search_name.to_sym, scope_lamba
|
105
|
+
named_scope ('t' + search_name).to_sym, trigram_scope_lambda
|
106
|
+
end
|
74
107
|
end
|
108
|
+
|
75
109
|
index_name = [table_name, name, 'fts_idx'].compact.join('_')
|
76
110
|
(self.full_text_indexes ||= []) <<
|
77
111
|
FullTextIndex.new(index_name, dictionary, self, &block)
|
@@ -11,6 +11,10 @@ module Texticle
|
|
11
11
|
instance_eval(&block)
|
12
12
|
end
|
13
13
|
|
14
|
+
def self.find_constant_of(filename)
|
15
|
+
File.basename(filename, '.rb').pluralize.classify.constantize
|
16
|
+
end
|
17
|
+
|
14
18
|
def create
|
15
19
|
@model_class.connection.execute create_sql
|
16
20
|
end
|
@@ -20,10 +24,10 @@ module Texticle
|
|
20
24
|
end
|
21
25
|
|
22
26
|
def create_sql
|
23
|
-
<<-eosql
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
+
<<-eosql.chomp
|
28
|
+
CREATE index #{@name}
|
29
|
+
ON #{@model_class.table_name}
|
30
|
+
USING gin((#{to_s}))
|
27
31
|
eosql
|
28
32
|
end
|
29
33
|
|
@@ -35,7 +39,7 @@ module Texticle
|
|
35
39
|
return @string if @string
|
36
40
|
vectors = []
|
37
41
|
@index_columns.sort_by { |k,v| k }.each do |weight, columns|
|
38
|
-
c = columns.map { |x| "coalesce(#{@model_class.table_name}
|
42
|
+
c = columns.map { |x| "coalesce(\"#{@model_class.table_name}\".\"#{x}\", '')" }
|
39
43
|
if weight == 'none'
|
40
44
|
vectors << "to_tsvector('#{@dictionary}', #{c.join(" || ' ' || ")})"
|
41
45
|
else
|
data/lib/texticle/tasks.rb
CHANGED
@@ -5,34 +5,39 @@ namespace :texticle do
|
|
5
5
|
desc "Create full text index migration"
|
6
6
|
task :migration => :environment do
|
7
7
|
now = Time.now.utc
|
8
|
-
filename = "#{now.strftime('%Y%m%d%H%
|
9
|
-
File.open(
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
8
|
+
filename = "#{now.strftime('%Y%m%d%H%M%S')}_full_text_search_#{now.to_i}.rb"
|
9
|
+
File.open(Rails.root + 'db' + 'migrate' + filename, 'wb') do |fh|
|
10
|
+
up_sql_statements = []
|
11
|
+
dn_sql_statements = []
|
12
|
+
|
13
|
+
Dir[Rails.root + 'app' + 'models' + '*.rb'].each do |f|
|
14
|
+
klass = Texticle::FullTextIndex.find_constant_of(f)
|
14
15
|
if klass.respond_to?(:full_text_indexes)
|
15
16
|
(klass.full_text_indexes || []).each do |fti|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
eosql
|
20
|
-
ActiveRecord::Base.connection.execute(<<-'eosql')
|
21
|
-
#{fti.create_sql}
|
22
|
-
eosql
|
23
|
-
eostmt
|
17
|
+
up_sql_statements << fti.destroy_sql
|
18
|
+
up_sql_statements << fti.create_sql
|
19
|
+
dn_sql_statements << fti.destroy_sql
|
24
20
|
end
|
25
21
|
end
|
26
22
|
end
|
23
|
+
|
24
|
+
fh.puts "class FullTextSearch#{now.to_i} < ActiveRecord::Migration"
|
25
|
+
fh.puts " def self.up"
|
26
|
+
insert_sql_statements_into_migration_file(up_sql_statements, fh)
|
27
|
+
fh.puts " end"
|
28
|
+
fh.puts ""
|
29
|
+
|
30
|
+
fh.puts " def self.down"
|
31
|
+
insert_sql_statements_into_migration_file(dn_sql_statements, fh)
|
27
32
|
fh.puts " end"
|
28
33
|
fh.puts "end"
|
29
|
-
|
34
|
+
end
|
30
35
|
end
|
31
36
|
|
32
37
|
desc "Create full text indexes"
|
33
38
|
task :create_indexes => ['texticle:destroy_indexes'] do
|
34
|
-
Dir[
|
35
|
-
klass =
|
39
|
+
Dir[Rails.root + 'app' + 'models' + '*.rb'].each do |f|
|
40
|
+
klass = Texticle::FullTextIndex.find_constant_of(f)
|
36
41
|
if klass.respond_to?(:full_text_indexes)
|
37
42
|
(klass.full_text_indexes || []).each do |fti|
|
38
43
|
begin
|
@@ -47,8 +52,8 @@ namespace :texticle do
|
|
47
52
|
|
48
53
|
desc "Destroy full text indexes"
|
49
54
|
task :destroy_indexes => [:environment] do
|
50
|
-
Dir[
|
51
|
-
klass =
|
55
|
+
Dir[Rails.root + 'app' + 'models' + '*.rb'].each do |f|
|
56
|
+
klass = Texticle::FullTextIndex.find_constant_of(f)
|
52
57
|
if klass.respond_to?(:full_text_indexes)
|
53
58
|
(klass.full_text_indexes || []).each do |fti|
|
54
59
|
fti.destroy
|
@@ -56,4 +61,30 @@ namespace :texticle do
|
|
56
61
|
end
|
57
62
|
end
|
58
63
|
end
|
64
|
+
|
65
|
+
desc "Install trigram text search module"
|
66
|
+
task :install_trigram => [:environment] do
|
67
|
+
share_dir = `pg_config --sharedir`.chomp
|
68
|
+
raise RuntimeError, 'cannot find Postgres\' shared directory' unless $?.exitstatus.zero?
|
69
|
+
trigram = "#{share_dir}/contrib/pg_trgm.sql"
|
70
|
+
unless system("ls #{trigram}")
|
71
|
+
raise RuntimeError, 'cannot find trigram module; was it compiled and installed?'
|
72
|
+
end
|
73
|
+
db_name = ActiveRecord::Base.connection.current_database
|
74
|
+
unless system("psql -d #{db_name} -f #{trigram}")
|
75
|
+
raise RuntimeError, "`psql -d #{db_name} -f #{trigram}` cannot complete successfully"
|
76
|
+
end
|
77
|
+
puts "Trigram text search module successfully installed into '#{db_name}' database."
|
78
|
+
end
|
79
|
+
|
80
|
+
def insert_sql_statements_into_migration_file statements, fh
|
81
|
+
statements.each do |statement|
|
82
|
+
fh.puts <<-eostmt
|
83
|
+
execute(<<-'eosql'.strip)
|
84
|
+
#{statement}
|
85
|
+
eosql
|
86
|
+
eostmt
|
87
|
+
end
|
88
|
+
end
|
59
89
|
end
|
90
|
+
|
data/test/helper.rb
CHANGED
@@ -45,14 +45,14 @@ class TestFullTextIndex < TexticleTestCase
|
|
45
45
|
fti = Texticle::FullTextIndex.new('ft_index', 'english', @fm) do
|
46
46
|
name
|
47
47
|
end
|
48
|
-
assert_equal "to_tsvector('english', coalesce(#{@fm.table_name}
|
48
|
+
assert_equal "to_tsvector('english', coalesce(\"#{@fm.table_name}\".\"name\", ''))", fti.to_s
|
49
49
|
end
|
50
50
|
|
51
51
|
def test_to_s_A_weight
|
52
52
|
fti = Texticle::FullTextIndex.new('ft_index', 'english', @fm) do
|
53
53
|
name 'A'
|
54
54
|
end
|
55
|
-
assert_equal "setweight(to_tsvector('english', coalesce(#{@fm.table_name}
|
55
|
+
assert_equal "setweight(to_tsvector('english', coalesce(\"#{@fm.table_name}\".\"name\", '')), 'A')", fti.to_s
|
56
56
|
end
|
57
57
|
|
58
58
|
def test_to_s_multi_weight
|
@@ -61,7 +61,7 @@ class TestFullTextIndex < TexticleTestCase
|
|
61
61
|
value 'A'
|
62
62
|
description 'B'
|
63
63
|
end
|
64
|
-
assert_equal "setweight(to_tsvector('english', coalesce(#{@fm.table_name}
|
64
|
+
assert_equal "setweight(to_tsvector('english', coalesce(\"#{@fm.table_name}\".\"name\", '') || ' ' || coalesce(\"#{@fm.table_name}\".\"value\", '')), 'A') || ' ' || setweight(to_tsvector('english', coalesce(\"#{@fm.table_name}\".\"description\", '')), 'B')", fti.to_s
|
65
65
|
end
|
66
66
|
|
67
67
|
def test_mixed_weight
|
@@ -69,6 +69,6 @@ class TestFullTextIndex < TexticleTestCase
|
|
69
69
|
name
|
70
70
|
value 'A'
|
71
71
|
end
|
72
|
-
assert_equal "setweight(to_tsvector('english', coalesce(#{@fm.table_name}
|
72
|
+
assert_equal "setweight(to_tsvector('english', coalesce(\"#{@fm.table_name}\".\"value\", '')), 'A') || ' ' || to_tsvector('english', coalesce(\"#{@fm.table_name}\".\"name\", ''))", fti.to_s
|
73
73
|
end
|
74
74
|
end
|
data/test/test_texticle.rb
CHANGED
@@ -10,11 +10,13 @@ class TestTexticle < TexticleTestCase
|
|
10
10
|
end
|
11
11
|
end
|
12
12
|
assert_equal 1, x.full_text_indexes.length
|
13
|
-
|
13
|
+
# One named_scope for search, another for trigram search
|
14
|
+
assert_equal 2, x.named_scopes.length
|
14
15
|
|
15
16
|
x.full_text_indexes.first.create
|
16
17
|
assert_match "#{x.table_name}_fts_idx", x.executed.first
|
17
18
|
assert_equal :search, x.named_scopes.first.first
|
19
|
+
assert_equal :tsearch, x.named_scopes[1].first
|
18
20
|
end
|
19
21
|
|
20
22
|
def test_named_index
|
@@ -26,11 +28,12 @@ class TestTexticle < TexticleTestCase
|
|
26
28
|
end
|
27
29
|
end
|
28
30
|
assert_equal 1, x.full_text_indexes.length
|
29
|
-
assert_equal
|
31
|
+
assert_equal 2, x.named_scopes.length
|
30
32
|
|
31
33
|
x.full_text_indexes.first.create
|
32
34
|
assert_match "#{x.table_name}_awesome_fts_idx", x.executed.first
|
33
35
|
assert_equal :search_awesome, x.named_scopes.first.first
|
36
|
+
assert_equal :tsearch_awesome, x.named_scopes[1].first
|
34
37
|
end
|
35
38
|
|
36
39
|
def test_named_scope_select
|
@@ -57,4 +60,17 @@ class TestTexticle < TexticleTestCase
|
|
57
60
|
ns = x.named_scopes.first[1].call('foo bar "foo bar"')
|
58
61
|
assert_match(/'foo' & 'bar' & 'foo bar'/, ns[:select])
|
59
62
|
end
|
63
|
+
|
64
|
+
def test_wildcard_queries
|
65
|
+
x = fake_model
|
66
|
+
x.class_eval do
|
67
|
+
extend Texticle
|
68
|
+
index('awesome') do
|
69
|
+
name
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
ns = x.named_scopes.first[1].call('foo bar*')
|
74
|
+
assert_match(/'foo' & 'bar:*'/, ns[:select])
|
75
|
+
end
|
60
76
|
end
|
metadata
CHANGED
@@ -1,13 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: texticle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 40202008246577
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 1
|
8
8
|
- 0
|
9
|
-
-
|
10
|
-
|
9
|
+
- 4
|
10
|
+
- 20101004123327
|
11
|
+
version: 1.0.4.20101004123327
|
11
12
|
platform: ruby
|
12
13
|
authors:
|
13
14
|
- Aaron Patterson
|
@@ -15,7 +16,7 @@ autorequire:
|
|
15
16
|
bindir: bin
|
16
17
|
cert_chain: []
|
17
18
|
|
18
|
-
date: 2010-
|
19
|
+
date: 2010-10-04 00:00:00 -04:00
|
19
20
|
default_executable:
|
20
21
|
dependencies:
|
21
22
|
- !ruby/object:Gem::Dependency
|
@@ -42,12 +43,12 @@ dependencies:
|
|
42
43
|
requirements:
|
43
44
|
- - ">="
|
44
45
|
- !ruby/object:Gem::Version
|
45
|
-
hash:
|
46
|
+
hash: 19
|
46
47
|
segments:
|
47
48
|
- 2
|
48
49
|
- 6
|
49
|
-
-
|
50
|
-
version: 2.6.
|
50
|
+
- 2
|
51
|
+
version: 2.6.2
|
51
52
|
type: :development
|
52
53
|
version_requirements: *id002
|
53
54
|
description: |-
|