pg_textsearch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.devcontainer/devcontainer.json +32 -0
- data/.devcontainer/setup.sh +25 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +514 -0
- data/Rakefile +23 -0
- data/lib/pg_textsearch/maintenance.rb +133 -0
- data/lib/pg_textsearch/model.rb +172 -0
- data/lib/pg_textsearch/schema.rb +53 -0
- data/lib/pg_textsearch/version.rb +5 -0
- data/lib/pg_textsearch.rb +30 -0
- data/samples/01_basic_search.rb +113 -0
- data/samples/02_advanced_query_options.rb +106 -0
- data/samples/03_highlighting.rb +100 -0
- data/samples/04_custom_ranking.rb +110 -0
- data/samples/05_fuzzy_synonyms.rb +112 -0
- data/samples/06_maintenance.rb +111 -0
- data/samples/07_extension_setup.rb +89 -0
- data/sig/pg_textsearch.rbs +4 -0
- data/test/integration_test.rb +125 -0
- data/test/pg_textsearch_test.rb +162 -0
- metadata +107 -0
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
# Highlighting with ts_headline
|
|
4
|
+
# Run: ruby samples/03_highlighting.rb
|
|
5
|
+
|
|
6
|
+
require "bundler/inline"
|
|
7
|
+
|
|
8
|
+
gemfile do
|
|
9
|
+
source "https://rubygems.org"
|
|
10
|
+
gem "activerecord", ">= 7.0"
|
|
11
|
+
gem "pg", ">= 1.5"
|
|
12
|
+
gem "pg_textsearch", path: File.expand_path("..", __dir__)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
require "active_record"
|
|
16
|
+
require "pg_textsearch"
|
|
17
|
+
require "logger"
|
|
18
|
+
|
|
19
|
+
DB_CONFIG = {
|
|
20
|
+
adapter: "postgresql",
|
|
21
|
+
database: ENV.fetch("PG_DATABASE", "pg_textsearch_test"),
|
|
22
|
+
username: ENV.fetch("PG_USER", "postgres"),
|
|
23
|
+
password: ENV.fetch("PG_PASSWORD", "postgres"),
|
|
24
|
+
host: ENV.fetch("PG_HOST", "localhost"),
|
|
25
|
+
port: ENV.fetch("PG_PORT", 5432)
|
|
26
|
+
}.freeze
|
|
27
|
+
|
|
28
|
+
ActiveRecord::Base.establish_connection(DB_CONFIG)
|
|
29
|
+
|
|
30
|
+
def setup_schema
|
|
31
|
+
conn = ActiveRecord::Base.connection
|
|
32
|
+
conn.execute("CREATE EXTENSION IF NOT EXISTS pg_textsearch")
|
|
33
|
+
conn.execute("DROP TABLE IF EXISTS products CASCADE")
|
|
34
|
+
conn.execute(<<~SQL)
|
|
35
|
+
CREATE TABLE products (
|
|
36
|
+
id BIGSERIAL PRIMARY KEY,
|
|
37
|
+
name TEXT NOT NULL,
|
|
38
|
+
description TEXT,
|
|
39
|
+
search_text TEXT
|
|
40
|
+
)
|
|
41
|
+
SQL
|
|
42
|
+
conn.execute(<<~SQL)
|
|
43
|
+
CREATE INDEX products_search_idx ON products
|
|
44
|
+
USING bm25 (search_text)
|
|
45
|
+
WITH (text_config = 'english', k1 = 1.5, b = 0.75);
|
|
46
|
+
SQL
|
|
47
|
+
puts "ā Schema created"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def seed_data
|
|
51
|
+
Product.delete_all
|
|
52
|
+
products = [
|
|
53
|
+
{ name: "Sony WH-1000XM5", description: "Industry-leading noise cancellation with Auto NC Optimizer. 30-hour battery life. Multipoint connection." },
|
|
54
|
+
{ name: "Bose QuietComfort Ultra", description: "World-class noise cancellation. Spatial audio. 24-hour battery. Premium comfort." },
|
|
55
|
+
{ name: "Apple AirPods Pro", description: "Active Noise Cancellation. Transparency mode. Adaptive Audio. USB-C charging." },
|
|
56
|
+
{ name: "Sennheiser Momentum 4", description: "Audiophile sound. 60-hour battery. Adaptive noise cancellation. Premium build." },
|
|
57
|
+
{ name: "Sony WF-1000XM5", description: "Best-in-class noise cancellation in earbud form. 8-hour battery. LDAC support." }
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
products.each do |p|
|
|
61
|
+
p[:search_text] = [p[:name], p[:description]].compact.join(" ")
|
|
62
|
+
Product.create!(p)
|
|
63
|
+
end
|
|
64
|
+
puts "ā Seeded #{Product.count} products"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
class Product < ActiveRecord::Base
|
|
68
|
+
has_bm25 :search_text, index: :products_search_idx
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def run
|
|
72
|
+
setup_schema
|
|
73
|
+
seed_data
|
|
74
|
+
|
|
75
|
+
puts "\n=== Basic Highlighting ==="
|
|
76
|
+
results = Product.bm25_search(:search_text, "wireless", index: :products_search_idx,
|
|
77
|
+
highlight: true).limit(3)
|
|
78
|
+
results.each do |p|
|
|
79
|
+
puts " #{p.name}"
|
|
80
|
+
puts " #{p.pg_search_highlight&.truncate(120)}"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
puts "\n=== Custom Highlight Options ==="
|
|
84
|
+
results = Product.bm25_search(:search_text, "noise cancellation", index: :products_search_idx,
|
|
85
|
+
highlight: {
|
|
86
|
+
start_sel: "<mark>",
|
|
87
|
+
stop_sel: "</mark>",
|
|
88
|
+
max_words: 30,
|
|
89
|
+
min_words: 10,
|
|
90
|
+
fragment_delimiter: " ... "
|
|
91
|
+
}).limit(3)
|
|
92
|
+
results.each do |p|
|
|
93
|
+
puts " #{p.name}"
|
|
94
|
+
puts " #{p.pg_search_highlight&.truncate(120)}"
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
puts "\nā
Highlighting examples completed!"
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
run if $PROGRAM_NAME == __FILE__
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
# Custom ranking with ranked_by and order_within_rank
|
|
4
|
+
# Run: ruby samples/04_custom_ranking.rb
|
|
5
|
+
|
|
6
|
+
require "bundler/inline"
|
|
7
|
+
|
|
8
|
+
gemfile do
|
|
9
|
+
source "https://rubygems.org"
|
|
10
|
+
gem "activerecord", ">= 7.0"
|
|
11
|
+
gem "pg", ">= 1.5"
|
|
12
|
+
gem "pg_textsearch", path: File.expand_path("..", __dir__)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
require "active_record"
|
|
16
|
+
require "pg_textsearch"
|
|
17
|
+
require "logger"
|
|
18
|
+
|
|
19
|
+
DB_CONFIG = {
|
|
20
|
+
adapter: "postgresql",
|
|
21
|
+
database: ENV.fetch("PG_DATABASE", "pg_textsearch_test"),
|
|
22
|
+
username: ENV.fetch("PG_USER", "postgres"),
|
|
23
|
+
password: ENV.fetch("PG_PASSWORD", "postgres"),
|
|
24
|
+
host: ENV.fetch("PG_HOST", "localhost"),
|
|
25
|
+
port: ENV.fetch("PG_PORT", 5432)
|
|
26
|
+
}.freeze
|
|
27
|
+
|
|
28
|
+
ActiveRecord::Base.establish_connection(DB_CONFIG)
|
|
29
|
+
|
|
30
|
+
def setup_schema
|
|
31
|
+
conn = ActiveRecord::Base.connection
|
|
32
|
+
conn.execute("CREATE EXTENSION IF NOT EXISTS pg_textsearch")
|
|
33
|
+
conn.execute("DROP TABLE IF EXISTS products CASCADE")
|
|
34
|
+
conn.execute(<<~SQL)
|
|
35
|
+
CREATE TABLE products (
|
|
36
|
+
id BIGSERIAL PRIMARY KEY,
|
|
37
|
+
name TEXT NOT NULL,
|
|
38
|
+
description TEXT,
|
|
39
|
+
search_text TEXT,
|
|
40
|
+
popularity INT DEFAULT 0, -- e.g., view count
|
|
41
|
+
rating NUMERIC(2,1) DEFAULT 0,
|
|
42
|
+
updated_at TIMESTAMPTZ DEFAULT now()
|
|
43
|
+
)
|
|
44
|
+
SQL
|
|
45
|
+
conn.execute(<<~SQL)
|
|
46
|
+
CREATE INDEX products_search_idx ON products
|
|
47
|
+
USING bm25 (search_text)
|
|
48
|
+
WITH (text_config = 'english', k1 = 1.5, b = 0.75);
|
|
49
|
+
SQL
|
|
50
|
+
puts "ā Schema created"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def seed_data
|
|
54
|
+
Product.delete_all
|
|
55
|
+
products = [
|
|
56
|
+
{ name: "Sony WH-1000XM5", description: "Industry-leading noise cancellation", popularity: 10000, rating: 4.8 },
|
|
57
|
+
{ name: "Bose QuietComfort Ultra", description: "World-class noise cancellation", popularity: 8000, rating: 4.7 },
|
|
58
|
+
{ name: "Apple AirPods Pro", description: "Active Noise Cancellation", popularity: 15000, rating: 4.9 },
|
|
59
|
+
{ name: "Sennheiser Momentum 4", description: "Audiophile sound", popularity: 3000, rating: 4.6 },
|
|
60
|
+
{ name: "Sony WF-1000XM5", description: "Best-in-class noise cancellation earbuds", popularity: 7000, rating: 4.7 },
|
|
61
|
+
{ name: "Jabra Elite 10", description: "Dolby Atmos spatial audio", popularity: 2000, rating: 4.5 },
|
|
62
|
+
{ name: "Sony WH-CH720N", description: "Lightweight noise cancellation", popularity: 5000, rating: 4.3 },
|
|
63
|
+
{ name: "Bose QuietComfort 45", description: "Proven noise cancellation", popularity: 6000, rating: 4.6 },
|
|
64
|
+
{ name: "Sennheiser Accentum", description: "Great value ANC", popularity: 1500, rating: 4.4 },
|
|
65
|
+
{ name: "Nothing Ear (2)", description: "Transparent design ANC", popularity: 4000, rating: 4.2 }
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
products.each_with_index do |p, i|
|
|
69
|
+
p[:search_text] = [p[:name], p[:description]].compact.join(" ")
|
|
70
|
+
p[:updated_at] = Time.now - (i * 86400) # different update times
|
|
71
|
+
Product.create!(p)
|
|
72
|
+
end
|
|
73
|
+
puts "ā Seeded #{Product.count} products"
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
class Product < ActiveRecord::Base
|
|
77
|
+
has_bm25 :search_text, index: :products_search_idx
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def run
|
|
81
|
+
setup_schema
|
|
82
|
+
seed_data
|
|
83
|
+
|
|
84
|
+
puts "\n=== Default BM25 Ranking ==="
|
|
85
|
+
results = Product.bm25_search(:search_text, "headphones", index: :products_search_idx, select_score: true).limit(5)
|
|
86
|
+
results.each { |p| puts " #{p.name}: score=#{p.bm25_score&.round(3)}, pop=#{p.popularity}" }
|
|
87
|
+
|
|
88
|
+
puts "\n=== Custom Ranking: BM25 + Popularity Boost ==="
|
|
89
|
+
# :bm25 is replaced with the actual BM25 score expression
|
|
90
|
+
results = Product.bm25_search(:search_text, "headphones", index: :products_search_idx,
|
|
91
|
+
ranked_by: ":bm25 + 0.0001 * products.popularity",
|
|
92
|
+
select_score: true).limit(5)
|
|
93
|
+
results.each { |p| puts " #{p.name}: score=#{p.bm25_score&.round(3)}, pop=#{p.popularity}" }
|
|
94
|
+
|
|
95
|
+
puts "\n=== Custom Ranking: BM25 + Rating Boost ==="
|
|
96
|
+
results = Product.bm25_search(:search_text, "headphones", index: :products_search_idx,
|
|
97
|
+
ranked_by: ":bm25 + 0.5 * products.rating",
|
|
98
|
+
select_score: true).limit(5)
|
|
99
|
+
results.each { |p| puts " #{p.name}: score=#{p.bm25_score&.round(3)}, rating=#{p.rating}" }
|
|
100
|
+
|
|
101
|
+
puts "\n=== Tiebreaker: Most Recently Updated First ==="
|
|
102
|
+
results = Product.bm25_search(:search_text, "noise cancellation", index: :products_search_idx,
|
|
103
|
+
order_within_rank: "products.updated_at DESC",
|
|
104
|
+
select_score: true).limit(5)
|
|
105
|
+
results.each { |p| puts " #{p.name}: updated=#{p.updated_at}" }
|
|
106
|
+
|
|
107
|
+
puts "\nā
Custom ranking examples completed!"
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
run if $PROGRAM_NAME == __FILE__
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
# Fuzzy search (pg_trgm) and Synonyms (thesaurus)
|
|
4
|
+
# Run: ruby samples/05_fuzzy_synonyms.rb
|
|
5
|
+
|
|
6
|
+
require "bundler/inline"
|
|
7
|
+
|
|
8
|
+
gemfile do
|
|
9
|
+
source "https://rubygems.org"
|
|
10
|
+
gem "activerecord", ">= 7.0"
|
|
11
|
+
gem "pg", ">= 1.5"
|
|
12
|
+
gem "pg_textsearch", path: File.expand_path("..", __dir__)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
require "active_record"
|
|
16
|
+
require "pg_textsearch"
|
|
17
|
+
require "logger"
|
|
18
|
+
|
|
19
|
+
DB_CONFIG = {
|
|
20
|
+
adapter: "postgresql",
|
|
21
|
+
database: ENV.fetch("PG_DATABASE", "pg_textsearch_test"),
|
|
22
|
+
username: ENV.fetch("PG_USER", "postgres"),
|
|
23
|
+
password: ENV.fetch("PG_PASSWORD", "postgres"),
|
|
24
|
+
host: ENV.fetch("PG_HOST", "localhost"),
|
|
25
|
+
port: ENV.fetch("PG_PORT", 5432)
|
|
26
|
+
}.freeze
|
|
27
|
+
|
|
28
|
+
ActiveRecord::Base.establish_connection(DB_CONFIG)
|
|
29
|
+
|
|
30
|
+
def setup_schema
|
|
31
|
+
conn = ActiveRecord::Base.connection
|
|
32
|
+
conn.execute("CREATE EXTENSION IF NOT EXISTS pg_textsearch")
|
|
33
|
+
conn.execute("DROP TABLE IF EXISTS products CASCADE")
|
|
34
|
+
conn.execute(<<~SQL)
|
|
35
|
+
CREATE TABLE products (
|
|
36
|
+
id BIGSERIAL PRIMARY KEY,
|
|
37
|
+
name TEXT NOT NULL,
|
|
38
|
+
description TEXT,
|
|
39
|
+
search_text TEXT
|
|
40
|
+
)
|
|
41
|
+
SQL
|
|
42
|
+
conn.execute(<<~SQL)
|
|
43
|
+
CREATE INDEX products_search_idx ON products
|
|
44
|
+
USING bm25 (search_text)
|
|
45
|
+
WITH (text_config = 'english', k1 = 1.5, b = 0.75);
|
|
46
|
+
SQL
|
|
47
|
+
puts "ā Schema created"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def seed_data
|
|
51
|
+
Product.delete_all
|
|
52
|
+
products = [
|
|
53
|
+
{ name: "Sony WH-1000XM5", description: "Industry-leading noise cancellation with Auto NC Optimizer" },
|
|
54
|
+
{ name: "Bose QuietComfort Ultra", description: "World-class noise cancellation. Spatial audio." },
|
|
55
|
+
{ name: "Apple AirPods Pro", description: "Active Noise Cancellation. Transparency mode." },
|
|
56
|
+
{ name: "Sennheiser Momentum 4", description: "Audiophile sound. 60-hour battery." },
|
|
57
|
+
{ name: "Sony WF-1000XM5", description: "Best-in-class noise cancellation in earbud form" },
|
|
58
|
+
{ name: "Jabra Elite 10", description: "Dolby Atmos spatial audio. Advanced ANC." },
|
|
59
|
+
{ name: "Sony WH-CH720N", description: "Lightweight noise cancellation. 35-hour battery." },
|
|
60
|
+
{ name: "Bose QuietComfort 45", description: "Proven noise cancellation. Comfortable fit." },
|
|
61
|
+
{ name: "Sennheiser Accentum", description: "Great value ANC. 50-hour battery." },
|
|
62
|
+
{ name: "Nothing Ear (2)", description: "Transparent design. ANC. 36-hour battery." }
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
products.each do |p|
|
|
66
|
+
p[:search_text] = [p[:name], p[:description]].compact.join(" ")
|
|
67
|
+
Product.create!(p)
|
|
68
|
+
end
|
|
69
|
+
puts "ā Seeded #{Product.count} products"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
class Product < ActiveRecord::Base
|
|
73
|
+
has_bm25 :search_text, index: :products_search_idx
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def run
|
|
77
|
+
setup_schema
|
|
78
|
+
seed_data
|
|
79
|
+
|
|
80
|
+
puts "\n=== Fuzzy Search (pg_trgm) ==="
|
|
81
|
+
if PgTextsearch::Maintenance.pg_trgm_available?
|
|
82
|
+
# Matches "headphon" -> "headphones" (typo tolerance via trigram similarity)
|
|
83
|
+
# Using search_text column which has the products_search_idx
|
|
84
|
+
results = Product.bm25_search(:search_text, "headphon", index: :products_search_idx,
|
|
85
|
+
fuzzy: {threshold: 0.3}).limit(5)
|
|
86
|
+
results.each { |p| puts " #{p.name}" }
|
|
87
|
+
|
|
88
|
+
puts "\n Fuzzy: 'zucini' -> 'zucchini' style matching"
|
|
89
|
+
results = Product.bm25_search(:search_text, "airpod", index: :products_search_idx,
|
|
90
|
+
fuzzy: {threshold: 0.3}).limit(5)
|
|
91
|
+
results.each { |p| puts " #{p.name}" }
|
|
92
|
+
else
|
|
93
|
+
puts " pg_trgm extension not installed"
|
|
94
|
+
puts " Run: PgTextsearch::Maintenance.enable_pg_trgm!"
|
|
95
|
+
puts " Then: CREATE EXTENSION pg_trgm;"
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
puts "\n=== Synonyms (Thesaurus Dictionary) ==="
|
|
99
|
+
if PgTextsearch::Maintenance.thesaurus_available?("thesaurus_en")
|
|
100
|
+
puts " thesaurus_en dictionary available"
|
|
101
|
+
puts " Search 'pop' -> would match 'soda' products"
|
|
102
|
+
puts " (Requires index created with thesaurus text_config)"
|
|
103
|
+
else
|
|
104
|
+
puts " thesaurus dictionary not installed"
|
|
105
|
+
puts " Run: PgTextsearch::Maintenance.enable_thesaurus!('thesaurus_en', 'thesaurus_en')"
|
|
106
|
+
puts " Requires thesaurus_en.ths file in \$SHAREDIR/tsearch_data/"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
puts "\nā
Fuzzy & synonyms examples completed!"
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
run if $PROGRAM_NAME == __FILE__
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
# Index maintenance operations
|
|
4
|
+
# Run: ruby samples/06_maintenance.rb
|
|
5
|
+
|
|
6
|
+
require "bundler/inline"
|
|
7
|
+
|
|
8
|
+
gemfile do
|
|
9
|
+
source "https://rubygems.org"
|
|
10
|
+
gem "activerecord", ">= 7.0"
|
|
11
|
+
gem "pg", ">= 1.5"
|
|
12
|
+
gem "pg_textsearch", path: File.expand_path("..", __dir__)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
require "active_record"
|
|
16
|
+
require "pg_textsearch"
|
|
17
|
+
require "logger"
|
|
18
|
+
|
|
19
|
+
DB_CONFIG = {
|
|
20
|
+
adapter: "postgresql",
|
|
21
|
+
database: ENV.fetch("PG_DATABASE", "pg_textsearch_test"),
|
|
22
|
+
username: ENV.fetch("PG_USER", "postgres"),
|
|
23
|
+
password: ENV.fetch("PG_PASSWORD", "postgres"),
|
|
24
|
+
host: ENV.fetch("PG_HOST", "localhost"),
|
|
25
|
+
port: ENV.fetch("PG_PORT", 5432)
|
|
26
|
+
}.freeze
|
|
27
|
+
|
|
28
|
+
ActiveRecord::Base.establish_connection(DB_CONFIG)
|
|
29
|
+
|
|
30
|
+
def setup_schema
|
|
31
|
+
conn = ActiveRecord::Base.connection
|
|
32
|
+
conn.execute("CREATE EXTENSION IF NOT EXISTS pg_textsearch")
|
|
33
|
+
conn.execute("DROP TABLE IF EXISTS products CASCADE")
|
|
34
|
+
conn.execute(<<~SQL)
|
|
35
|
+
CREATE TABLE products (
|
|
36
|
+
id BIGSERIAL PRIMARY KEY,
|
|
37
|
+
name TEXT NOT NULL,
|
|
38
|
+
description TEXT,
|
|
39
|
+
search_text TEXT
|
|
40
|
+
)
|
|
41
|
+
SQL
|
|
42
|
+
conn.execute(<<~SQL)
|
|
43
|
+
CREATE INDEX products_search_idx ON products
|
|
44
|
+
USING bm25 (search_text)
|
|
45
|
+
WITH (text_config = 'english', k1 = 1.5, b = 0.75);
|
|
46
|
+
SQL
|
|
47
|
+
conn.execute(<<~SQL)
|
|
48
|
+
CREATE INDEX products_name_idx ON products USING bm25 (name)
|
|
49
|
+
WITH (text_config = 'english');
|
|
50
|
+
SQL
|
|
51
|
+
conn.execute(<<~SQL)
|
|
52
|
+
CREATE INDEX products_desc_idx ON products USING bm25 (description)
|
|
53
|
+
WITH (text_config = 'english');
|
|
54
|
+
SQL
|
|
55
|
+
puts "ā Schema created"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def seed_data
|
|
59
|
+
Product.delete_all
|
|
60
|
+
100.times do |i|
|
|
61
|
+
Product.create!(
|
|
62
|
+
name: "Product #{i}",
|
|
63
|
+
description: "Description for product #{i} with some searchable content",
|
|
64
|
+
search_text: "Product #{i} Description for product #{i} with some searchable content"
|
|
65
|
+
)
|
|
66
|
+
end
|
|
67
|
+
puts "ā Seeded #{Product.count} products"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
class Product < ActiveRecord::Base
|
|
71
|
+
has_bm25 :search_text, index: :products_search_idx
|
|
72
|
+
has_bm25 :name, index: :products_name_idx
|
|
73
|
+
has_bm25 :description, index: :products_desc_idx
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def run
|
|
77
|
+
setup_schema
|
|
78
|
+
seed_data
|
|
79
|
+
|
|
80
|
+
indexes = %w[products_search_idx products_name_idx products_desc_idx]
|
|
81
|
+
|
|
82
|
+
puts "\n=== Check Compaction Status ==="
|
|
83
|
+
indexes.each do |idx|
|
|
84
|
+
needs = PgTextsearch::Maintenance.needs_compaction?(idx)
|
|
85
|
+
levels = PgTextsearch::Maintenance.level_counts(idx)
|
|
86
|
+
puts " #{idx}: needs_compaction=#{needs}, levels=#{levels}"
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
puts "\n=== Compact Step (Cron-Safe) ==="
|
|
90
|
+
indexes.each do |idx|
|
|
91
|
+
ran = PgTextsearch::Maintenance.compact_step(idx)
|
|
92
|
+
puts " #{idx}: #{ran ? 'ran step' : 'nothing to do'}"
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
puts "\n=== Full Compaction ==="
|
|
96
|
+
indexes.each do |idx|
|
|
97
|
+
puts " Compacting #{idx}..."
|
|
98
|
+
PgTextsearch::Maintenance.compact(idx)
|
|
99
|
+
puts " Done"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
puts "\n=== Spill Memtables (Post-Bulk-Load) ==="
|
|
103
|
+
indexes.each do |idx|
|
|
104
|
+
pages = PgTextsearch::Maintenance.spill_index(idx)
|
|
105
|
+
puts " #{idx}: spilled #{pages} pages"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
puts "\nā
Maintenance examples completed!"
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
run if $PROGRAM_NAME == __FILE__
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
# Enable optional PostgreSQL extensions for advanced features
|
|
4
|
+
# Run: ruby samples/07_extension_setup.rb
|
|
5
|
+
|
|
6
|
+
require "bundler/inline"
|
|
7
|
+
|
|
8
|
+
gemfile do
|
|
9
|
+
source "https://rubygems.org"
|
|
10
|
+
gem "activerecord", ">= 7.0"
|
|
11
|
+
gem "pg", ">= 1.5"
|
|
12
|
+
gem "pg_textsearch", path: File.expand_path("..", __dir__)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
require "active_record"
|
|
16
|
+
require "pg_textsearch"
|
|
17
|
+
require "logger"
|
|
18
|
+
|
|
19
|
+
DB_CONFIG = {
|
|
20
|
+
adapter: "postgresql",
|
|
21
|
+
database: ENV.fetch("PG_DATABASE", "pg_textsearch_test"),
|
|
22
|
+
username: ENV.fetch("PG_USER", "postgres"),
|
|
23
|
+
password: ENV.fetch("PG_PASSWORD", "postgres"),
|
|
24
|
+
host: ENV.fetch("PG_HOST", "localhost"),
|
|
25
|
+
port: ENV.fetch("PG_PORT", 5432)
|
|
26
|
+
}.freeze
|
|
27
|
+
|
|
28
|
+
ActiveRecord::Base.establish_connection(DB_CONFIG)
|
|
29
|
+
|
|
30
|
+
def run
|
|
31
|
+
puts "=== Extension Setup Helpers ===\n"
|
|
32
|
+
|
|
33
|
+
# unaccent: accent-insensitive search
|
|
34
|
+
puts "1. unaccent (accent-insensitive search)"
|
|
35
|
+
puts " Available: #{PgTextsearch::Maintenance.unaccent_available?}"
|
|
36
|
+
if !PgTextsearch::Maintenance.unaccent_available?
|
|
37
|
+
puts " To enable: PgTextsearch::Maintenance.enable_unaccent!"
|
|
38
|
+
puts " This creates the extension and an IMMUTABLE wrapper function"
|
|
39
|
+
end
|
|
40
|
+
puts
|
|
41
|
+
|
|
42
|
+
# pg_trgm: fuzzy/prefix matching
|
|
43
|
+
puts "2. pg_trgm (fuzzy/prefix matching)"
|
|
44
|
+
puts " Available: #{PgTextsearch::Maintenance.pg_trgm_available?}"
|
|
45
|
+
if !PgTextsearch::Maintenance.pg_trgm_available?
|
|
46
|
+
puts " To enable: PgTextsearch::Maintenance.enable_pg_trgm!"
|
|
47
|
+
puts " Then use: fuzzy: {threshold: 0.3} in bm25_search"
|
|
48
|
+
end
|
|
49
|
+
puts
|
|
50
|
+
|
|
51
|
+
# thesaurus: synonyms
|
|
52
|
+
puts "3. thesaurus (synonym dictionary)"
|
|
53
|
+
puts " Available: #{PgTextsearch::Maintenance.thesaurus_available?('thesaurus_en')}"
|
|
54
|
+
if !PgTextsearch::Maintenance.thesaurus_available?('thesaurus_en')
|
|
55
|
+
puts " To enable: PgTextsearch::Maintenance.enable_thesaurus!('thesaurus_en', 'thesaurus_en')"
|
|
56
|
+
puts " Requires thesaurus_en.ths file in \$SHAREDIR/tsearch_data/"
|
|
57
|
+
puts " Example thesaurus file content:"
|
|
58
|
+
puts " pop, soda, cola"
|
|
59
|
+
puts " headphones, earphones, cans"
|
|
60
|
+
puts " laptop, notebook"
|
|
61
|
+
end
|
|
62
|
+
puts
|
|
63
|
+
|
|
64
|
+
puts "=== Usage in bm25_search ==="
|
|
65
|
+
puts <<~RUBY
|
|
66
|
+
# Unaccent
|
|
67
|
+
Product.bm25_search(:name, "cafe", index: :idx, unaccent: true)
|
|
68
|
+
|
|
69
|
+
# Fuzzy (requires pg_trgm)
|
|
70
|
+
Product.bm25_search(:name, "headphon", index: :idx, fuzzy: {threshold: 0.3})
|
|
71
|
+
|
|
72
|
+
# Synonyms (requires thesaurus dict in index text_config)
|
|
73
|
+
Product.bm25_search(:name, "pop", index: :idx, synonyms: true)
|
|
74
|
+
|
|
75
|
+
# Combined
|
|
76
|
+
Product.bm25_search(:search_text, "headphones", index: :idx,
|
|
77
|
+
unaccent: true,
|
|
78
|
+
prefix: true,
|
|
79
|
+
any_word: true,
|
|
80
|
+
negation: true,
|
|
81
|
+
fuzzy: {threshold: 0.3},
|
|
82
|
+
highlight: true,
|
|
83
|
+
ranked_by: ":bm25 + 0.1 * products.popularity",
|
|
84
|
+
order_within_rank: "products.updated_at DESC"
|
|
85
|
+
)
|
|
86
|
+
RUBY
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
run if $PROGRAM_NAME == __FILE__
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_record"
|
|
4
|
+
require "pg_textsearch"
|
|
5
|
+
require "minitest/autorun"
|
|
6
|
+
|
|
7
|
+
ActiveRecord::Base.establish_connection(
|
|
8
|
+
adapter: "postgresql",
|
|
9
|
+
database: "pg_textsearch_test"
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
class Document < ActiveRecord::Base
|
|
13
|
+
self.table_name = "documents"
|
|
14
|
+
has_bm25 :content, index: :docs_idx
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
ActiveRecord::Schema.define do
|
|
18
|
+
create_table :documents, force: true do |t|
|
|
19
|
+
t.text :content
|
|
20
|
+
t.integer :category_id
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
enable_extension "pg_textsearch"
|
|
24
|
+
|
|
25
|
+
execute "CREATE INDEX docs_idx ON documents USING bm25(content) WITH (text_config='english')"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
class PgTextsearchIntegrationTest < Minitest::Test
|
|
29
|
+
def setup
|
|
30
|
+
Document.delete_all
|
|
31
|
+
Document.create!([
|
|
32
|
+
{ content: "PostgreSQL is a powerful database system", category_id: 1 },
|
|
33
|
+
{ content: "BM25 is an effective ranking function", category_id: 1 },
|
|
34
|
+
{ content: "Full text search with custom scoring", category_id: 2 },
|
|
35
|
+
{ content: "Database indexes improve query performance", category_id: 2 },
|
|
36
|
+
{ content: "PostgreSQL full text search tutorial", category_id: 1 }
|
|
37
|
+
])
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def test_bm25_search_returns_results
|
|
41
|
+
results = Document.bm25_search(:content, "database system", index: :docs_idx).limit(3).to_a
|
|
42
|
+
assert results.size >= 1
|
|
43
|
+
assert results.first.content.include?("database") || results.first.content.include?("PostgreSQL")
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def test_bm25_search_with_has_bm25_declaration
|
|
47
|
+
results = Document.bm25_search(:content, "database system").limit(3).to_a
|
|
48
|
+
assert results.size >= 1
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def test_bm25_search_composes_with_where
|
|
52
|
+
results = Document.where(category_id: 1).bm25_search(:content, "database", index: :docs_idx).to_a
|
|
53
|
+
assert(results.all? { |d| d.category_id == 1 })
|
|
54
|
+
assert(results.any? { |d| d.content.include?("database") || d.content.include?("PostgreSQL") })
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def test_bm25_search_with_select_score
|
|
58
|
+
results = Document.bm25_search(:content, "database system", index: :docs_idx, select_score: true).limit(3).to_a
|
|
59
|
+
assert results.first.respond_to?(:bm25_score)
|
|
60
|
+
assert results.first.bm25_score.is_a?(Numeric)
|
|
61
|
+
assert results.first.bm25_score <= 0 # BM25 scores are negative
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def test_bm25_search_with_custom_score_alias
|
|
65
|
+
results = Document.bm25_search(:content, "database system", index: :docs_idx, select_score: :my_score).limit(3).to_a
|
|
66
|
+
assert results.first.respond_to?(:my_score)
|
|
67
|
+
assert results.first.my_score.is_a?(Numeric)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def test_bm25_search_with_arel_expression
|
|
71
|
+
arel = Arel.sql("(data->>'description')")
|
|
72
|
+
# This test just verifies SQL generation works; we don't have a JSON column here
|
|
73
|
+
sql = Document.bm25_search(arel, "network error", index: :events_expr_idx).to_sql
|
|
74
|
+
assert_match(/ORDER BY \(data->>'description'\) <@> to_bm25query/, sql)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def test_bm25_search_with_raw_sql_string
|
|
78
|
+
sql = Document.bm25_search("data->>'description'", "network error", index: :events_expr_idx).to_sql
|
|
79
|
+
assert_match(/ORDER BY data->>'description' <@> to_bm25query/, sql)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def test_bm25_search_scores_are_ordered
|
|
83
|
+
results = Document.bm25_search(:content, "database", index: :docs_idx, select_score: true).to_a
|
|
84
|
+
scores = results.map(&:bm25_score)
|
|
85
|
+
assert_equal scores.sort, scores # Should be in ascending order (more negative = better)
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
class PgTextsearchMaintenanceIntegrationTest < Minitest::Test
|
|
90
|
+
def setup
|
|
91
|
+
Document.delete_all
|
|
92
|
+
Document.create!([
|
|
93
|
+
{ content: "PostgreSQL is a powerful database system" },
|
|
94
|
+
{ content: "BM25 is an effective ranking function" }
|
|
95
|
+
])
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def test_needs_compaction_returns_boolean
|
|
99
|
+
result = PgTextsearch::Maintenance.needs_compaction?(:docs_idx)
|
|
100
|
+
assert [true, false].include?(result)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def test_compact_returns_void
|
|
104
|
+
result = PgTextsearch::Maintenance.compact(:docs_idx)
|
|
105
|
+
# bm25_compact returns void, which comes back as empty string
|
|
106
|
+
assert [true, false, ""].include?(result)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def test_compact_step_returns_boolean
|
|
110
|
+
result = PgTextsearch::Maintenance.compact_step(:docs_idx)
|
|
111
|
+
assert [true, false].include?(result)
|
|
112
|
+
# false means no compaction pass was run (nothing to do), which is expected
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def test_spill_index_returns_integer
|
|
116
|
+
result = PgTextsearch::Maintenance.spill_index(:docs_idx)
|
|
117
|
+
assert result.is_a?(Integer)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def test_level_counts_returns_array
|
|
121
|
+
result = PgTextsearch::Maintenance.level_counts(:docs_idx)
|
|
122
|
+
assert result.is_a?(Array)
|
|
123
|
+
assert(result.all? { |v| v.is_a?(Integer) })
|
|
124
|
+
end
|
|
125
|
+
end
|