pg_textsearch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.devcontainer/devcontainer.json +32 -0
- data/.devcontainer/setup.sh +25 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +514 -0
- data/Rakefile +23 -0
- data/lib/pg_textsearch/maintenance.rb +133 -0
- data/lib/pg_textsearch/model.rb +172 -0
- data/lib/pg_textsearch/schema.rb +53 -0
- data/lib/pg_textsearch/version.rb +5 -0
- data/lib/pg_textsearch.rb +30 -0
- data/samples/01_basic_search.rb +113 -0
- data/samples/02_advanced_query_options.rb +106 -0
- data/samples/03_highlighting.rb +100 -0
- data/samples/04_custom_ranking.rb +110 -0
- data/samples/05_fuzzy_synonyms.rb +112 -0
- data/samples/06_maintenance.rb +111 -0
- data/samples/07_extension_setup.rb +89 -0
- data/sig/pg_textsearch.rbs +4 -0
- data/test/integration_test.rb +125 -0
- data/test/pg_textsearch_test.rb +162 -0
- metadata +107 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_record"
|
|
4
|
+
|
|
5
|
+
module PgTextsearch
|
|
6
|
+
# Thin module-level wrappers for index maintenance. Use in Rake tasks or
|
|
7
|
+
# background jobs. All take an index name (symbol or string).
|
|
8
|
+
module Maintenance
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
def needs_compaction?(index)
|
|
12
|
+
connection.select_value(<<~SQL.squish)
|
|
13
|
+
SELECT bm25_needs_compaction(#{quote(index)})
|
|
14
|
+
SQL
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def compact(index)
|
|
18
|
+
connection.select_value(<<~SQL.squish)
|
|
19
|
+
SELECT bm25_compact(#{quote(index)}::regclass)
|
|
20
|
+
SQL
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def compact_step(index)
|
|
24
|
+
connection.select_value(<<~SQL.squish)
|
|
25
|
+
SELECT bm25_compact_step(#{quote(index)}::regclass)
|
|
26
|
+
SQL
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def spill_index(index)
|
|
30
|
+
connection.select_value(<<~SQL.squish)
|
|
31
|
+
SELECT bm25_spill_index(#{quote(index)})
|
|
32
|
+
SQL
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def level_counts(index)
|
|
36
|
+
result = connection.select_value(<<~SQL.squish)
|
|
37
|
+
SELECT bm25_level_counts(#{quote(index)}::regclass)
|
|
38
|
+
SQL
|
|
39
|
+
result&.tr("{}", "").split(",").map(&:to_i) || []
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# --- Optional Extension Helpers ---
|
|
43
|
+
|
|
44
|
+
# unaccent: makes searches accent-insensitive
|
|
45
|
+
# Usage: enable_unaccent! then use unaccent: true in bm25_search
|
|
46
|
+
def unaccent_available?
|
|
47
|
+
extension_installed?("unaccent")
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def enable_unaccent!
|
|
51
|
+
return if unaccent_available?
|
|
52
|
+
|
|
53
|
+
connection.execute("CREATE EXTENSION IF NOT EXISTS unaccent")
|
|
54
|
+
# Create wrapper function for indexing (unaccent is not IMMUTABLE by default)
|
|
55
|
+
connection.execute(<<~SQL)
|
|
56
|
+
CREATE OR REPLACE FUNCTION public.unaccent_immutable(text) RETURNS text
|
|
57
|
+
LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
|
|
58
|
+
SELECT public.unaccent('public.unaccent', $1);
|
|
59
|
+
$$;
|
|
60
|
+
SQL
|
|
61
|
+
puts "✓ unaccent enabled. Use unaccent: true in bm25_search"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# pg_trgm: enables fuzzy/prefix matching
|
|
65
|
+
# Usage: enable_pg_trgm! then use fuzzy: true in bm25_search
|
|
66
|
+
def pg_trgm_available?
|
|
67
|
+
extension_installed?("pg_trgm")
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def enable_pg_trgm!
|
|
71
|
+
return if pg_trgm_available?
|
|
72
|
+
|
|
73
|
+
connection.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
|
|
74
|
+
puts "✓ pg_trgm enabled. Use fuzzy: {threshold: 0.3} in bm25_search"
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# thesaurus: synonym dictionary
|
|
78
|
+
# Usage: enable_thesaurus!('english', '/path/to/thesaurus_en') then use synonyms: true
|
|
79
|
+
def thesaurus_available?(dict_name = "thesaurus_en")
|
|
80
|
+
connection.select_value(<<~SQL)
|
|
81
|
+
SELECT 1 FROM pg_ts_dict WHERE dictname = '#{dict_name}'
|
|
82
|
+
SQL
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def enable_thesaurus!(dict_name = "thesaurus_en", dict_file = nil)
|
|
86
|
+
# dict_file should be basename without extension (e.g., 'thesaurus_en')
|
|
87
|
+
# File must be in $SHAREDIR/tsearch_data/
|
|
88
|
+
return if thesaurus_available?(dict_name)
|
|
89
|
+
|
|
90
|
+
unless dict_file
|
|
91
|
+
puts "⚠ thesaurus file required. Place #{dict_file}.ths in $SHAREDIR/tsearch_data/"
|
|
92
|
+
return
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
connection.execute("CREATE EXTENSION IF NOT EXISTS dict_thesaurus")
|
|
96
|
+
connection.execute(<<~SQL)
|
|
97
|
+
CREATE TEXT SEARCH DICTIONARY #{dict_name} (
|
|
98
|
+
TEMPLATE = thesaurus,
|
|
99
|
+
DictFile = #{quote(dict_file)},
|
|
100
|
+
Dictionary = pg_catalog.english_stem
|
|
101
|
+
);
|
|
102
|
+
SQL
|
|
103
|
+
connection.execute("ALTER TEXT SEARCH CONFIGURATION english ALTER MAPPING FOR asciiword, asciihword, hword_asciipart WITH #{dict_name}, pg_catalog.english_stem;")
|
|
104
|
+
puts "✓ Thesaurus '#{dict_name}' enabled. Use synonyms: true in bm25_search"
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def fuzzy_search_sql(target, query, _index, threshold: 0.3)
|
|
108
|
+
# Uses pg_trgm similarity for fuzzy matching
|
|
109
|
+
# Returns SQL for: similarity(target, query) > threshold
|
|
110
|
+
target_sql = case target
|
|
111
|
+
when Symbol then connection.quote_column_name(target)
|
|
112
|
+
when Arel::Nodes::Node, Arel::Attributes::Attribute then target.to_sql
|
|
113
|
+
else target.to_s
|
|
114
|
+
end
|
|
115
|
+
q = quote(query.to_s)
|
|
116
|
+
"similarity(#{target_sql}, #{q}) > #{threshold}"
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def connection
|
|
120
|
+
ActiveRecord::Base.connection
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def quote(name)
|
|
124
|
+
ActiveRecord::Base.connection.quote(name.to_s)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def extension_installed?(name)
|
|
128
|
+
connection.select_value(<<~SQL)
|
|
129
|
+
SELECT 1 FROM pg_extension WHERE extname = '#{name}'
|
|
130
|
+
SQL
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PgTextsearch
|
|
4
|
+
# Extended into ActiveRecord::Base. Wraps the awkward SQL bits (`<@>` and
|
|
5
|
+
# `to_bm25query`) and returns plain relations so the rest of ActiveRecord
|
|
6
|
+
# composes unchanged.
|
|
7
|
+
module Model
|
|
8
|
+
# has_bm25 :content, index: :docs_idx, unaccent: true, synonyms: true
|
|
9
|
+
def has_bm25(target, index:, **default_options)
|
|
10
|
+
class_attribute :bm25_indexes, instance_accessor: false, default: {} unless respond_to?(:bm25_indexes)
|
|
11
|
+
class_attribute :bm25_default_options, instance_accessor: false, default: {} unless respond_to?(:bm25_default_options)
|
|
12
|
+
self.bm25_indexes = bm25_indexes.merge(target.to_s => index.to_s)
|
|
13
|
+
self.bm25_default_options = default_options
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Document.bm25_search(:content, "database system", index: :docs_idx,
|
|
17
|
+
# prefix: true, any_word: true, negation: true, unaccent: true,
|
|
18
|
+
# fuzzy: {threshold: 0.3}, synonyms: true,
|
|
19
|
+
# select_score: true, highlight: true, ranked_by: "expr", order_within_rank: "col DESC"
|
|
20
|
+
# ).limit(5)
|
|
21
|
+
def bm25_search(target, query, index: nil, **options)
|
|
22
|
+
ensure_pg_textsearch_available!
|
|
23
|
+
|
|
24
|
+
# Merge with model-level defaults
|
|
25
|
+
defaults = respond_to?(:bm25_default_options) ? bm25_default_options : {}
|
|
26
|
+
options = defaults.merge(options)
|
|
27
|
+
|
|
28
|
+
index ||= bm25_indexes[target.to_s] if respond_to?(:bm25_indexes)
|
|
29
|
+
|
|
30
|
+
target_sql = bm25_target_sql(target, options[:unaccent])
|
|
31
|
+
query_sql = bm25_query_sql(query, index, options)
|
|
32
|
+
score_sql = "#{target_sql} <@> #{query_sql}"
|
|
33
|
+
|
|
34
|
+
# Build ORDER BY
|
|
35
|
+
order_sql = build_order_sql(score_sql, options)
|
|
36
|
+
|
|
37
|
+
relation = order(Arel.sql(order_sql))
|
|
38
|
+
|
|
39
|
+
# Score selection
|
|
40
|
+
if options[:select_score]
|
|
41
|
+
name = options[:select_score] == true ? "bm25_score" : options[:select_score].to_s
|
|
42
|
+
relation = relation.select(arel_table[Arel.star],
|
|
43
|
+
Arel.sql("#{score_sql} AS #{connection.quote_column_name(name)}"))
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Highlighting
|
|
47
|
+
relation = apply_highlight(relation, target, query, index, options[:highlight]) if options[:highlight]
|
|
48
|
+
|
|
49
|
+
# Fuzzy search (pg_trgm) - adds WHERE clause
|
|
50
|
+
relation = apply_fuzzy(relation, target, query, options[:fuzzy]) if options[:fuzzy]
|
|
51
|
+
|
|
52
|
+
# Synonyms - uses thesaurus dictionary (already handled via text_config)
|
|
53
|
+
# Just pass through; the index must be created with thesaurus dict
|
|
54
|
+
|
|
55
|
+
relation
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def bm25_target_sql(target, unaccent = false)
|
|
61
|
+
base = case target
|
|
62
|
+
when Symbol then connection.quote_column_name(target)
|
|
63
|
+
when Arel::Nodes::Node, Arel::Attributes::Attribute then target.to_sql
|
|
64
|
+
else target.to_s
|
|
65
|
+
end
|
|
66
|
+
unaccent ? "unaccent(#{base})" : base
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def bm25_query_sql(query, index, options)
|
|
70
|
+
# Build the query text with options
|
|
71
|
+
q = query.to_s.dup
|
|
72
|
+
|
|
73
|
+
# Prefix matching - append :* to each word
|
|
74
|
+
q = q.split.map { |w| "#{w}:*" }.join(" ") if options[:prefix]
|
|
75
|
+
|
|
76
|
+
# Any word (OR) - join with |
|
|
77
|
+
q = q.split.map { |w| w.delete_suffix(":*") }.join(" | ") if options[:any_word]
|
|
78
|
+
|
|
79
|
+
# Negation - words starting with ! are already handled by tsquery syntax
|
|
80
|
+
# Just pass through as-is
|
|
81
|
+
|
|
82
|
+
# Unaccent
|
|
83
|
+
q = "unaccent(#{sanitize_sql_array(["?", q])})" if options[:unaccent]
|
|
84
|
+
|
|
85
|
+
if index
|
|
86
|
+
sanitize_sql_array(["to_bm25query(?, ?)", q, index.to_s])
|
|
87
|
+
else
|
|
88
|
+
sanitize_sql_array(["to_bm25query(?)", q])
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def build_order_sql(score_sql, options)
|
|
93
|
+
if options[:ranked_by]
|
|
94
|
+
# Custom ranking expression - wrap score_sql in parentheses for operator precedence
|
|
95
|
+
# <@> has lower precedence than +, so we need (score_sql) + ...
|
|
96
|
+
ranked = options[:ranked_by].gsub(":bm25", "(#{score_sql})")
|
|
97
|
+
base = Arel.sql(ranked)
|
|
98
|
+
else
|
|
99
|
+
base = Arel.sql(score_sql)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
if options[:order_within_rank]
|
|
103
|
+
# Tiebreaker
|
|
104
|
+
Arel.sql("#{base}, #{options[:order_within_rank]}")
|
|
105
|
+
else
|
|
106
|
+
base
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def apply_highlight(relation, target, query, _index, highlight_opts)
|
|
111
|
+
# ts_headline(text, query, [options])
|
|
112
|
+
target_sql = bm25_target_sql(target, false)
|
|
113
|
+
# Convert query to valid tsquery format: join words with & (AND)
|
|
114
|
+
query_for_highlight = query.to_s.split.join(" & ")
|
|
115
|
+
|
|
116
|
+
# Build ts_headline options
|
|
117
|
+
hl_opts = highlight_opts.is_a?(Hash) ? highlight_opts : {}
|
|
118
|
+
start_sel = hl_opts[:start_sel] || "<b>"
|
|
119
|
+
stop_sel = hl_opts[:stop_sel] || "</b>"
|
|
120
|
+
max_words = hl_opts[:max_words] || 35
|
|
121
|
+
min_words = hl_opts[:min_words] || 15
|
|
122
|
+
short_word = hl_opts[:short_word] || 3
|
|
123
|
+
highlight_all = hl_opts[:highlight_all] ? "true" : "false"
|
|
124
|
+
max_fragments = hl_opts[:max_fragments] || 0
|
|
125
|
+
fragment_delim = hl_opts[:fragment_delimiter] || "..."
|
|
126
|
+
|
|
127
|
+
# ts_headline options string - escape single quotes by doubling
|
|
128
|
+
esc = ->(s) { s.to_s.gsub("'", "''") }
|
|
129
|
+
opts_str = "StartSel=#{esc.call(start_sel)}, StopSel=#{esc.call(stop_sel)}, " \
|
|
130
|
+
"MaxWords=#{max_words}, MinWords=#{min_words}, " \
|
|
131
|
+
"ShortWord=#{short_word}, HighlightAll=#{highlight_all}, " \
|
|
132
|
+
"MaxFragments=#{max_fragments}, FragmentDelimiter=#{esc.call(fragment_delim)}"
|
|
133
|
+
|
|
134
|
+
# Use to_tsquery for highlighting (not bm25query)
|
|
135
|
+
tsquery = "to_tsquery('english', #{sanitize_sql_array(["?", query_for_highlight])})"
|
|
136
|
+
# The opts_str is a string literal in SQL, so escape internal single quotes
|
|
137
|
+
highlight_sql = "ts_headline('english', #{target_sql}, #{tsquery}, '#{opts_str.gsub("'", "''")}')"
|
|
138
|
+
|
|
139
|
+
alias_name = "pg_search_highlight"
|
|
140
|
+
relation.select(arel_table[Arel.star],
|
|
141
|
+
Arel.sql("#{highlight_sql} AS #{connection.quote_column_name(alias_name)}"))
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def apply_fuzzy(relation, target, query, fuzzy_opts)
|
|
145
|
+
# Uses pg_trgm similarity for fuzzy matching
|
|
146
|
+
# Adds WHERE similarity(target, query) > threshold
|
|
147
|
+
return relation unless Maintenance.pg_trgm_available?
|
|
148
|
+
|
|
149
|
+
threshold = fuzzy_opts.is_a?(Hash) ? (fuzzy_opts[:threshold] || 0.3) : 0.3
|
|
150
|
+
target_sql = bm25_target_sql(target, false)
|
|
151
|
+
q = connection.quote(query.to_s)
|
|
152
|
+
|
|
153
|
+
where_sql = "similarity(#{target_sql}, #{q}) > #{threshold}"
|
|
154
|
+
relation.where(Arel.sql(where_sql))
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def ensure_pg_textsearch_available!
|
|
158
|
+
return unless connection.adapter_name.downcase.include?("postgres")
|
|
159
|
+
return if connection.extensions.include?("pg_textsearch")
|
|
160
|
+
|
|
161
|
+
raise StandardError, <<~MSG.squish
|
|
162
|
+
pg_textsearch extension is not enabled. To fix this:
|
|
163
|
+
|
|
164
|
+
1. Add to postgresql.conf: shared_preload_libraries = 'pg_textsearch'
|
|
165
|
+
2. Restart PostgreSQL
|
|
166
|
+
3. Run: CREATE EXTENSION pg_textsearch;
|
|
167
|
+
|
|
168
|
+
See https://github.com/timescale/pg_textsearch for installation instructions.
|
|
169
|
+
MSG
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PgTextsearch
|
|
4
|
+
# Adds index storage parameters to add_index, e.g.
|
|
5
|
+
#
|
|
6
|
+
# add_index :documents, :content, using: :bm25,
|
|
7
|
+
# with: { text_config: "english", k1: 1.5 }
|
|
8
|
+
#
|
|
9
|
+
# and surfaces them on IndexDefinition so the schema dumper can emit them.
|
|
10
|
+
module SchemaStatements
|
|
11
|
+
def add_index(table_name, column_name, **options)
|
|
12
|
+
with = options.delete(:with)
|
|
13
|
+
return super if with.blank?
|
|
14
|
+
|
|
15
|
+
index, algorithm, if_not_exists = add_index_options(table_name, column_name, **options)
|
|
16
|
+
definition = ActiveRecord::ConnectionAdapters::CreateIndexDefinition.new(index, algorithm, if_not_exists)
|
|
17
|
+
sql = schema_creation.accept(definition)
|
|
18
|
+
with_sql = "WITH (#{with.map { |k, v| "#{k}=#{quote(v)}" }.join(", ")})"
|
|
19
|
+
where_sql = index.where ? " WHERE #{index.where}" : ""
|
|
20
|
+
execute "#{sql.delete_suffix(where_sql)} #{with_sql}#{where_sql}"
|
|
21
|
+
execute "COMMENT ON INDEX #{quote_column_name(index.name)} IS #{quote(index.comment)}" if index.comment
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def indexes(table_name)
|
|
25
|
+
indexes = super
|
|
26
|
+
reloptions = select_rows(<<~SQL.squish, "SCHEMA").to_h
|
|
27
|
+
SELECT c.relname, array_to_string(c.reloptions, ',')
|
|
28
|
+
FROM pg_class c JOIN pg_index i ON i.indexrelid = c.oid
|
|
29
|
+
WHERE i.indrelid = #{quote(quote_table_name(table_name))}::regclass
|
|
30
|
+
AND c.reloptions IS NOT NULL
|
|
31
|
+
SQL
|
|
32
|
+
indexes.each do |index|
|
|
33
|
+
next unless (options = reloptions[index.name])
|
|
34
|
+
|
|
35
|
+
index.instance_variable_set(:@with, options.split(",").to_h { |opt| opt.split("=", 2) })
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Dumps storage parameters back out as `with: { ... }`.
|
|
41
|
+
module SchemaDumper
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def index_parts(index)
|
|
45
|
+
parts = super
|
|
46
|
+
if (with = index.instance_variable_get(:@with))
|
|
47
|
+
formatted = with.map { |k, v| "#{k}: #{(Integer(v, exception: false) || Float(v, exception: false) || v).inspect}" }
|
|
48
|
+
parts << "with: { #{formatted.join(", ")} }"
|
|
49
|
+
end
|
|
50
|
+
parts
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support"
|
|
4
|
+
require "active_record"
|
|
5
|
+
|
|
6
|
+
require_relative "pg_textsearch/version"
|
|
7
|
+
|
|
8
|
+
module PgTextsearch
|
|
9
|
+
class Error < StandardError; end
|
|
10
|
+
|
|
11
|
+
autoload :Model, "pg_textsearch/model"
|
|
12
|
+
autoload :SchemaStatements, "pg_textsearch/schema"
|
|
13
|
+
autoload :SchemaDumper, "pg_textsearch/schema"
|
|
14
|
+
autoload :Maintenance, "pg_textsearch/maintenance"
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Extend ActiveRecord::Base with bm25_search / has_bm25
|
|
18
|
+
ActiveSupport.on_load(:active_record) do
|
|
19
|
+
extend PgTextsearch::Model
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Extend the PostgreSQL adapter with `with:` storage parameters (only if pg adapter is loaded)
|
|
23
|
+
ActiveSupport.on_load(:active_record_postgresqladapter) do
|
|
24
|
+
prepend PgTextsearch::SchemaStatements
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Extend the schema dumper
|
|
28
|
+
ActiveSupport.on_load(:active_record) do
|
|
29
|
+
ActiveRecord::SchemaDumper.prepend(PgTextsearch::SchemaDumper)
|
|
30
|
+
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
# Basic search with pg_textsearch
|
|
4
|
+
# Run: ruby samples/01_basic_search.rb
|
|
5
|
+
|
|
6
|
+
require "bundler/inline"
|
|
7
|
+
|
|
8
|
+
gemfile do
|
|
9
|
+
source "https://rubygems.org"
|
|
10
|
+
gem "activerecord", ">= 7.0"
|
|
11
|
+
gem "pg", ">= 1.5"
|
|
12
|
+
gem "pg_textsearch", path: File.expand_path("..", __dir__)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
require "active_record"
|
|
16
|
+
require "pg_textsearch"
|
|
17
|
+
require "logger"
|
|
18
|
+
|
|
19
|
+
DB_CONFIG = {
|
|
20
|
+
adapter: "postgresql",
|
|
21
|
+
database: ENV.fetch("PG_DATABASE", "pg_textsearch_test"),
|
|
22
|
+
username: ENV.fetch("PG_USER", "postgres"),
|
|
23
|
+
password: ENV.fetch("PG_PASSWORD", "postgres"),
|
|
24
|
+
host: ENV.fetch("PG_HOST", "localhost"),
|
|
25
|
+
port: ENV.fetch("PG_PORT", 5432)
|
|
26
|
+
}.freeze
|
|
27
|
+
|
|
28
|
+
ActiveRecord::Base.establish_connection(DB_CONFIG)
|
|
29
|
+
ActiveRecord::Base.logger = Logger.new($stdout) if ENV["DEBUG"]
|
|
30
|
+
|
|
31
|
+
# --- Schema ---
|
|
32
|
+
def setup_schema
|
|
33
|
+
conn = ActiveRecord::Base.connection
|
|
34
|
+
conn.execute("CREATE EXTENSION IF NOT EXISTS pg_textsearch")
|
|
35
|
+
conn.execute("DROP TABLE IF EXISTS products CASCADE")
|
|
36
|
+
conn.execute(<<~SQL)
|
|
37
|
+
CREATE TABLE products (
|
|
38
|
+
id BIGSERIAL PRIMARY KEY,
|
|
39
|
+
name TEXT NOT NULL,
|
|
40
|
+
description TEXT,
|
|
41
|
+
category VARCHAR(50),
|
|
42
|
+
brand VARCHAR(50),
|
|
43
|
+
price NUMERIC(10,2),
|
|
44
|
+
active BOOLEAN DEFAULT true,
|
|
45
|
+
search_text TEXT
|
|
46
|
+
)
|
|
47
|
+
SQL
|
|
48
|
+
conn.execute(<<~SQL)
|
|
49
|
+
CREATE INDEX products_search_idx ON products
|
|
50
|
+
USING bm25 (search_text)
|
|
51
|
+
WITH (text_config = 'english', k1 = 1.5, b = 0.75);
|
|
52
|
+
SQL
|
|
53
|
+
conn.execute(<<~SQL)
|
|
54
|
+
CREATE INDEX products_name_idx ON products USING bm25 (name)
|
|
55
|
+
WITH (text_config = 'english');
|
|
56
|
+
SQL
|
|
57
|
+
puts "✓ Schema created"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def seed_data
|
|
61
|
+
Product.delete_all
|
|
62
|
+
products = [
|
|
63
|
+
{ name: "Sony WH-1000XM5 Wireless Headphones", description: "Industry-leading noise cancellation", category: "electronics", brand: "Sony", price: 399.99, active: true },
|
|
64
|
+
{ name: "Bose QuietComfort Ultra Headphones", description: "World-class noise cancellation", category: "electronics", brand: "Bose", price: 429.00, active: true },
|
|
65
|
+
{ name: "Apple AirPods Pro (2nd Gen)", description: "Active Noise Cancellation", category: "electronics", brand: "Apple", price: 249.00, active: true },
|
|
66
|
+
{ name: "Sennheiser Momentum 4 Wireless", description: "Audiophile sound", category: "electronics", brand: "Sennheiser", price: 379.95, active: true },
|
|
67
|
+
{ name: "Sony WF-1000XM5 Earbuds", description: "Best-in-class noise cancellation in earbud form", category: "electronics", brand: "Sony", price: 299.99, active: true },
|
|
68
|
+
{ name: "Jabra Elite 10", description: "Dolby Atmos spatial audio", category: "electronics", brand: "Jabra", price: 249.99, active: true },
|
|
69
|
+
{ name: "Sony WH-CH720N Budget Headphones", description: "Lightweight noise cancellation", category: "electronics", brand: "Sony", price: 149.99, active: true },
|
|
70
|
+
{ name: "Bose QuietComfort 45", description: "Proven noise cancellation", category: "electronics", brand: "Bose", price: 329.00, active: true },
|
|
71
|
+
{ name: "Sennheiser Accentum Wireless", description: "Great value ANC", category: "electronics", brand: "Sennheiser", price: 179.95, active: true },
|
|
72
|
+
{ name: "Nothing Ear (2)", description: "Transparent design ANC", category: "electronics", brand: "Nothing", price: 149.00, active: true }
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
products.each do |p|
|
|
76
|
+
p[:search_text] = [p[:name], p[:description]].compact.join(" ")
|
|
77
|
+
Product.create!(p)
|
|
78
|
+
end
|
|
79
|
+
puts "✓ Seeded #{Product.count} products"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
class Product < ActiveRecord::Base
|
|
83
|
+
has_bm25 :search_text, index: :products_search_idx
|
|
84
|
+
has_bm25 :name, index: :products_name_idx
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def run
|
|
88
|
+
setup_schema
|
|
89
|
+
seed_data
|
|
90
|
+
|
|
91
|
+
puts "\n=== Basic Search ==="
|
|
92
|
+
results = Product.where(active: true)
|
|
93
|
+
.bm25_search(:search_text, "wireless headphones", index: :products_search_idx)
|
|
94
|
+
.limit(5)
|
|
95
|
+
results.each { |p| puts " #{p.name} (#{p.brand})" }
|
|
96
|
+
|
|
97
|
+
puts "\n=== Search with Score ==="
|
|
98
|
+
results = Product.where(active: true)
|
|
99
|
+
.bm25_search(:search_text, "headphones", index: :products_search_idx, select_score: true)
|
|
100
|
+
.limit(5)
|
|
101
|
+
results.each { |p| puts " #{p.name}: score = #{p.bm25_score&.round(3)}" }
|
|
102
|
+
|
|
103
|
+
puts "\n=== Search Composed with WHERE ==="
|
|
104
|
+
results = Product.where(active: true, category: "electronics")
|
|
105
|
+
.where("price BETWEEN ? AND ?", 200, 400)
|
|
106
|
+
.bm25_search(:search_text, "noise cancelling", index: :products_search_idx)
|
|
107
|
+
.limit(5)
|
|
108
|
+
results.each { |p| puts " #{p.name} - $#{p.price}" }
|
|
109
|
+
|
|
110
|
+
puts "\n✅ Basic search examples completed!"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
run if $PROGRAM_NAME == __FILE__
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
# Advanced query options: prefix, any_word, negation, unaccent
|
|
4
|
+
# Run: ruby samples/02_advanced_query_options.rb
|
|
5
|
+
|
|
6
|
+
require "bundler/inline"
|
|
7
|
+
|
|
8
|
+
gemfile do
|
|
9
|
+
source "https://rubygems.org"
|
|
10
|
+
gem "activerecord", ">= 7.0"
|
|
11
|
+
gem "pg", ">= 1.5"
|
|
12
|
+
gem "pg_textsearch", path: File.expand_path("..", __dir__)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
require "active_record"
|
|
16
|
+
require "pg_textsearch"
|
|
17
|
+
require "logger"
|
|
18
|
+
|
|
19
|
+
DB_CONFIG = {
|
|
20
|
+
adapter: "postgresql",
|
|
21
|
+
database: ENV.fetch("PG_DATABASE", "pg_textsearch_test"),
|
|
22
|
+
username: ENV.fetch("PG_USER", "postgres"),
|
|
23
|
+
password: ENV.fetch("PG_PASSWORD", "postgres"),
|
|
24
|
+
host: ENV.fetch("PG_HOST", "localhost"),
|
|
25
|
+
port: ENV.fetch("PG_PORT", 5432)
|
|
26
|
+
}.freeze
|
|
27
|
+
|
|
28
|
+
ActiveRecord::Base.establish_connection(DB_CONFIG)
|
|
29
|
+
|
|
30
|
+
def setup_schema
|
|
31
|
+
conn = ActiveRecord::Base.connection
|
|
32
|
+
conn.execute("CREATE EXTENSION IF NOT EXISTS pg_textsearch")
|
|
33
|
+
conn.execute("DROP TABLE IF EXISTS products CASCADE")
|
|
34
|
+
conn.execute(<<~SQL)
|
|
35
|
+
CREATE TABLE products (
|
|
36
|
+
id BIGSERIAL PRIMARY KEY,
|
|
37
|
+
name TEXT NOT NULL,
|
|
38
|
+
description TEXT,
|
|
39
|
+
search_text TEXT
|
|
40
|
+
)
|
|
41
|
+
SQL
|
|
42
|
+
conn.execute(<<~SQL)
|
|
43
|
+
CREATE INDEX products_search_idx ON products
|
|
44
|
+
USING bm25 (search_text)
|
|
45
|
+
WITH (text_config = 'english', k1 = 1.5, b = 0.75);
|
|
46
|
+
SQL
|
|
47
|
+
puts "✓ Schema created"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def seed_data
|
|
51
|
+
Product.delete_all
|
|
52
|
+
products = [
|
|
53
|
+
{ name: "Sony WH-1000XM5", description: "Industry-leading noise cancellation with Auto NC Optimizer" },
|
|
54
|
+
{ name: "Bose QuietComfort Ultra", description: "World-class noise cancellation. Spatial audio." },
|
|
55
|
+
{ name: "Apple AirPods Pro", description: "Active Noise Cancellation. Transparency mode." },
|
|
56
|
+
{ name: "Sennheiser Momentum 4", description: "Audiophile sound. 60-hour battery." },
|
|
57
|
+
{ name: "Sony WF-1000XM5", description: "Best-in-class noise cancellation in earbud form" },
|
|
58
|
+
{ name: "Jabra Elite 10", description: "Dolby Atmos spatial audio. Advanced ANC." },
|
|
59
|
+
{ name: "Sony WH-CH720N", description: "Lightweight noise cancellation. 35-hour battery." },
|
|
60
|
+
{ name: "Bose QuietComfort 45", description: "Proven noise cancellation. Comfortable fit." },
|
|
61
|
+
{ name: "Sennheiser Accentum", description: "Great value ANC. 50-hour battery." },
|
|
62
|
+
{ name: "Nothing Ear (2)", description: "Transparent design. ANC. 36-hour battery." }
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
products.each do |p|
|
|
66
|
+
p[:search_text] = [p[:name], p[:description]].compact.join(" ")
|
|
67
|
+
Product.create!(p)
|
|
68
|
+
end
|
|
69
|
+
puts "✓ Seeded #{Product.count} products"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
class Product < ActiveRecord::Base
|
|
73
|
+
has_bm25 :search_text, index: :products_search_idx
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def run
|
|
77
|
+
setup_schema
|
|
78
|
+
seed_data
|
|
79
|
+
|
|
80
|
+
puts "\n=== Prefix Matching (autocomplete) ==="
|
|
81
|
+
# "wire" matches "wireless", "Wire" etc.
|
|
82
|
+
results = Product.bm25_search(:search_text, "wire", index: :products_search_idx, prefix: true).limit(5)
|
|
83
|
+
results.each { |p| puts " #{p.name}" }
|
|
84
|
+
|
|
85
|
+
puts "\n=== Any Word (OR) ==="
|
|
86
|
+
# Matches "sony" OR "bose" OR "wireless"
|
|
87
|
+
results = Product.bm25_search(:search_text, "sony bose wireless", index: :products_search_idx, any_word: true).limit(5)
|
|
88
|
+
results.each { |p| puts " #{p.name}" }
|
|
89
|
+
|
|
90
|
+
puts "\n=== Negation (Exclude Terms) ==="
|
|
91
|
+
# Matches "wireless" but NOT "sony"
|
|
92
|
+
results = Product.bm25_search(:search_text, "wireless !sony", index: :products_search_idx, negation: true).limit(5)
|
|
93
|
+
results.each { |p| puts " #{p.name}" }
|
|
94
|
+
|
|
95
|
+
puts "\n=== Unaccent (Accent Insensitivity) ==="
|
|
96
|
+
if PgTextsearch::Maintenance.unaccent_available?
|
|
97
|
+
puts " unaccent available - would match 'café' when searching 'cafe'"
|
|
98
|
+
else
|
|
99
|
+
puts " unaccent not installed"
|
|
100
|
+
puts " Run: PgTextsearch::Maintenance.enable_unaccent!"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
puts "\n✅ Advanced query options completed!"
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
run if $PROGRAM_NAME == __FILE__
|