pg_textsearch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.devcontainer/devcontainer.json +32 -0
- data/.devcontainer/setup.sh +25 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +514 -0
- data/Rakefile +23 -0
- data/lib/pg_textsearch/maintenance.rb +133 -0
- data/lib/pg_textsearch/model.rb +172 -0
- data/lib/pg_textsearch/schema.rb +53 -0
- data/lib/pg_textsearch/version.rb +5 -0
- data/lib/pg_textsearch.rb +30 -0
- data/samples/01_basic_search.rb +113 -0
- data/samples/02_advanced_query_options.rb +106 -0
- data/samples/03_highlighting.rb +100 -0
- data/samples/04_custom_ranking.rb +110 -0
- data/samples/05_fuzzy_synonyms.rb +112 -0
- data/samples/06_maintenance.rb +111 -0
- data/samples/07_extension_setup.rb +89 -0
- data/sig/pg_textsearch.rbs +4 -0
- data/test/integration_test.rb +125 -0
- data/test/pg_textsearch_test.rb +162 -0
- metadata +107 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: e0c0ef88fb11e3afdfe1c90841580eb8d641b4c9bd1139aee1a10ebab976a325
|
|
4
|
+
data.tar.gz: b4a25e509f37632b5d6001723fc9905d1d8fb708cac9355eeac47ebab0ede0b8
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 31e5fb06824a01e60d1d4f127ec75bff50520e387611786e1e6fea8e886876325fa82dadf39cec6d9a2afa74da5fff07ad84a7d66e21f235328c4f136ac3ade7
|
|
7
|
+
data.tar.gz: adc61b3b9ac2982aaac97be1bd86dbff45d13d77942e1425d53e8ff63e2369c6f4fb4d5882c7e92a4b45d71cede5b6b9b2e4b221fec15fcd126ad64954e31ff5
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "pg_textsearch",
|
|
3
|
+
"image": "mcr.microsoft.com/devcontainers/ruby:3.3-bookworm",
|
|
4
|
+
"features": {
|
|
5
|
+
"ghcr.io/devcontainers/features/postgresql:1": {
|
|
6
|
+
"version": "18",
|
|
7
|
+
"password": "postgres"
|
|
8
|
+
}
|
|
9
|
+
},
|
|
10
|
+
"postCreateCommand": "bash .devcontainer/setup.sh",
|
|
11
|
+
"customizations": {
|
|
12
|
+
"vscode": {
|
|
13
|
+
"extensions": [
|
|
14
|
+
"shopify.ruby-lsp",
|
|
15
|
+
"rubocop.vscode-rubocop",
|
|
16
|
+
"ms-azuretools.vscode-docker"
|
|
17
|
+
],
|
|
18
|
+
"settings": {
|
|
19
|
+
"ruby.lint": {
|
|
20
|
+
"rubocop": true
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
"forwardPorts": [5432],
|
|
26
|
+
"remoteEnv": {
|
|
27
|
+
"PGHOST": "localhost",
|
|
28
|
+
"PGUSER": "postgres",
|
|
29
|
+
"PGPASSWORD": "postgres",
|
|
30
|
+
"PGDATABASE": "postgres"
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
# Install pg_textsearch extension from apt (if available) or build from source
|
|
5
|
+
# Timescale provides apt repos for pg_textsearch
|
|
6
|
+
|
|
7
|
+
echo "deb https://packagecloud.io/timescale/timescaledb/debian/ bookworm main" | sudo tee /etc/apt/sources.list.d/timescaledb.list
|
|
8
|
+
curl -fsSL https://packagecloud.io/timescale/timescaledb/gpgkey | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/timescaledb.gpg
|
|
9
|
+
sudo apt-get update
|
|
10
|
+
sudo apt-get install -y postgresql-18-pg-textsearch
|
|
11
|
+
|
|
12
|
+
# Configure PostgreSQL to load pg_textsearch
|
|
13
|
+
echo "shared_preload_libraries = 'pg_textsearch'" | sudo tee -a /etc/postgresql/18/main/postgresql.conf
|
|
14
|
+
sudo systemctl restart postgresql
|
|
15
|
+
|
|
16
|
+
# Create extension and test database
|
|
17
|
+
sleep 3
|
|
18
|
+
sudo -u postgres psql -c "CREATE EXTENSION pg_textsearch;"
|
|
19
|
+
sudo -u postgres psql -c "CREATE DATABASE pg_textsearch_test;"
|
|
20
|
+
|
|
21
|
+
# Install Ruby dependencies
|
|
22
|
+
bundle install
|
|
23
|
+
|
|
24
|
+
echo "Dev container setup complete!"
|
|
25
|
+
echo "Run tests with: bundle exec rake test"
|
data/CHANGELOG.md
ADDED
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Aboobacker MK
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,514 @@
|
|
|
1
|
+
# PgTextsearch
|
|
2
|
+
|
|
3
|
+
Minimal ActiveRecord wrapper for [pg_textsearch](https://github.com/timescale/pg_textsearch) BM25 indexes.
|
|
4
|
+
|
|
5
|
+
Follows the [`neighbor`](https://github.com/ankane/neighbor) philosophy: **wrap only the awkward SQL bits** (`<@>` operator, `to_bm25query()`, and `WITH (...)` index options) and return standard `ActiveRecord::Relation` objects so `.where`, `.limit`, `.includes`, `.select`, `.group`, `.joins` all work unchanged.
|
|
6
|
+
|
|
7
|
+
## Requirements
|
|
8
|
+
|
|
9
|
+
- PostgreSQL 17+ with `pg_textsearch` extension
|
|
10
|
+
- `shared_preload_libraries = 'pg_textsearch'` in `postgresql.conf`
|
|
11
|
+
- Ruby 3.2+
|
|
12
|
+
- ActiveRecord 7.0+
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
bundle add pg_textsearch
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Setup
|
|
21
|
+
|
|
22
|
+
1. Add to `postgresql.conf`:
|
|
23
|
+
```
|
|
24
|
+
shared_preload_libraries = 'pg_textsearch'
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
2. Restart PostgreSQL
|
|
28
|
+
|
|
29
|
+
3. Enable the extension in your database:
|
|
30
|
+
```sql
|
|
31
|
+
CREATE EXTENSION pg_textsearch;
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## Core API
|
|
37
|
+
|
|
38
|
+
```ruby
|
|
39
|
+
# Model declaration (infers index automatically)
|
|
40
|
+
class Product < ApplicationRecord
|
|
41
|
+
has_bm25 :name, index: :products_name_idx
|
|
42
|
+
has_bm25 :description, index: :products_desc_idx
|
|
43
|
+
has_bm25 "coalesce(name, '') || ' ' || coalesce(description, '')", index: :products_all_idx
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Basic search
|
|
47
|
+
Product.bm25_search(:name, "wireless headphones").limit(10)
|
|
48
|
+
|
|
49
|
+
# With explicit index (required for partial indexes, prepared statements)
|
|
50
|
+
Product.bm25_search(:description, "noise cancelling", index: :products_desc_idx).limit(10)
|
|
51
|
+
|
|
52
|
+
# Compose with standard ActiveRecord
|
|
53
|
+
Product
|
|
54
|
+
.where(active: true, category: "electronics")
|
|
55
|
+
.bm25_search(:name, "wireless headphones")
|
|
56
|
+
.limit(20)
|
|
57
|
+
|
|
58
|
+
# Expose BM25 score (raw negative value, more negative = better match)
|
|
59
|
+
Product.bm25_search(:name, "headphones", select_score: true).limit(5)
|
|
60
|
+
# product.bm25_score # => -2.34
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
## Query Options
|
|
66
|
+
|
|
67
|
+
All options can be passed per-query or as defaults in `has_bm25`:
|
|
68
|
+
|
|
69
|
+
```ruby
|
|
70
|
+
class Product < ApplicationRecord
|
|
71
|
+
has_bm25 :search_text, index: :products_idx,
|
|
72
|
+
unaccent: true, # default: accent-insensitive
|
|
73
|
+
prefix: true # default: prefix matching
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Per-query overrides
|
|
77
|
+
Product.bm25_search(:search_text, "headphones", index: :products_idx,
|
|
78
|
+
prefix: true, # "headphones*" - autocomplete style
|
|
79
|
+
any_word: true, # "headphones OR wireless" (default: AND)
|
|
80
|
+
negation: true, # "headphones !sony" - exclude terms
|
|
81
|
+
unaccent: true, # "cafe" matches "café"
|
|
82
|
+
fuzzy: {threshold: 0.3},# typo tolerance via pg_trgm
|
|
83
|
+
synonyms: true, # "pop" matches "soda" (requires thesaurus dict)
|
|
84
|
+
highlight: true, # adds pg_search_highlight attribute
|
|
85
|
+
ranked_by: ":bm25 + 0.1 * products.popularity",
|
|
86
|
+
order_within_rank: "products.updated_at DESC"
|
|
87
|
+
)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### Option Reference
|
|
91
|
+
|
|
92
|
+
| Option | Type | Description |
|
|
93
|
+
|--------|------|-------------|
|
|
94
|
+
| `index` | Symbol/String | Explicit index name (required for partial indexes, prepared statements) |
|
|
95
|
+
| `select_score` | Boolean/Symbol | Expose BM25 score as `bm25_score` or custom alias |
|
|
96
|
+
| `prefix` | Boolean | Append `:*` to each word for prefix matching |
|
|
97
|
+
| `any_word` | Boolean | Match ANY term (OR) instead of ALL (AND) |
|
|
98
|
+
| `negation` | Boolean | Support `!term` exclusion syntax |
|
|
99
|
+
| `unaccent` | Boolean | Accent-insensitive search (requires `unaccent` extension) |
|
|
100
|
+
| `fuzzy` | Hash/Boolean | Typo tolerance via `pg_trgm` similarity (`threshold: 0.3`) |
|
|
101
|
+
| `synonyms` | Boolean | Use thesaurus dictionary (requires `dict_thesaurus` + thesaurus file) |
|
|
102
|
+
| `highlight` | Boolean/Hash | Add `pg_search_highlight` via `ts_headline` |
|
|
103
|
+
| `ranked_by` | String | Custom ranking expression; `:bm25` replaced with score SQL |
|
|
104
|
+
| `order_within_rank` | String | Tiebreaker ORDER BY (e.g., `updated_at DESC`) |
|
|
105
|
+
|
|
106
|
+
### Highlight Options
|
|
107
|
+
|
|
108
|
+
```ruby
|
|
109
|
+
highlight: {
|
|
110
|
+
start_sel: "<mark>", # default: "<b>"
|
|
111
|
+
stop_sel: "</mark>", # default: "</b>"
|
|
112
|
+
max_words: 35, # default: 35
|
|
113
|
+
min_words: 15, # default: 15
|
|
114
|
+
short_word: 3, # default: 3
|
|
115
|
+
highlight_all: false, # default: false
|
|
116
|
+
max_fragments: 0, # default: 0 (no fragmentation)
|
|
117
|
+
fragment_delimiter: "..." # default: "..."
|
|
118
|
+
}
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
---
|
|
122
|
+
|
|
123
|
+
## Common Use Cases
|
|
124
|
+
|
|
125
|
+
### 1. E-commerce Product Search
|
|
126
|
+
|
|
127
|
+
```ruby
|
|
128
|
+
# Migration: multi-column search index with custom BM25 params
|
|
129
|
+
class AddProductSearchIndex < ActiveRecord::Migration[8.0]
|
|
130
|
+
def change
|
|
131
|
+
enable_extension "pg_textsearch"
|
|
132
|
+
|
|
133
|
+
# Combined name + description + tags index
|
|
134
|
+
execute <<~SQL
|
|
135
|
+
CREATE INDEX products_search_idx ON products
|
|
136
|
+
USING bm25 (
|
|
137
|
+
(coalesce(name, '') || ' ' || coalesce(description, '') || ' ' || coalesce(tags, ''))
|
|
138
|
+
)
|
|
139
|
+
WITH (text_config = 'english', k1 = 1.5, b = 0.75);
|
|
140
|
+
SQL
|
|
141
|
+
|
|
142
|
+
# Partial index for active products only
|
|
143
|
+
execute <<~SQL
|
|
144
|
+
CREATE INDEX products_active_search_idx ON products
|
|
145
|
+
USING bm25 (
|
|
146
|
+
(coalesce(name, '') || ' ' || coalesce(description, ''))
|
|
147
|
+
)
|
|
148
|
+
WITH (text_config = 'english', k1 = 1.5, b = 0.75)
|
|
149
|
+
WHERE active = true;
|
|
150
|
+
SQL
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# Model
|
|
155
|
+
class Product < ApplicationRecord
|
|
156
|
+
has_bm25 "coalesce(name, '') || ' ' || coalesce(description, '') || ' ' || coalesce(tags, '')",
|
|
157
|
+
index: :products_search_idx
|
|
158
|
+
has_bm25 :name, index: :products_active_search_idx # partial index
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Search endpoint
|
|
162
|
+
def search
|
|
163
|
+
scope = Product.where(active: true)
|
|
164
|
+
|
|
165
|
+
scope = scope.where(category_id: params[:category]) if params[:category].present?
|
|
166
|
+
scope = scope.where("price BETWEEN ? AND ?", params[:min_price], params[:max_price]) if params[:min_price]
|
|
167
|
+
scope = scope.where(brand_id: params[:brand]) if params[:brand].present?
|
|
168
|
+
|
|
169
|
+
results = scope.bm25_search(
|
|
170
|
+
"coalesce(name, '') || ' ' || coalesce(description, '') || ' ' || coalesce(tags, '')",
|
|
171
|
+
params[:q],
|
|
172
|
+
index: :products_search_idx,
|
|
173
|
+
select_score: true,
|
|
174
|
+
prefix: true,
|
|
175
|
+
unaccent: true
|
|
176
|
+
).limit(params[:limit] || 20)
|
|
177
|
+
|
|
178
|
+
render json: results.as_json(methods: :bm25_score)
|
|
179
|
+
end
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### 2. Faceted Search with Aggregates
|
|
183
|
+
|
|
184
|
+
```ruby
|
|
185
|
+
def search_with_facets
|
|
186
|
+
base = Product.where(active: true)
|
|
187
|
+
.bm25_search(:search_text, params[:q], index: :products_search_idx)
|
|
188
|
+
|
|
189
|
+
# Results with score
|
|
190
|
+
@products = base.select_score(true).limit(20)
|
|
191
|
+
|
|
192
|
+
# Facets via standard ActiveRecord aggregations
|
|
193
|
+
@facets = {
|
|
194
|
+
categories: Product.where(active: true).group(:category).count,
|
|
195
|
+
brands: Product.where(active: true).group(:brand).count,
|
|
196
|
+
price_ranges: Product.where(active: true).group(
|
|
197
|
+
Arel.sql("CASE
|
|
198
|
+
WHEN price < 50 THEN 'Under $50'
|
|
199
|
+
WHEN price < 100 THEN '$50-$100'
|
|
200
|
+
WHEN price < 200 THEN '$100-$200'
|
|
201
|
+
ELSE '$200+'
|
|
202
|
+
END")
|
|
203
|
+
).count,
|
|
204
|
+
ratings: Product.where(active: true).group(:rating).count
|
|
205
|
+
}
|
|
206
|
+
end
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### 3. Related Products (Content-Based)
|
|
210
|
+
|
|
211
|
+
```ruby
|
|
212
|
+
class Product < ApplicationRecord
|
|
213
|
+
# Find products with similar descriptions
|
|
214
|
+
def related_products(limit: 5)
|
|
215
|
+
return Product.none if description.blank?
|
|
216
|
+
|
|
217
|
+
Product.where(active: true)
|
|
218
|
+
.where.not(id: id)
|
|
219
|
+
.bm25_search(:description, description, index: :products_desc_idx)
|
|
220
|
+
.limit(limit)
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# Find products sharing tags (using JSONB expression index)
|
|
224
|
+
# Migration: CREATE INDEX products_tags_idx ON products USING bm25 ((tags)) WITH (text_config='simple');
|
|
225
|
+
def related_by_tags(limit: 5)
|
|
226
|
+
Product.where(active: true)
|
|
227
|
+
.where.not(id: id)
|
|
228
|
+
.bm25_search("tags", tags.to_json, index: :products_tags_idx)
|
|
229
|
+
.limit(limit)
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
### 4. Multilingual Search
|
|
235
|
+
|
|
236
|
+
```ruby
|
|
237
|
+
# Migration: one partial index per language
|
|
238
|
+
class AddMultilingualIndexes < ActiveRecord::Migration[8.0]
|
|
239
|
+
def change
|
|
240
|
+
enable_extension "pg_textsearch"
|
|
241
|
+
|
|
242
|
+
execute <<~SQL
|
|
243
|
+
CREATE INDEX products_en_idx ON products USING bm25 (
|
|
244
|
+
(coalesce(name, '') || ' ' || coalesce(description, ''))
|
|
245
|
+
) WITH (text_config = 'english') WHERE locale = 'en';
|
|
246
|
+
SQL
|
|
247
|
+
|
|
248
|
+
execute <<~SQL
|
|
249
|
+
CREATE INDEX products_de_idx ON products USING bm25 (
|
|
250
|
+
(coalesce(name, '') || ' ' || coalesce(description, ''))
|
|
251
|
+
) WITH (text_config = 'german') WHERE locale = 'de';
|
|
252
|
+
SQL
|
|
253
|
+
|
|
254
|
+
execute <<~SQL
|
|
255
|
+
CREATE INDEX products_fr_idx ON products USING bm25 (
|
|
256
|
+
(coalesce(name, '') || ' ' || coalesce(description, ''))
|
|
257
|
+
) WITH (text_config = 'french') WHERE locale = 'fr';
|
|
258
|
+
SQL
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
class Product < ApplicationRecord
|
|
263
|
+
has_bm25 :search_text, index: :products_en_idx # default fallback
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def search
|
|
267
|
+
locale = params[:locale] || I18n.locale.to_s
|
|
268
|
+
index_name = "products_#{locale}_idx".to_sym
|
|
269
|
+
|
|
270
|
+
Product.where(locale: locale, active: true)
|
|
271
|
+
.bm25_search(:search_text, params[:q], index: index_name)
|
|
272
|
+
.limit(20)
|
|
273
|
+
end
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
### 5. Hybrid Search (BM25 + Vector)
|
|
277
|
+
|
|
278
|
+
```ruby
|
|
279
|
+
# Requires pgvector extension
|
|
280
|
+
# Simple linear combination of BM25 score + vector distance
|
|
281
|
+
def hybrid_search(query_text, query_embedding, limit: 10)
|
|
282
|
+
bm25_sql = "search_text <@> to_bm25query(?, 'products_search_idx')"
|
|
283
|
+
vector_sql = "embedding <=> ?"
|
|
284
|
+
|
|
285
|
+
Product.where(active: true)
|
|
286
|
+
.select("products.*, #{bm25_sql} AS bm25_score, #{vector_sql} AS vector_distance")
|
|
287
|
+
.order(Arel.sql("#{bm25_sql} + #{vector_sql}"))
|
|
288
|
+
.limit(limit)
|
|
289
|
+
end
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
```ruby
|
|
293
|
+
# RRF (Reciprocal Rank Fusion) using ActiveRecord CTE (Rails 6.1+)
|
|
294
|
+
def hybrid_search_rrf(query_text, query_embedding, limit: 10)
|
|
295
|
+
k = 60 # RRF constant
|
|
296
|
+
|
|
297
|
+
bm25_cte = Product.select(:id)
|
|
298
|
+
.select(Arel.sql("ROW_NUMBER() OVER (ORDER BY search_text <@> to_bm25query(?, 'products_search_idx')) AS rank"))
|
|
299
|
+
.where(active: true)
|
|
300
|
+
.limit(20)
|
|
301
|
+
|
|
302
|
+
vec_cte = Product.select(:id)
|
|
303
|
+
.select(Arel.sql("ROW_NUMBER() OVER (ORDER BY embedding <=> ?) AS rank"))
|
|
304
|
+
.where(active: true)
|
|
305
|
+
.limit(20)
|
|
306
|
+
|
|
307
|
+
Product.with(bm25: bm25_cte, vec: vec_cte)
|
|
308
|
+
.select("products.*, COALESCE(1.0/(#{k}+b.rank),0) + COALESCE(1.0/(#{k}+v.rank),0) AS rrf_score")
|
|
309
|
+
.joins("LEFT JOIN bm25 b ON b.id = products.id")
|
|
310
|
+
.joins("LEFT JOIN vec v ON v.id = products.id")
|
|
311
|
+
.where(active: true)
|
|
312
|
+
.order("rrf_score DESC")
|
|
313
|
+
.limit(limit)
|
|
314
|
+
end
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
### 6. Autocomplete / Prefix Search
|
|
318
|
+
|
|
319
|
+
```ruby
|
|
320
|
+
# Built-in prefix option
|
|
321
|
+
def autocomplete(query, limit: 10)
|
|
322
|
+
Product.where(active: true)
|
|
323
|
+
.bm25_search(:name, query, index: :products_name_idx, prefix: true)
|
|
324
|
+
.limit(limit)
|
|
325
|
+
end
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
### 7. Search Analytics / Logging
|
|
329
|
+
|
|
330
|
+
```ruby
|
|
331
|
+
class SearchLog < ApplicationRecord
|
|
332
|
+
def self.log(query, user_id, results_count, latency_ms)
|
|
333
|
+
create!(query: query, user_id: user_id, results_count: results_count, latency_ms: latency_ms)
|
|
334
|
+
end
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
def search
|
|
338
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
339
|
+
|
|
340
|
+
results = Product.where(active: true)
|
|
341
|
+
.bm25_search(:search_text, params[:q], index: :products_search_idx)
|
|
342
|
+
.limit(20)
|
|
343
|
+
|
|
344
|
+
latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000).round
|
|
345
|
+
SearchLog.log(params[:q], current_user&.id, results.size, latency_ms)
|
|
346
|
+
|
|
347
|
+
render json: results
|
|
348
|
+
end
|
|
349
|
+
```
|
|
350
|
+
|
|
351
|
+
### 8. Admin: Index Maintenance Rake Tasks
|
|
352
|
+
|
|
353
|
+
```ruby
|
|
354
|
+
# lib/tasks/pg_textsearch.rake
|
|
355
|
+
namespace :pg_textsearch do
|
|
356
|
+
desc "Check if indexes need compaction"
|
|
357
|
+
task check_compaction: :environment do
|
|
358
|
+
indexes = %w[products_search_idx products_name_idx products_desc_idx]
|
|
359
|
+
indexes.each do |idx|
|
|
360
|
+
needs = PgTextsearch::Maintenance.needs_compaction?(idx)
|
|
361
|
+
levels = PgTextsearch::Maintenance.level_counts(idx)
|
|
362
|
+
puts "#{idx}: needs_compaction=#{needs} levels=#{levels}"
|
|
363
|
+
end
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
desc "Compact all indexes"
|
|
367
|
+
task compact: :environment do
|
|
368
|
+
indexes = %w[products_search_idx products_name_idx products_desc_idx]
|
|
369
|
+
indexes.each do |idx|
|
|
370
|
+
puts "Compacting #{idx}..."
|
|
371
|
+
PgTextsearch::Maintenance.compact(idx)
|
|
372
|
+
end
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
desc "Run one compaction step per index (safe for cron)"
|
|
376
|
+
task compact_step: :environment do
|
|
377
|
+
indexes = %w[products_search_idx products_name_idx products_desc_idx]
|
|
378
|
+
indexes.each do |idx|
|
|
379
|
+
ran = PgTextsearch::Maintenance.compact_step(idx)
|
|
380
|
+
puts "#{idx}: #{ran ? 'ran step' : 'nothing to do'}"
|
|
381
|
+
end
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
desc "Spill memtables to disk (after bulk load)"
|
|
385
|
+
task spill: :environment do
|
|
386
|
+
indexes = %w[products_search_idx products_name_idx products_desc_idx]
|
|
387
|
+
indexes.each do |idx|
|
|
388
|
+
pages = PgTextsearch::Maintenance.spill_index(idx)
|
|
389
|
+
puts "#{idx}: spilled #{pages} pages"
|
|
390
|
+
end
|
|
391
|
+
end
|
|
392
|
+
end
|
|
393
|
+
```
|
|
394
|
+
|
|
395
|
+
```bash
|
|
396
|
+
# Run via cron (every 5 min)
|
|
397
|
+
*/5 * * * * cd /app && bundle exec rake pg_textsearch:compact_step
|
|
398
|
+
|
|
399
|
+
# After bulk import
|
|
400
|
+
bundle exec rake pg_textsearch:spill
|
|
401
|
+
```
|
|
402
|
+
|
|
403
|
+
---
|
|
404
|
+
|
|
405
|
+
## Optional Extensions
|
|
406
|
+
|
|
407
|
+
Enable these PostgreSQL extensions for advanced features:
|
|
408
|
+
|
|
409
|
+
```ruby
|
|
410
|
+
# Accent-insensitive search
|
|
411
|
+
PgTextsearch::Maintenance.enable_unaccent!
|
|
412
|
+
# Creates unaccent extension + IMMUTABLE wrapper function
|
|
413
|
+
|
|
414
|
+
# Fuzzy/prefix matching (pg_trgm)
|
|
415
|
+
PgTextsearch::Maintenance.enable_pg_trgm!
|
|
416
|
+
# Then use: fuzzy: {threshold: 0.3} in bm25_search
|
|
417
|
+
|
|
418
|
+
# Synonyms (thesaurus dictionary)
|
|
419
|
+
PgTextsearch::Maintenance.enable_thesaurus!('thesaurus_en', 'thesaurus_en')
|
|
420
|
+
# Requires thesaurus_en.ths file in $SHAREDIR/tsearch_data/
|
|
421
|
+
# Example thesaurus file content:
|
|
422
|
+
# pop, soda, cola
|
|
423
|
+
# headphones, earphones, cans
|
|
424
|
+
# laptop, notebook
|
|
425
|
+
```
|
|
426
|
+
|
|
427
|
+
---
|
|
428
|
+
|
|
429
|
+
## Score Selection
|
|
430
|
+
|
|
431
|
+
By default, only `ORDER BY` is added. Add `select_score: true` to expose the raw negative BM25 score:
|
|
432
|
+
|
|
433
|
+
```ruby
|
|
434
|
+
products = Product.bm25_search(:search_text, "headphones", index: :products_search_idx, select_score: true).limit(5)
|
|
435
|
+
products.first.bm25_score # => -2.34 (more negative = better match)
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
Custom alias:
|
|
439
|
+
|
|
440
|
+
```ruby
|
|
441
|
+
products = Product.bm25_search(:search_text, "query", index: :idx, select_score: :relevance)
|
|
442
|
+
products.first.relevance
|
|
443
|
+
```
|
|
444
|
+
|
|
445
|
+
---
|
|
446
|
+
|
|
447
|
+
## Migrations & `schema.rb` Support
|
|
448
|
+
|
|
449
|
+
```ruby
|
|
450
|
+
class AddSearchIndexes < ActiveRecord::Migration[8.0]
|
|
451
|
+
def change
|
|
452
|
+
enable_extension "pg_textsearch"
|
|
453
|
+
|
|
454
|
+
# Main search index with custom BM25 params
|
|
455
|
+
add_index :products, :search_text,
|
|
456
|
+
name: :products_search_idx,
|
|
457
|
+
using: :bm25,
|
|
458
|
+
with: { text_config: "english", k1: 1.5, b: 0.75 }
|
|
459
|
+
|
|
460
|
+
# Partial index for active products
|
|
461
|
+
add_index :products, :search_text,
|
|
462
|
+
name: :products_active_idx,
|
|
463
|
+
using: :bm25,
|
|
464
|
+
where: "active = true",
|
|
465
|
+
with: { text_config: "english", k1: 1.5, compaction: "background" }
|
|
466
|
+
|
|
467
|
+
# Expression index for JSONB tags
|
|
468
|
+
execute <<~SQL
|
|
469
|
+
CREATE INDEX products_tags_idx ON products USING bm25 ((tags))
|
|
470
|
+
WITH (text_config = 'simple');
|
|
471
|
+
SQL
|
|
472
|
+
end
|
|
473
|
+
end
|
|
474
|
+
```
|
|
475
|
+
|
|
476
|
+
The `with:` options are dumped to `schema.rb` automatically.
|
|
477
|
+
|
|
478
|
+
---
|
|
479
|
+
|
|
480
|
+
## Maintenance Helpers
|
|
481
|
+
|
|
482
|
+
For Rake tasks or background jobs:
|
|
483
|
+
|
|
484
|
+
```ruby
|
|
485
|
+
PgTextsearch::Maintenance.needs_compaction?(:products_search_idx) # => true / false
|
|
486
|
+
PgTextsearch::Maintenance.compact(:products_search_idx) # full compaction
|
|
487
|
+
PgTextsearch::Maintenance.compact_step(:products_search_idx) # one pass (cron-safe)
|
|
488
|
+
PgTextsearch::Maintenance.spill_index(:products_search_idx) # force memtable spill
|
|
489
|
+
PgTextsearch::Maintenance.level_counts(:products_search_idx) # => [2, 1, 0, ...]
|
|
490
|
+
|
|
491
|
+
# Optional extension helpers
|
|
492
|
+
PgTextsearch::Maintenance.enable_unaccent!
|
|
493
|
+
PgTextsearch::Maintenance.enable_pg_trgm!
|
|
494
|
+
PgTextsearch::Maintenance.enable_thesaurus!('thesaurus_en', 'thesaurus_en')
|
|
495
|
+
```
|
|
496
|
+
|
|
497
|
+
---
|
|
498
|
+
|
|
499
|
+
## What's Intentionally Left Out
|
|
500
|
+
|
|
501
|
+
To keep the gem under ~300 lines and avoid fighting ActiveRecord:
|
|
502
|
+
|
|
503
|
+
- **No Boolean `tsquery` DSL** — Standard `.where("content @@ to_tsquery(?, ?)", "english", query)` is already concise.
|
|
504
|
+
- **No automatic score negation** — `pg_textsearch` returns negative BM25 scores for ascending index scans. Keep the raw value so `ORDER BY` behavior matches `EXPLAIN` output and Postgres docs.
|
|
505
|
+
- **No automatic `LIMIT` injection** — Let callers chain `.limit(n)` explicitly so they understand how Block-Max WAND and `pg_textsearch.default_limit` behave.
|
|
506
|
+
- **No `pg_durable` orchestration** — Treat `compaction: "background"` as a pass-through index option, not a gem-managed worker system.
|
|
507
|
+
- **No multi-search/global index** — Different architecture; use separate models + UNION if needed.
|
|
508
|
+
- **No automatic synonym expansion** — Requires manual thesaurus dictionary setup.
|
|
509
|
+
|
|
510
|
+
---
|
|
511
|
+
|
|
512
|
+
## License
|
|
513
|
+
|
|
514
|
+
MIT License. See LICENSE.txt.
|
data/Rakefile
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bundler/gem_tasks"
|
|
4
|
+
require "rubocop/rake_task"
|
|
5
|
+
require "rake/testtask"
|
|
6
|
+
|
|
7
|
+
RuboCop::RakeTask.new
|
|
8
|
+
|
|
9
|
+
Rake::TestTask.new(:test) do |t|
|
|
10
|
+
t.libs << "test"
|
|
11
|
+
t.libs << "lib"
|
|
12
|
+
t.test_files = FileList["test/pg_textsearch_test.rb"]
|
|
13
|
+
t.verbose = false
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
Rake::TestTask.new("test:integration") do |t|
|
|
17
|
+
t.libs << "test"
|
|
18
|
+
t.libs << "lib"
|
|
19
|
+
t.test_files = FileList["test/integration_test.rb"]
|
|
20
|
+
t.verbose = false
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
task default: %i[rubocop test]
|