pg_textsearch 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/pg_textsearch/maintenance.rb +6 -0
- data/lib/pg_textsearch/model.rb +33 -9
- data/lib/pg_textsearch/version.rb +1 -1
- data/test/issues_test.rb +121 -0
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9935a5e2397bd2e50ac8db9f5bbaab9c7f8aa66d3aeeac61e8ca895729bcc1fd
|
|
4
|
+
data.tar.gz: 6e37334d9eef591e298ff7b942ffcd7ed53d7798cdeb77b80d8c3fc86e32e861
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9605c4ec7df9779b2a3dcf1835d1d5359b759cc738a48c0cb38707366acf930b775b8e15b9b4b0163d29aecf0c3d04f485073c3b586f8afc3496f4bb810f254b
|
|
7
|
+
data.tar.gz: 1f02a31640b879edae64ca0f27009db47b74cd85f445ddc43e0b82d7d636d9565b58920c580a8b8e70460fba2e30158c09fe85a7fa0e4a7d8fdd549abd502b78
|
|
@@ -125,9 +125,15 @@ module PgTextsearch
|
|
|
125
125
|
end
|
|
126
126
|
|
|
127
127
|
def extension_installed?(name)
|
|
128
|
+
return false unless postgresql?
|
|
129
|
+
|
|
128
130
|
connection.select_value(<<~SQL)
|
|
129
131
|
SELECT 1 FROM pg_extension WHERE extname = '#{name}'
|
|
130
132
|
SQL
|
|
131
133
|
end
|
|
134
|
+
|
|
135
|
+
def postgresql?
|
|
136
|
+
connection.adapter_name.downcase.include?("postgres")
|
|
137
|
+
end
|
|
132
138
|
end
|
|
133
139
|
end
|
data/lib/pg_textsearch/model.rb
CHANGED
|
@@ -16,7 +16,8 @@ module PgTextsearch
|
|
|
16
16
|
# Document.bm25_search(:content, "database system", index: :docs_idx,
|
|
17
17
|
# prefix: true, any_word: true, negation: true, unaccent: true,
|
|
18
18
|
# fuzzy: {threshold: 0.3}, synonyms: true,
|
|
19
|
-
# select_score: true,
|
|
19
|
+
# select_score: true, select_score_only: false, highlight: true,
|
|
20
|
+
# ranked_by: "expr", order_within_rank: "col DESC", where: true
|
|
20
21
|
# ).limit(5)
|
|
21
22
|
def bm25_search(target, query, index: nil, **options)
|
|
22
23
|
ensure_pg_textsearch_available!
|
|
@@ -31,14 +32,23 @@ module PgTextsearch
|
|
|
31
32
|
query_sql = bm25_query_sql(query, index, options)
|
|
32
33
|
score_sql = "#{target_sql} <@> #{query_sql}"
|
|
33
34
|
|
|
34
|
-
# Build ORDER BY
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
# Build ORDER BY (unless select_score_only: true)
|
|
36
|
+
if options[:select_score_only]
|
|
37
|
+
relation = all
|
|
38
|
+
else
|
|
39
|
+
order_sql = build_order_sql(score_sql, options)
|
|
40
|
+
relation = order(Arel.sql(order_sql))
|
|
41
|
+
end
|
|
38
42
|
|
|
39
43
|
# Score selection
|
|
40
|
-
if options[:select_score]
|
|
41
|
-
name = options[:select_score] == true
|
|
44
|
+
if options[:select_score] || options[:select_score_only]
|
|
45
|
+
name = if options[:select_score] == true || options[:select_score_only] == true
|
|
46
|
+
"bm25_score"
|
|
47
|
+
elsif options[:select_score]
|
|
48
|
+
options[:select_score].to_s
|
|
49
|
+
else
|
|
50
|
+
options[:select_score_only].to_s
|
|
51
|
+
end
|
|
42
52
|
relation = relation.select(arel_table[Arel.star],
|
|
43
53
|
Arel.sql("#{score_sql} AS #{connection.quote_column_name(name)}"))
|
|
44
54
|
end
|
|
@@ -46,8 +56,10 @@ module PgTextsearch
|
|
|
46
56
|
# Highlighting
|
|
47
57
|
relation = apply_highlight(relation, target, query, index, options[:highlight]) if options[:highlight]
|
|
48
58
|
|
|
49
|
-
# Fuzzy search (pg_trgm) - adds WHERE clause
|
|
50
|
-
|
|
59
|
+
# Fuzzy search (pg_trgm) - adds WHERE clause (unless where: false)
|
|
60
|
+
if options[:fuzzy] && options[:where] != false
|
|
61
|
+
relation = apply_fuzzy(relation, target, query, options[:fuzzy])
|
|
62
|
+
end
|
|
51
63
|
|
|
52
64
|
# Synonyms - uses thesaurus dictionary (already handled via text_config)
|
|
53
65
|
# Just pass through; the index must be created with thesaurus dict
|
|
@@ -55,6 +67,18 @@ module PgTextsearch
|
|
|
55
67
|
relation
|
|
56
68
|
end
|
|
57
69
|
|
|
70
|
+
# Match operator for use in WHERE clauses
|
|
71
|
+
# Usage: scope.where("search_text <@> ? < 0", query)
|
|
72
|
+
def bm25_match(query, index: nil, **options)
|
|
73
|
+
ensure_pg_textsearch_available!
|
|
74
|
+
|
|
75
|
+
index ||= bm25_indexes[target.to_s] if respond_to?(:bm25_indexes)
|
|
76
|
+
query_sql = bm25_query_sql(query, index, options)
|
|
77
|
+
target_sql = bm25_target_sql(target, options[:unaccent])
|
|
78
|
+
|
|
79
|
+
where(Arel.sql("#{target_sql} <@> #{query_sql} < 0"))
|
|
80
|
+
end
|
|
81
|
+
|
|
58
82
|
private
|
|
59
83
|
|
|
60
84
|
def bm25_target_sql(target, unaccent = false)
|
data/test/issues_test.rb
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_record"
|
|
4
|
+
require "pg_textsearch"
|
|
5
|
+
require "minitest/autorun"
|
|
6
|
+
|
|
7
|
+
# Set up a minimal SQLite connection for SQL generation tests
|
|
8
|
+
ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: ":memory:")
|
|
9
|
+
|
|
10
|
+
class IssueProduct < ActiveRecord::Base
|
|
11
|
+
self.table_name = "products"
|
|
12
|
+
has_bm25 :search_text, index: :products_search_idx
|
|
13
|
+
has_bm25 :name, index: :products_name_idx
|
|
14
|
+
has_bm25 "coalesce(name, '') || ' ' || coalesce(description, '')", index: :products_all_idx
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
class IssueProductNoDefault < ActiveRecord::Base
|
|
18
|
+
self.table_name = "products"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
ActiveRecord::Schema.define do
|
|
22
|
+
create_table :products do |t|
|
|
23
|
+
t.text :search_text
|
|
24
|
+
t.text :name
|
|
25
|
+
t.text :description
|
|
26
|
+
t.boolean :active, default: true
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
class PgTextsearchIssuesTest < Minitest::Test
|
|
31
|
+
def setup
|
|
32
|
+
# No database needed for SQL generation tests
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Issue #1: prefix: true doesn't apply to WHERE clause matching
|
|
36
|
+
def test_prefix_applies_to_where_clause
|
|
37
|
+
# When using prefix: true, the WHERE clause should use the prefixed query
|
|
38
|
+
# For SQLite tests, we just verify the query SQL is built correctly
|
|
39
|
+
sql = IssueProductNoDefault.bm25_search(:search_text, "iph", index: :idx, prefix: true).to_sql
|
|
40
|
+
assert_match(/iph:\*/, sql, "prefix: true should add :* to query terms")
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def test_prefix_with_multiple_terms
|
|
44
|
+
sql = IssueProductNoDefault.bm25_search(:search_text, "iphone wireless", index: :idx, prefix: true).to_sql
|
|
45
|
+
assert_match(/iphone:\*/, sql)
|
|
46
|
+
assert_match(/wireless:\*/, sql)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Issue #2: unaccent: true requires manual unaccent() on query parameter
|
|
50
|
+
def test_unaccent_applies_to_query_parameter
|
|
51
|
+
sql = IssueProductNoDefault.bm25_search(:search_text, "café", index: :idx, unaccent: true).to_sql
|
|
52
|
+
assert_match(/unaccent/, sql, "unaccent: true should wrap query in unaccent()")
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Issue #3: fuzzy/negation options add unwanted WHERE conditions
|
|
56
|
+
def test_fuzzy_does_not_add_unwanted_where
|
|
57
|
+
# fuzzy should not add extra WHERE conditions that filter out BM25 results
|
|
58
|
+
# For SQLite tests, we verify the SQL structure
|
|
59
|
+
IssueProductNoDefault.bm25_search(:search_text, "iphone", index: :idx, fuzzy: { threshold: 0.3 }).to_sql
|
|
60
|
+
# Should only have ORDER BY, no extra WHERE from fuzzy
|
|
61
|
+
# The fuzzy logic is only applied when pg_trgm is available
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def test_negation_does_not_add_unwanted_where
|
|
65
|
+
# negation should not add extra WHERE conditions that filter out BM25 results
|
|
66
|
+
IssueProductNoDefault.bm25_search(:search_text, "iphone", index: :idx, negation: true).to_sql
|
|
67
|
+
# Should only have ORDER BY with negation in the query
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Issue #4: Short queries (1-2 chars) return no results due to stop word filtering
|
|
71
|
+
def test_short_query_documentation
|
|
72
|
+
# This is a documentation issue - we should at least not crash
|
|
73
|
+
sql = IssueProductNoDefault.bm25_search(:search_text, "A", index: :idx).to_sql
|
|
74
|
+
assert_match(/ORDER BY/, sql)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Issue #5: Highlighting uses to_tsquery instead of to_bm25query
|
|
78
|
+
def test_highlight_uses_bm25query_for_consistency
|
|
79
|
+
sql = IssueProductNoDefault.bm25_search(:search_text, "iphone", index: :idx, highlight: true).to_sql
|
|
80
|
+
# Should use to_bm25query or at least be consistent with the search query
|
|
81
|
+
assert_match(/ts_headline/, sql)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def test_highlight_with_prefix
|
|
85
|
+
sql = IssueProductNoDefault.bm25_search(:search_text, "iph", index: :idx, prefix: true, highlight: true).to_sql
|
|
86
|
+
assert_match(/ts_headline/, sql)
|
|
87
|
+
# Highlight query should use the same prefix logic
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Issue #6: Partial index WHERE clause not auto-applied in bm25_search
|
|
91
|
+
def test_partial_index_where_clause_not_auto_applied
|
|
92
|
+
# This is a limitation - we can't auto-detect partial index predicates
|
|
93
|
+
# But we can document it and provide a way to specify it
|
|
94
|
+
skip "This requires index metadata lookup - not yet implemented"
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Issue #7: Background compaction requires pg_durable 0.2.8+ (undocumented)
|
|
98
|
+
def test_background_compaction_documentation
|
|
99
|
+
# This is a documentation issue - we should document the requirement
|
|
100
|
+
skip "Documentation issue - verify README mentions pg_durable requirement"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Issue #8: No way to get BM25 score without ordering (select_score_only option)
|
|
104
|
+
def test_select_score_only_option
|
|
105
|
+
# Should be able to get score without ORDER BY
|
|
106
|
+
sql = IssueProductNoDefault.bm25_search(:search_text, "test", index: :idx, select_score_only: true).to_sql
|
|
107
|
+
assert_match(/SELECT.*AS "bm25_score"/, sql)
|
|
108
|
+
refute_match(/ORDER BY/, sql, "select_score_only should not add ORDER BY")
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def test_select_score_only_with_custom_alias
|
|
112
|
+
sql = IssueProductNoDefault.bm25_search(:search_text, "test", index: :idx, select_score_only: :my_score).to_sql
|
|
113
|
+
assert_match(/AS "my_score"/, sql)
|
|
114
|
+
refute_match(/ORDER BY/, sql)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def test_select_score_true_still_adds_order_by
|
|
118
|
+
sql = IssueProductNoDefault.bm25_search(:search_text, "test", index: :idx, select_score: true).to_sql
|
|
119
|
+
assert_match(/ORDER BY/, sql, "select_score: true should still add ORDER BY")
|
|
120
|
+
end
|
|
121
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pg_textsearch
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Aboobacker MK
|
|
@@ -78,6 +78,7 @@ files:
|
|
|
78
78
|
- samples/07_extension_setup.rb
|
|
79
79
|
- sig/pg_textsearch.rbs
|
|
80
80
|
- test/integration_test.rb
|
|
81
|
+
- test/issues_test.rb
|
|
81
82
|
- test/pg_textsearch_test.rb
|
|
82
83
|
homepage: https://github.com/timescale/pg_textsearch
|
|
83
84
|
licenses:
|