full_search 0.3.7 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +26 -0
- data/lib/full_search/backfill_job.rb +1 -0
- data/lib/full_search/callbacks.rb +31 -13
- data/lib/full_search/config.rb +5 -3
- data/lib/full_search/constants.rb +12 -0
- data/lib/full_search/dsl.rb +14 -4
- data/lib/full_search/highlighter.rb +3 -3
- data/lib/full_search/index.rb +52 -34
- data/lib/full_search/instrumentation.rb +9 -0
- data/lib/full_search/schema_dumper_patch.rb +2 -0
- data/lib/full_search/search.rb +41 -36
- data/lib/full_search/version.rb +1 -1
- data/lib/full_search.rb +8 -2
- data/lib/generators/full_search/install/templates/full_search.rb +4 -0
- data/lib/tasks/full_search.rake +4 -4
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d8865ec65e8e55ecc1648ab56cd7f65f77ea7a05b6928f0771492a1e2ed1bf68
|
|
4
|
+
data.tar.gz: 8bbe570d9fa72cd9ed8d567174501807bc4b4d1726fcc4600e857219c3d59b2c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1d91fea14e22584bca2ecdd617119068a54ef60d3f8d79d9109c5e51a6f8b4cd562f0a37e34a3e0ce01ec4dda29ebe5e0ef85d7ef7a51a8d5a922efa59086fdc
|
|
7
|
+
data.tar.gz: 2089966acfa5bc274cde458cb27a7a739e70493d75027b507794ffa905048f772f2e72d8a213c8ad779f312424c8509181cf57ae1aeaccf252a9522e690099a4
|
data/README.md
CHANGED
|
@@ -193,6 +193,20 @@ end
|
|
|
193
193
|
|
|
194
194
|
### In production
|
|
195
195
|
|
|
196
|
+
**`auto_rebuild_schema`** — Must be `false` in production. The generated initializer defaults to `Rails.env.local?`, so it's already off. If enabled, every process (web, worker, console) attempts schema rebuilds on boot, which is unnecessary and can cause issues during zero-downtime deploys where old and new processes overlap.
|
|
197
|
+
|
|
198
|
+
**`auto_rebuild_on_stale_query`** — Must be `false` in production. A query-time rebuild is dangerous — it can cause timeouts or race conditions under load. The generated initializer defaults to `Rails.env.local?`, keeping it off in production.
|
|
199
|
+
|
|
200
|
+
**`lock_rebuilds`** — This option uses a Ruby `Mutex` and only prevents concurrent rebuilds within the same process. It does **not** coordinate across processes or hosts. Multi-process or multi-host deployments must run `full_search:rebuild` from a single deployment step.
|
|
201
|
+
|
|
202
|
+
**Docker entrypoint** — For containerized environments, add `full_search:prepare` after `db:prepare:with_data`:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
./bin/rails full_search:prepare
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
`full_search:prepare` is idempotent — it only creates missing FTS tables and installs triggers, making it safe to run on every deploy.
|
|
209
|
+
|
|
196
210
|
Auto-rebuild runs on every Rails process boot (web, worker, console). For zero-downtime deploys where old processes still serve traffic, or if you prefer explicit control, set `auto_rebuild_schema` to `false` and run the rebuild task manually:
|
|
197
211
|
|
|
198
212
|
```bash
|
|
@@ -209,6 +223,18 @@ The rebuild task checks each model's stored config hash against the current DSL
|
|
|
209
223
|
bin/rails full_search:reset
|
|
210
224
|
```
|
|
211
225
|
|
|
226
|
+
### Schema dump
|
|
227
|
+
|
|
228
|
+
By default, `full_search` includes its FTS virtual tables in `db/schema.rb` so that `db:schema:load` produces a complete, loadable database. To exclude them:
|
|
229
|
+
|
|
230
|
+
```ruby
|
|
231
|
+
FullSearch.configure do |config|
|
|
232
|
+
config.dump_schema_virtual_tables = false
|
|
233
|
+
end
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
When disabled, you must run `bin/rails full_search:prepare` after every `db:schema:load` to recreate the indexes.
|
|
237
|
+
|
|
212
238
|
### Rake tasks
|
|
213
239
|
|
|
214
240
|
| Task | Description |
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "full_search/errors"
|
|
4
|
+
|
|
3
5
|
module FullSearch
|
|
4
6
|
module Callbacks
|
|
5
7
|
def self.install!(model)
|
|
@@ -64,34 +66,35 @@ module FullSearch
|
|
|
64
66
|
|
|
65
67
|
value = FullSearch::Model.evaluate_source(record, field)
|
|
66
68
|
table = qt(FullSearch::Index.fts_table_name(record.class))
|
|
67
|
-
conn =
|
|
69
|
+
conn = connection_for(record.class)
|
|
68
70
|
conn.execute(
|
|
69
71
|
"UPDATE #{table} SET #{qc(field.as || field.name)} = #{q(value.to_s)} WHERE rowid = #{q(record.id)}"
|
|
70
72
|
)
|
|
71
|
-
rescue => e
|
|
72
|
-
|
|
73
|
+
rescue ActiveRecord::StatementInvalid => e
|
|
74
|
+
raise_missing_table_or_original(e, record.class)
|
|
73
75
|
end
|
|
74
76
|
|
|
75
77
|
def self.remove_record!(record)
|
|
76
78
|
table = qt(FullSearch::Index.fts_table_name(record.class))
|
|
77
|
-
conn =
|
|
79
|
+
conn = connection_for(record.class)
|
|
78
80
|
conn.execute("DELETE FROM #{table} WHERE rowid = #{q(record.id)}")
|
|
79
|
-
rescue => e
|
|
80
|
-
|
|
81
|
+
rescue ActiveRecord::StatementInvalid => e
|
|
82
|
+
raise_missing_table_or_original(e, record.class)
|
|
81
83
|
end
|
|
82
84
|
|
|
83
85
|
def self.reindex_dependents!(parent_record, dependent_model, field)
|
|
84
86
|
fk = association_key(dependent_model, field.reindex_on)
|
|
85
|
-
conn =
|
|
87
|
+
conn = connection_for(dependent_model)
|
|
86
88
|
sql = "SELECT id FROM #{qt(dependent_model.table_name)} WHERE #{qc(fk)} = #{q(parent_record.id)}"
|
|
87
89
|
dependent_ids = conn.execute(sql).map { |r| r["id"] }
|
|
88
90
|
|
|
89
|
-
dependent_ids.
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
91
|
+
return if dependent_ids.empty?
|
|
92
|
+
|
|
93
|
+
if field.async
|
|
94
|
+
dependent_ids.each { |id| FullSearch::ReindexJob.perform_later(dependent_model.name, id, field.name) }
|
|
95
|
+
else
|
|
96
|
+
dependent_model.where(id: dependent_ids).find_each(batch_size: 500) do |record|
|
|
97
|
+
reindex_field!(record, field.name)
|
|
95
98
|
end
|
|
96
99
|
end
|
|
97
100
|
end
|
|
@@ -113,5 +116,20 @@ module FullSearch
|
|
|
113
116
|
reflection = model.reflect_on_association(association_name.to_sym)
|
|
114
117
|
reflection&.foreign_key&.to_s || "#{association_name}_id"
|
|
115
118
|
end
|
|
119
|
+
|
|
120
|
+
def self.connection_for(klass)
|
|
121
|
+
klass.connection
|
|
122
|
+
rescue NoMethodError
|
|
123
|
+
ActiveRecord::Base.connection
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def self.raise_missing_table_or_original(error, model_class)
|
|
127
|
+
if error.message.include?("no such table")
|
|
128
|
+
FullSearch::Instrumentation.instrument("missing_table", model: model_class.name, error: error.message)
|
|
129
|
+
return
|
|
130
|
+
end
|
|
131
|
+
raise error
|
|
132
|
+
end
|
|
133
|
+
private_class_method :raise_missing_table_or_original
|
|
116
134
|
end
|
|
117
135
|
end
|
data/lib/full_search/config.rb
CHANGED
|
@@ -4,7 +4,8 @@ module FullSearch
|
|
|
4
4
|
class Config
|
|
5
5
|
attr_accessor :auto_rebuild_schema, :stale_query_behavior, :lock_rebuilds,
|
|
6
6
|
:default_async_reindex, :default_async_source_reindex,
|
|
7
|
-
:default_tokenizer, :auto_rebuild_on_stale_query, :min_like_prefix_length
|
|
7
|
+
:default_tokenizer, :auto_rebuild_on_stale_query, :min_like_prefix_length,
|
|
8
|
+
:dump_schema_virtual_tables
|
|
8
9
|
|
|
9
10
|
def initialize
|
|
10
11
|
@auto_rebuild_schema = false
|
|
@@ -12,9 +13,10 @@ module FullSearch
|
|
|
12
13
|
@lock_rebuilds = true
|
|
13
14
|
@default_async_reindex = true
|
|
14
15
|
@default_async_source_reindex = true
|
|
15
|
-
@default_tokenizer =
|
|
16
|
+
@default_tokenizer = FullSearch::Constants::DEFAULT_TOKENIZER
|
|
16
17
|
@auto_rebuild_on_stale_query = false
|
|
17
|
-
@min_like_prefix_length =
|
|
18
|
+
@min_like_prefix_length = FullSearch::Constants::DEFAULT_MIN_LIKE_PREFIX_LENGTH
|
|
19
|
+
@dump_schema_virtual_tables = true
|
|
18
20
|
end
|
|
19
21
|
end
|
|
20
22
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FullSearch
|
|
4
|
+
module Constants
|
|
5
|
+
MIN_TERM_LENGTH = 3
|
|
6
|
+
TWO_TYPO_MIN_LENGTH = 9
|
|
7
|
+
DEFAULT_MIN_LIKE_PREFIX_LENGTH = 3
|
|
8
|
+
REBUILDING_HASH = "__rebuilding__"
|
|
9
|
+
DEFAULT_TOKENIZER = "porter"
|
|
10
|
+
MAX_EXACT_MATCH_BOOST_IDS = 100
|
|
11
|
+
end
|
|
12
|
+
end
|
data/lib/full_search/dsl.rb
CHANGED
|
@@ -45,7 +45,9 @@ module FullSearch
|
|
|
45
45
|
end
|
|
46
46
|
str = name.to_s
|
|
47
47
|
raise InvalidFieldError, "#{model_class.name}: duplicate exact_match name #{name.inspect}" if exact_matches.any? { |e| e.name == str }
|
|
48
|
-
|
|
48
|
+
sql_str = sql&.to_s
|
|
49
|
+
validate_sql_fragment!(sql_str, "exact_match sql:") if sql_str
|
|
50
|
+
@exact_matches << ExactMatch.new(name: str, source: source, sql: sql_str, normalize: normalize, version: version)
|
|
49
51
|
end
|
|
50
52
|
|
|
51
53
|
def rank_by(column, direction = :desc)
|
|
@@ -92,7 +94,9 @@ module FullSearch
|
|
|
92
94
|
end
|
|
93
95
|
|
|
94
96
|
def index_if(sql: nil)
|
|
95
|
-
|
|
97
|
+
sql_str = sql&.to_s
|
|
98
|
+
validate_sql_fragment!(sql_str, "index_if sql:") if sql_str
|
|
99
|
+
@index_if_sql = sql_str
|
|
96
100
|
end
|
|
97
101
|
|
|
98
102
|
def conditional_index?
|
|
@@ -109,7 +113,7 @@ module FullSearch
|
|
|
109
113
|
|
|
110
114
|
def typo_tolerance(enabled = true, min_term_length: nil)
|
|
111
115
|
@typo_tolerance = enabled
|
|
112
|
-
@typo_tolerance_min_term_length = min_term_length ||
|
|
116
|
+
@typo_tolerance_min_term_length = min_term_length || FullSearch::Constants::MIN_TERM_LENGTH
|
|
113
117
|
end
|
|
114
118
|
|
|
115
119
|
def typo_tolerance?
|
|
@@ -117,7 +121,7 @@ module FullSearch
|
|
|
117
121
|
end
|
|
118
122
|
|
|
119
123
|
def typo_tolerance_min_term_length
|
|
120
|
-
@typo_tolerance_min_term_length ||
|
|
124
|
+
@typo_tolerance_min_term_length || FullSearch::Constants::MIN_TERM_LENGTH
|
|
121
125
|
end
|
|
122
126
|
|
|
123
127
|
def config_hash
|
|
@@ -139,6 +143,12 @@ module FullSearch
|
|
|
139
143
|
|
|
140
144
|
private
|
|
141
145
|
|
|
146
|
+
def validate_sql_fragment!(sql, option_name)
|
|
147
|
+
if sql.match?(/;|--|\/\*|\*\//)
|
|
148
|
+
raise InvalidFieldError, "#{model_class.name}: #{option_name} contains unsafe characters"
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
142
152
|
def valid_name?(name)
|
|
143
153
|
name.to_s.match?(/\A[a-zA-Z_]\w*\z/)
|
|
144
154
|
end
|
|
@@ -59,7 +59,7 @@ module FullSearch
|
|
|
59
59
|
length = query.length
|
|
60
60
|
min_length = dsl.typo_tolerance_min_term_length.to_i
|
|
61
61
|
return nil if length < min_length
|
|
62
|
-
return 2 if length >=
|
|
62
|
+
return 2 if length >= FullSearch::Constants::TWO_TYPO_MIN_LENGTH
|
|
63
63
|
1
|
|
64
64
|
end
|
|
65
65
|
|
|
@@ -170,8 +170,8 @@ module FullSearch
|
|
|
170
170
|
end
|
|
171
171
|
|
|
172
172
|
def max_allowed_typos(length)
|
|
173
|
-
return -1 if length <
|
|
174
|
-
return 2 if length >=
|
|
173
|
+
return -1 if length < FullSearch::Constants::MIN_TERM_LENGTH
|
|
174
|
+
return 2 if length >= FullSearch::Constants::TWO_TYPO_MIN_LENGTH
|
|
175
175
|
1
|
|
176
176
|
end
|
|
177
177
|
|
data/lib/full_search/index.rb
CHANGED
|
@@ -36,7 +36,7 @@ module FullSearch
|
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
trigram_was_created = false
|
|
39
|
-
if
|
|
39
|
+
if trigram_table_needed?(model) && !trigram_table_exists?(model)
|
|
40
40
|
FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
|
|
41
41
|
conn.execute(create_trigram_virtual_table_sql(model))
|
|
42
42
|
trigram_was_created = true
|
|
@@ -65,21 +65,23 @@ module FullSearch
|
|
|
65
65
|
create_metadata_table!
|
|
66
66
|
|
|
67
67
|
with_rebuild_lock(model) do
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
68
|
+
FullSearch::Instrumentation.instrument("rebuild", model: model.name) do
|
|
69
|
+
drop_triggers!(model)
|
|
70
|
+
conn.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
|
|
71
|
+
conn.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
|
|
72
|
+
conn.execute(create_virtual_table_sql(model))
|
|
73
|
+
if trigram_table_needed?(model)
|
|
74
|
+
FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
|
|
75
|
+
conn.execute(create_trigram_virtual_table_sql(model))
|
|
76
|
+
end
|
|
77
|
+
conn.execute(backfill_sql(model))
|
|
78
|
+
conn.execute(backfill_trigram_sql(model)) if trigram_table_needed?(model)
|
|
79
|
+
reindex_source_fields!(model) if dsl.fields.any?(&:source)
|
|
80
|
+
create_triggers!(model)
|
|
81
|
+
optimize!(model)
|
|
82
|
+
store_config_hash!(model, rebuilt_at: Time.current)
|
|
83
|
+
verified_tables.add(model.table_name)
|
|
75
84
|
end
|
|
76
|
-
conn.execute(backfill_sql(model))
|
|
77
|
-
conn.execute(backfill_trigram_sql(model)) if dsl.typo_tolerance?
|
|
78
|
-
reindex_source_fields!(model) if dsl.fields.any?(&:source)
|
|
79
|
-
create_triggers!(model)
|
|
80
|
-
optimize!(model)
|
|
81
|
-
store_config_hash!(model, rebuilt_at: Time.current)
|
|
82
|
-
verified_tables.add(model.table_name)
|
|
83
85
|
end
|
|
84
86
|
end
|
|
85
87
|
|
|
@@ -103,21 +105,35 @@ module FullSearch
|
|
|
103
105
|
end
|
|
104
106
|
|
|
105
107
|
def optimize!(model)
|
|
106
|
-
|
|
107
|
-
|
|
108
|
+
FullSearch::Instrumentation.instrument("optimize", model: model.name) do
|
|
109
|
+
sqlite!(model)
|
|
110
|
+
connection.execute("INSERT INTO #{qt(fts_table_name(model))}(#{qt(fts_table_name(model))}) VALUES('optimize');")
|
|
111
|
+
end
|
|
108
112
|
end
|
|
109
113
|
|
|
110
114
|
def reindex_source_fields!(model)
|
|
111
115
|
dsl = model.full_search_dsl
|
|
112
116
|
return unless dsl
|
|
113
117
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
+
source_fields = dsl.fields.select(&:source)
|
|
119
|
+
return if source_fields.empty?
|
|
120
|
+
|
|
121
|
+
model.find_each(batch_size: 500) do |record|
|
|
122
|
+
pairs = source_fields.to_h { |f| [column_name(f), FullSearch::Model.evaluate_source(record, f).to_s] }
|
|
123
|
+
next if pairs.empty?
|
|
124
|
+
upsert_source_row!(model, record.id, pairs)
|
|
118
125
|
end
|
|
119
126
|
end
|
|
120
127
|
|
|
128
|
+
def upsert_source_row!(model, rowid, values)
|
|
129
|
+
table = qt(fts_table_name(model))
|
|
130
|
+
sets = values.map { |name, value| "#{qc(name)} = #{q(value)}" }.join(", ")
|
|
131
|
+
|
|
132
|
+
connection.execute(<<~SQL)
|
|
133
|
+
UPDATE #{table} SET #{sets} WHERE rowid = #{q(rowid)};
|
|
134
|
+
SQL
|
|
135
|
+
end
|
|
136
|
+
|
|
121
137
|
def create_triggers!(model)
|
|
122
138
|
connection.execute(insert_trigger_sql(model))
|
|
123
139
|
connection.execute(delete_trigger_sql(model))
|
|
@@ -125,7 +141,7 @@ module FullSearch
|
|
|
125
141
|
if model.full_search_dsl.soft_delete_column
|
|
126
142
|
connection.execute(soft_delete_removal_trigger_sql(model))
|
|
127
143
|
end
|
|
128
|
-
if model
|
|
144
|
+
if trigram_table_needed?(model)
|
|
129
145
|
connection.execute(insert_trigram_trigger_sql(model))
|
|
130
146
|
connection.execute(delete_trigram_trigger_sql(model))
|
|
131
147
|
connection.execute(update_trigram_trigger_sql(model))
|
|
@@ -183,6 +199,16 @@ module FullSearch
|
|
|
183
199
|
!table_exists?(model)
|
|
184
200
|
end
|
|
185
201
|
|
|
202
|
+
def trigram_table_needed?(model)
|
|
203
|
+
model.full_search_dsl&.typo_tolerance? && model.full_search_dsl.tokenize != "trigram"
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def trigram_table_exists?(model)
|
|
207
|
+
connection.execute(
|
|
208
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(trigram_table_name(model))} LIMIT 1"
|
|
209
|
+
).any?
|
|
210
|
+
end
|
|
211
|
+
|
|
186
212
|
private
|
|
187
213
|
|
|
188
214
|
def connection
|
|
@@ -276,7 +302,7 @@ module FullSearch
|
|
|
276
302
|
).map { |r| r["name"] }
|
|
277
303
|
|
|
278
304
|
expected = trigger_names(model)
|
|
279
|
-
expected += trigram_trigger_names(model) if model
|
|
305
|
+
expected += trigram_trigger_names(model) if trigram_table_needed?(model)
|
|
280
306
|
return if (expected - existing).empty? && (existing - expected).empty?
|
|
281
307
|
|
|
282
308
|
rebuild!(model)
|
|
@@ -375,12 +401,6 @@ module FullSearch
|
|
|
375
401
|
SQL
|
|
376
402
|
end
|
|
377
403
|
|
|
378
|
-
def trigram_table_exists?(model)
|
|
379
|
-
connection.execute(
|
|
380
|
-
"SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(trigram_table_name(model))} LIMIT 1"
|
|
381
|
-
).any?
|
|
382
|
-
end
|
|
383
|
-
|
|
384
404
|
def create_trigram_virtual_table_sql(model)
|
|
385
405
|
dsl = model.full_search_dsl
|
|
386
406
|
columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) } + extra_columns(model))
|
|
@@ -501,15 +521,13 @@ module FullSearch
|
|
|
501
521
|
# For multi-process or multi-host deployments, run full_search:rebuild from a single
|
|
502
522
|
# deployment step. See README.
|
|
503
523
|
def with_rebuild_lock(model)
|
|
504
|
-
|
|
505
|
-
|
|
524
|
+
connection.transaction do
|
|
525
|
+
if FullSearch.config.lock_rebuilds
|
|
506
526
|
connection.execute(
|
|
507
|
-
"INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q(
|
|
527
|
+
"INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q(FullSearch::Constants::REBUILDING_HASH)}, datetime('now'))
|
|
508
528
|
ON CONFLICT(table_name) DO UPDATE SET config_hash=excluded.config_hash;"
|
|
509
529
|
)
|
|
510
|
-
yield
|
|
511
530
|
end
|
|
512
|
-
else
|
|
513
531
|
yield
|
|
514
532
|
end
|
|
515
533
|
end
|
data/lib/full_search/search.rb
CHANGED
|
@@ -19,42 +19,44 @@ module FullSearch
|
|
|
19
19
|
@includes = includes
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
MIN_TERM_LENGTH =
|
|
22
|
+
MIN_TERM_LENGTH = FullSearch::Constants::MIN_TERM_LENGTH
|
|
23
23
|
|
|
24
24
|
def relation
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
25
|
+
FullSearch::Instrumentation.instrument("query", model: model.name, query: query) do
|
|
26
|
+
validate_filter_keys!
|
|
27
|
+
validate_required_filters!
|
|
28
|
+
check_stale_config!
|
|
29
|
+
|
|
30
|
+
return model.none if dsl.tokenize == "trigram" && query.length < MIN_TERM_LENGTH && !dsl.typo_tolerance?
|
|
31
|
+
|
|
32
|
+
parsed = QueryParser.parse(query)
|
|
33
|
+
exact_ids = ExactMatch.ids_for(model, query, filters)
|
|
34
|
+
primary_ids = fts_match_ids(parsed)
|
|
35
|
+
fallback_ids = (dsl.typo_tolerance? && matching_strategy != "all") ? trigram_match_ids(parsed, primary_ids, candidate_limit: per_strategy_limit) : []
|
|
36
|
+
fuzzy_ids = (dsl.typo_tolerance? && matching_strategy != "all" && primary_ids.empty? && fallback_ids.empty?) ? fuzzy_match_ids(parsed, candidate_limit: per_strategy_limit) : []
|
|
37
|
+
|
|
38
|
+
all_ids = (exact_ids + primary_ids + fallback_ids + fuzzy_ids).uniq
|
|
39
|
+
return model.none if all_ids.empty?
|
|
40
|
+
|
|
41
|
+
rel = model.where(id: all_ids)
|
|
42
|
+
rel = rel.where(model.arel_table[dsl.soft_delete_column].eq(nil)) if dsl.soft_delete_column && !include_soft_deleted
|
|
43
|
+
rel = apply_ranking(rel, all_ids, exact_ids)
|
|
44
|
+
rel = scope.call(rel) if scope
|
|
45
|
+
rel = rel.limit(limit) if limit
|
|
46
|
+
rel = rel.offset(offset) if offset
|
|
47
|
+
rel = rel.includes(includes) if includes
|
|
48
|
+
|
|
49
|
+
if highlight
|
|
50
|
+
records = rel.to_a
|
|
51
|
+
Highlighter.apply!(records, model, query, record_ids: records.map(&:id))
|
|
52
|
+
records
|
|
53
|
+
elsif highlight_fields
|
|
54
|
+
records = rel.to_a
|
|
55
|
+
Highlighter.apply_fields!(records, model, query, record_ids: records.map(&:id))
|
|
56
|
+
records
|
|
57
|
+
else
|
|
58
|
+
rel
|
|
59
|
+
end
|
|
58
60
|
end
|
|
59
61
|
end
|
|
60
62
|
|
|
@@ -144,6 +146,8 @@ module FullSearch
|
|
|
144
146
|
return like_prefix_ids(term, candidate_limit: candidate_limit)
|
|
145
147
|
end
|
|
146
148
|
|
|
149
|
+
return [] unless FullSearch::Index.trigram_table_exists?(model)
|
|
150
|
+
|
|
147
151
|
trigram_table = qt(FullSearch::Index.trigram_table_name(model))
|
|
148
152
|
tbl = qt(model.table_name)
|
|
149
153
|
|
|
@@ -279,7 +283,7 @@ module FullSearch
|
|
|
279
283
|
def max_allowed_typos(length)
|
|
280
284
|
min_length = dsl.typo_tolerance_min_term_length.to_i
|
|
281
285
|
return -1 if length < min_length
|
|
282
|
-
return 2 if length >=
|
|
286
|
+
return 2 if length >= FullSearch::Constants::TWO_TYPO_MIN_LENGTH
|
|
283
287
|
1
|
|
284
288
|
end
|
|
285
289
|
|
|
@@ -305,7 +309,8 @@ module FullSearch
|
|
|
305
309
|
tbl = qt(model.table_name)
|
|
306
310
|
|
|
307
311
|
if exact_ids.any?
|
|
308
|
-
|
|
312
|
+
boost_ids = exact_ids.first(FullSearch::Constants::MAX_EXACT_MATCH_BOOST_IDS)
|
|
313
|
+
order_parts << "CASE #{tbl}.id #{boost_ids.map { |id| "WHEN #{q(id)} THEN 0" }.join(" ")} ELSE 1 END"
|
|
309
314
|
end
|
|
310
315
|
|
|
311
316
|
fts_table = qt(FullSearch::Index.fts_table_name(model))
|
data/lib/full_search/version.rb
CHANGED
data/lib/full_search.rb
CHANGED
|
@@ -4,7 +4,9 @@ require "active_record"
|
|
|
4
4
|
require "active_support"
|
|
5
5
|
require "active_support/concern"
|
|
6
6
|
|
|
7
|
+
require "full_search/constants"
|
|
7
8
|
require "full_search/index_cache"
|
|
9
|
+
require "full_search/instrumentation"
|
|
8
10
|
require "full_search/version"
|
|
9
11
|
require "full_search/config"
|
|
10
12
|
require "full_search/distance"
|
|
@@ -39,6 +41,10 @@ module FullSearch
|
|
|
39
41
|
@models ||= Set.new
|
|
40
42
|
end
|
|
41
43
|
|
|
44
|
+
def sorted_models
|
|
45
|
+
models.sort_by { |m| m.table_name.to_s }
|
|
46
|
+
end
|
|
47
|
+
|
|
42
48
|
def register_model(model)
|
|
43
49
|
models << model
|
|
44
50
|
end
|
|
@@ -48,11 +54,11 @@ module FullSearch
|
|
|
48
54
|
end
|
|
49
55
|
|
|
50
56
|
def optimize!
|
|
51
|
-
|
|
57
|
+
sorted_models.each { |model| Index.optimize!(model) }
|
|
52
58
|
end
|
|
53
59
|
|
|
54
60
|
def setup!
|
|
55
|
-
|
|
61
|
+
sorted_models.each do |model|
|
|
56
62
|
Index.ensure_table!(model)
|
|
57
63
|
Callbacks.install!(model)
|
|
58
64
|
end
|
|
@@ -12,4 +12,8 @@ FullSearch.configure do |config|
|
|
|
12
12
|
# Reindex computed source: fields synchronously (false) or via background job (true).
|
|
13
13
|
# Bulk imports should use FullSearch.bulk_import(Model) { ... } to defer reindexing.
|
|
14
14
|
config.default_async_source_reindex = true
|
|
15
|
+
|
|
16
|
+
# Uncomment to exclude full_search virtual tables from db/schema.rb.
|
|
17
|
+
# When false, you must run `bin/rails full_search:prepare` after db:schema:load.
|
|
18
|
+
# config.dump_schema_virtual_tables = false
|
|
15
19
|
end
|
data/lib/tasks/full_search.rake
CHANGED
|
@@ -12,7 +12,7 @@ def resolve_full_search_models(args)
|
|
|
12
12
|
klass || FullSearch.models.find { |m| m.table_name == name }
|
|
13
13
|
end.compact
|
|
14
14
|
else
|
|
15
|
-
FullSearch.
|
|
15
|
+
FullSearch.sorted_models
|
|
16
16
|
end
|
|
17
17
|
end
|
|
18
18
|
|
|
@@ -31,7 +31,7 @@ namespace :full_search do
|
|
|
31
31
|
klass
|
|
32
32
|
end.compact
|
|
33
33
|
else
|
|
34
|
-
FullSearch.
|
|
34
|
+
FullSearch.sorted_models
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
FullSearch.setup!
|
|
@@ -77,7 +77,7 @@ namespace :full_search do
|
|
|
77
77
|
task optimize: :environment do
|
|
78
78
|
Rails.application.eager_load!
|
|
79
79
|
FullSearch.optimize!
|
|
80
|
-
FullSearch.
|
|
80
|
+
FullSearch.sorted_models.each do |model|
|
|
81
81
|
puts "Optimized #{FullSearch::Index.fts_table_name(model)}"
|
|
82
82
|
end
|
|
83
83
|
end
|
|
@@ -85,7 +85,7 @@ namespace :full_search do
|
|
|
85
85
|
desc "Show full_search index status"
|
|
86
86
|
task status: :environment do
|
|
87
87
|
Rails.application.eager_load!
|
|
88
|
-
FullSearch.
|
|
88
|
+
FullSearch.sorted_models.each do |model|
|
|
89
89
|
stored = FullSearch::Index.stored_config_hash(model)
|
|
90
90
|
current = model.full_search_dsl.config_hash
|
|
91
91
|
status = (stored == current) ? "ok" : "stale"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: full_search
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.9
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ben D'Angelo
|
|
@@ -144,6 +144,7 @@ files:
|
|
|
144
144
|
- lib/full_search/bulk_import.rb
|
|
145
145
|
- lib/full_search/callbacks.rb
|
|
146
146
|
- lib/full_search/config.rb
|
|
147
|
+
- lib/full_search/constants.rb
|
|
147
148
|
- lib/full_search/distance.rb
|
|
148
149
|
- lib/full_search/dsl.rb
|
|
149
150
|
- lib/full_search/errors.rb
|
|
@@ -151,6 +152,7 @@ files:
|
|
|
151
152
|
- lib/full_search/highlighter.rb
|
|
152
153
|
- lib/full_search/index.rb
|
|
153
154
|
- lib/full_search/index_cache.rb
|
|
155
|
+
- lib/full_search/instrumentation.rb
|
|
154
156
|
- lib/full_search/model.rb
|
|
155
157
|
- lib/full_search/multi_search.rb
|
|
156
158
|
- lib/full_search/optimize_job.rb
|