full_search 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +56 -0
- data/lib/full_search/backfill_job.rb +14 -0
- data/lib/full_search/bulk_import.rb +35 -0
- data/lib/full_search/callbacks.rb +15 -6
- data/lib/full_search/config.rb +5 -1
- data/lib/full_search/dsl.rb +37 -21
- data/lib/full_search/exact_match.rb +24 -6
- data/lib/full_search/highlighter.rb +13 -9
- data/lib/full_search/index.rb +146 -68
- data/lib/full_search/model.rb +21 -4
- data/lib/full_search/multi_search.rb +3 -1
- data/lib/full_search/reindex_job.rb +9 -3
- data/lib/full_search/search.rb +55 -22
- data/lib/full_search/version.rb +1 -1
- data/lib/full_search.rb +10 -0
- data/lib/generators/full_search/install/templates/full_search.rb +6 -0
- data/lib/tasks/full_search.rake +10 -0
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ed7b0998cc6d4014ab815a6152ba07562da34c98b3cca3aa3886e63fb45acc5b
|
|
4
|
+
data.tar.gz: f29430f7ac4d5a3175e1eb46c803aa3a9b9ce68e3e930f2f75ecfd8ad03242d7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cce0189e77ec25883ad52dae321dea4f99cb8f1f245f331b5cdeb94f6c3554c675e84b0214e4036ca1c01fc53f22e41594324729d39688c8e48d0bcd791356f3
|
|
7
|
+
data.tar.gz: a9d3f31295d082892dc5cf797f2d38a9cb8151629f9e6d2c0ac0ad1b8dcfeb5e732b740fb91d138d0b3a7f1db513c72c36aa61a1410d2b052bd57bda0790c5dd
|
data/README.md
CHANGED
|
@@ -57,6 +57,24 @@ Customer.search("sam", filters: { account_id: 1 }).page(params[:page])
|
|
|
57
57
|
- FTS5 `highlight()` support for result snippets
|
|
58
58
|
- Opt-in trigram typo/substring fallback (requires SQLite >= 3.34)
|
|
59
59
|
|
|
60
|
+
### Conditional indexing
|
|
61
|
+
|
|
62
|
+
Use `index_if` to restrict which records are included in the FTS index. Records that don't match the SQL condition are excluded from search results entirely.
|
|
63
|
+
|
|
64
|
+
```ruby
|
|
65
|
+
class Customer < ApplicationRecord
|
|
66
|
+
full_search do
|
|
67
|
+
field :first_name, weight: 5
|
|
68
|
+
field :last_name, weight: 5
|
|
69
|
+
filter :account_id, required: true
|
|
70
|
+
|
|
71
|
+
index_if sql: "active = 1 AND deleted_at IS NULL"
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
This is enforced at the database level — the FTS virtual table only contains rows matching the condition, and triggers skip inserts/updates for non-matching records.
|
|
77
|
+
|
|
60
78
|
### Per-model operations
|
|
61
79
|
|
|
62
80
|
Once a model declares `full_search`, you can call these class methods:
|
|
@@ -78,6 +96,10 @@ These two operations are often confused:
|
|
|
78
96
|
- **Rebuild** (`full_search:rebuild` / `Customer.rebuild!`) — drops and recreates the FTS virtual table. Needed when the DSL changes (fields added/removed, tokenizer changed, etc.). The table is re-created from scratch, backfilled, triggers re-installed, and the index is optimized.
|
|
79
97
|
- **Reindex** (`Customer.reindex!` / `FullSearch::Index.reindex_source_fields!`) — updates existing FTS rows with fresh values from computed `source:` fields only. The table structure is untouched. Database triggers cover regular column changes automatically; only Ruby-evaluated `source:` blocks need an explicit reindex.
|
|
80
98
|
|
|
99
|
+
### Setup lifecycle
|
|
100
|
+
|
|
101
|
+
`full_search` evaluates the DSL block and installs callbacks when the model class loads, but it does **not** create the FTS table until Rails `after_initialize` (or until you call `FullSearch.setup!` or `FullSearch::Index.rebuild!` manually). This guarantees that `source:` blocks run against fully-loaded model definitions, avoiding load-order issues where associations or methods defined later in the class body are not yet available.
|
|
102
|
+
|
|
81
103
|
> **Note on rebuild locking:** The `lock_rebuilds` option prevents concurrent rebuilds
|
|
82
104
|
> within the same process/connection. For multi-process or multi-host deployments,
|
|
83
105
|
> run `full_search:rebuild` from a single deployment step.
|
|
@@ -122,6 +144,7 @@ bin/rails full_search:reset
|
|
|
122
144
|
| `full_search:rebuild` | Drops and recreates the FTS virtual table only when the DSL config hash has changed (safe for production). Pass model names to target specific tables. |
|
|
123
145
|
| `full_search:reset` | Force a full rebuild — drops and recreates all FTS tables regardless of config hash. Use when data may be out of sync. |
|
|
124
146
|
| `full_search:optimize` | Run FTS5 [`optimize`](https://www.sqlite.org/fts5.html#the_optimize_command) to merge b-tree segments. Useful after bulk updates. |
|
|
147
|
+
| `full_search:backfill` | Force-rebuild FTS indexes for specified models (or all). Useful for recovery after bulk operations. |
|
|
125
148
|
| `full_search:status` | Show each model's index status (`ok` / `stale`) and count of empty sourced fields. |
|
|
126
149
|
|
|
127
150
|
## Background jobs
|
|
@@ -146,6 +169,39 @@ If you're not on Solid Queue, most job frameworks support recurring schedules vi
|
|
|
146
169
|
|
|
147
170
|
Every FTS table stores a SHA256 digest of its DSL configuration in the `full_search_index_versions` table. When the DSL changes (e.g., adding a field), the stored hash no longer matches, and `rebuild!` is triggered automatically (if `auto_rebuild_schema` is true) to bring the index in line with the new definition. If `auto_rebuild_schema` is false, queries raise `ConfigChangedError` when the hash doesn't match (or log a warning and fall back if configured with `:log_and_fallback`).
|
|
148
171
|
|
|
172
|
+
### Bulk imports
|
|
173
|
+
|
|
174
|
+
When importing thousands of records, disable synchronous FTS maintenance and rebuild the index afterward:
|
|
175
|
+
|
|
176
|
+
```ruby
|
|
177
|
+
FullSearch.bulk_import(Customer) do
|
|
178
|
+
Customer.insert_all(records)
|
|
179
|
+
end
|
|
180
|
+
# automatically enqueues FullSearch::BackfillJob for Customer
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
While inside the block, database triggers are dropped so raw SQL operations (including `insert_all!`, `update_all`, and `delete_all`) bypass FTS table updates entirely. On exit, triggers are re-created and a `FullSearch::BackfillJob` is enqueued to rebuild the index from scratch. Reindexing of computed `source:` blocks within the block is also skipped.
|
|
184
|
+
|
|
185
|
+
You can also call `Customer.bulk_import { ... }` directly on the model.
|
|
186
|
+
|
|
187
|
+
For manual recovery outside a bulk-import block:
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
bin/rails 'full_search:backfill[customers]'
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
### Query-time auto-rebuild in local environments
|
|
194
|
+
|
|
195
|
+
By default, the generated initializer also enables `auto_rebuild_on_stale_query` in local environments:
|
|
196
|
+
|
|
197
|
+
```ruby
|
|
198
|
+
FullSearch.configure do |config|
|
|
199
|
+
config.auto_rebuild_on_stale_query = Rails.env.local?
|
|
200
|
+
end
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
When this option is `true`, the first search that detects a stale index automatically calls `FullSearch::Index.rebuild_if_needed!` and then proceeds with the query, so you don't need to restart the server or run a Rake task during iterative development. In production, this defaults to `false`; use `full_search:rebuild` or boot-time `auto_rebuild_schema` instead.
|
|
204
|
+
|
|
149
205
|
## Query operators
|
|
150
206
|
|
|
151
207
|
```ruby
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_job"
|
|
4
|
+
|
|
5
|
+
module FullSearch
|
|
6
|
+
class BackfillJob < ActiveJob::Base
|
|
7
|
+
queue_as :low
|
|
8
|
+
|
|
9
|
+
def perform(model_name)
|
|
10
|
+
model = model_name.to_s.constantize
|
|
11
|
+
FullSearch::Index.rebuild!(model)
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FullSearch
|
|
4
|
+
module BulkImport
|
|
5
|
+
class << self
|
|
6
|
+
def bulk_import(model)
|
|
7
|
+
start_bulk_import(model)
|
|
8
|
+
yield
|
|
9
|
+
ensure
|
|
10
|
+
end_bulk_import(model)
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def start_bulk_import(model)
|
|
14
|
+
models_in_bulk_import << model
|
|
15
|
+
FullSearch::Index.drop_triggers!(model)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def end_bulk_import(model)
|
|
19
|
+
models_in_bulk_import.delete(model)
|
|
20
|
+
FullSearch::Index.create_triggers!(model)
|
|
21
|
+
FullSearch::BackfillJob.perform_later(model.name)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def bulk_importing?(model)
|
|
25
|
+
models_in_bulk_import.include?(model)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def models_in_bulk_import
|
|
31
|
+
Thread.current[:full_search_bulk_import_models] ||= Set.new
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -10,11 +10,13 @@ module FullSearch
|
|
|
10
10
|
source_fields = dsl.fields.select(&:source)
|
|
11
11
|
return if source_fields.empty?
|
|
12
12
|
|
|
13
|
-
model.
|
|
13
|
+
model.after_save_commit do
|
|
14
|
+
next if FullSearch.bulk_importing?(self.class)
|
|
14
15
|
FullSearch::Callbacks.reindex_record!(self)
|
|
15
16
|
end
|
|
16
17
|
|
|
17
|
-
model.
|
|
18
|
+
model.after_destroy_commit do
|
|
19
|
+
next if FullSearch.bulk_importing?(self.class)
|
|
18
20
|
FullSearch::Callbacks.remove_record!(self)
|
|
19
21
|
end
|
|
20
22
|
|
|
@@ -22,10 +24,12 @@ module FullSearch
|
|
|
22
24
|
next unless field.reindex_on
|
|
23
25
|
|
|
24
26
|
assoc_class = associated_class(model, field.reindex_on)
|
|
25
|
-
assoc_class&.
|
|
27
|
+
assoc_class&.after_save_commit do |record|
|
|
28
|
+
next if FullSearch.bulk_importing?(record.class)
|
|
26
29
|
FullSearch::Callbacks.reindex_dependents!(record, model, field)
|
|
27
30
|
end
|
|
28
|
-
assoc_class&.
|
|
31
|
+
assoc_class&.after_destroy_commit do |record|
|
|
32
|
+
next if FullSearch.bulk_importing?(record.class)
|
|
29
33
|
FullSearch::Callbacks.reindex_dependents!(record, model, field)
|
|
30
34
|
end
|
|
31
35
|
end
|
|
@@ -41,10 +45,15 @@ module FullSearch
|
|
|
41
45
|
dsl = record.class.full_search_dsl
|
|
42
46
|
return unless dsl
|
|
43
47
|
|
|
48
|
+
model_name = record.class.name
|
|
44
49
|
dsl.fields.each do |field|
|
|
45
50
|
next unless field.source
|
|
46
51
|
|
|
47
|
-
|
|
52
|
+
if model_name && field.async_source
|
|
53
|
+
FullSearch::ReindexJob.perform_later(model_name, record.id, field.name)
|
|
54
|
+
else
|
|
55
|
+
reindex_field!(record, field.name)
|
|
56
|
+
end
|
|
48
57
|
end
|
|
49
58
|
end
|
|
50
59
|
|
|
@@ -53,7 +62,7 @@ module FullSearch
|
|
|
53
62
|
field = dsl.fields.find { |f| f.name == field_name }
|
|
54
63
|
return unless field&.source
|
|
55
64
|
|
|
56
|
-
value =
|
|
65
|
+
value = FullSearch::Model.evaluate_source(record, field)
|
|
57
66
|
table = qt(FullSearch::Index.fts_table_name(record.class))
|
|
58
67
|
conn = ActiveRecord::Base.connection
|
|
59
68
|
conn.execute(
|
data/lib/full_search/config.rb
CHANGED
|
@@ -2,14 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
module FullSearch
|
|
4
4
|
class Config
|
|
5
|
-
attr_accessor :auto_rebuild_schema, :stale_query_behavior, :lock_rebuilds,
|
|
5
|
+
attr_accessor :auto_rebuild_schema, :stale_query_behavior, :lock_rebuilds,
|
|
6
|
+
:default_async_reindex, :default_async_source_reindex,
|
|
7
|
+
:default_tokenizer, :auto_rebuild_on_stale_query
|
|
6
8
|
|
|
7
9
|
def initialize
|
|
8
10
|
@auto_rebuild_schema = false
|
|
9
11
|
@stale_query_behavior = :raise
|
|
10
12
|
@lock_rebuilds = true
|
|
11
13
|
@default_async_reindex = true
|
|
14
|
+
@default_async_source_reindex = true
|
|
12
15
|
@default_tokenizer = "unicode61"
|
|
16
|
+
@auto_rebuild_on_stale_query = false
|
|
13
17
|
end
|
|
14
18
|
end
|
|
15
19
|
|
data/lib/full_search/dsl.rb
CHANGED
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
module FullSearch
|
|
4
4
|
class Dsl
|
|
5
|
-
attr_reader :fields, :exact_matches, :filters, :model_class, :highlight_config, :rank_bys
|
|
5
|
+
attr_reader :fields, :exact_matches, :filters, :model_class, :highlight_config, :rank_bys, :index_if_sql
|
|
6
6
|
|
|
7
|
-
Field = Data.define(:name, :weight, :source, :reindex_on, :async, :as, :version)
|
|
8
|
-
ExactMatch = Data.define(:name, :source, :version)
|
|
7
|
+
Field = Data.define(:name, :weight, :source, :reindex_on, :async, :async_source, :as, :version)
|
|
8
|
+
ExactMatch = Data.define(:name, :source, :sql, :version)
|
|
9
9
|
Filter = Data.define(:name, :required)
|
|
10
10
|
RankBy = Data.define(:column, :direction)
|
|
11
11
|
|
|
@@ -19,48 +19,55 @@ module FullSearch
|
|
|
19
19
|
@soft_delete_column = nil
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
def field(name, weight: 1, source: nil, reindex_on: nil,
|
|
22
|
+
def field(name, weight: 1, source: nil, reindex_on: nil,
|
|
23
|
+
async: FullSearch.config.default_async_reindex,
|
|
24
|
+
async_source: FullSearch.config.default_async_source_reindex,
|
|
25
|
+
as: nil, version: nil)
|
|
23
26
|
unless valid_name?(name)
|
|
24
|
-
raise InvalidFieldError, "
|
|
27
|
+
raise InvalidFieldError, "#{model_class.name}: invalid field name #{name.inspect}"
|
|
25
28
|
end
|
|
26
29
|
if as && !valid_name?(as)
|
|
27
|
-
raise InvalidFieldError, "
|
|
30
|
+
raise InvalidFieldError, "#{model_class.name}: invalid field alias (as): #{as.inspect}"
|
|
28
31
|
end
|
|
29
32
|
str = name.to_s
|
|
30
|
-
raise InvalidFieldError, "
|
|
31
|
-
raise InvalidFieldError, "
|
|
32
|
-
@fields << Field.new(
|
|
33
|
+
raise InvalidFieldError, "#{model_class.name}: duplicate field name #{name.inspect}" if fields.any? { |f| f.name == str }
|
|
34
|
+
raise InvalidFieldError, "#{model_class.name}: field name #{name.inspect} conflicts with existing filter" if filters.any? { |f| f.name == str }
|
|
35
|
+
@fields << Field.new(
|
|
36
|
+
name: str, weight: weight.to_i, source: source,
|
|
37
|
+
reindex_on: reindex_on&.to_s, async: async,
|
|
38
|
+
async_source: async_source, as: as&.to_s, version: version
|
|
39
|
+
)
|
|
33
40
|
end
|
|
34
41
|
|
|
35
|
-
def exact_match(name, source: -> { public_send(name) }, version: nil)
|
|
42
|
+
def exact_match(name, source: -> { public_send(name) }, sql: nil, version: nil)
|
|
36
43
|
unless valid_name?(name)
|
|
37
|
-
raise InvalidFieldError, "
|
|
44
|
+
raise InvalidFieldError, "#{model_class.name}: invalid exact_match name #{name.inspect}"
|
|
38
45
|
end
|
|
39
46
|
str = name.to_s
|
|
40
|
-
raise InvalidFieldError, "
|
|
41
|
-
@exact_matches << ExactMatch.new(name: str, source: source, version: version)
|
|
47
|
+
raise InvalidFieldError, "#{model_class.name}: duplicate exact_match name #{name.inspect}" if exact_matches.any? { |e| e.name == str }
|
|
48
|
+
@exact_matches << ExactMatch.new(name: str, source: source, sql: sql&.to_s, version: version)
|
|
42
49
|
end
|
|
43
50
|
|
|
44
51
|
def rank_by(column, direction = :desc)
|
|
45
52
|
unless valid_name?(column)
|
|
46
|
-
raise InvalidFieldError, "
|
|
53
|
+
raise InvalidFieldError, "#{model_class.name}: invalid rank_by column #{column.inspect}"
|
|
47
54
|
end
|
|
48
55
|
dir = direction.to_s.downcase
|
|
49
56
|
unless %w[asc desc].include?(dir)
|
|
50
|
-
raise InvalidFieldError, "
|
|
57
|
+
raise InvalidFieldError, "#{model_class.name}: invalid rank_by direction #{direction.inspect}. Use :asc or :desc."
|
|
51
58
|
end
|
|
52
59
|
str = column.to_s
|
|
53
|
-
raise InvalidFieldError, "
|
|
60
|
+
raise InvalidFieldError, "#{model_class.name}: duplicate rank_by column #{column.inspect}" if rank_bys.any? { |r| r.column == str }
|
|
54
61
|
@rank_bys << RankBy.new(column: str, direction: dir.to_sym)
|
|
55
62
|
end
|
|
56
63
|
|
|
57
64
|
def filter(name, required: false)
|
|
58
65
|
unless valid_name?(name)
|
|
59
|
-
raise InvalidFieldError, "
|
|
66
|
+
raise InvalidFieldError, "#{model_class.name}: invalid filter name #{name.inspect}"
|
|
60
67
|
end
|
|
61
68
|
str = name.to_s
|
|
62
|
-
raise InvalidFieldError, "
|
|
63
|
-
raise InvalidFieldError, "
|
|
69
|
+
raise InvalidFieldError, "#{model_class.name}: duplicate filter name #{name.inspect}" if filters.any? { |f| f.name == str }
|
|
70
|
+
raise InvalidFieldError, "#{model_class.name}: filter name #{name.inspect} conflicts with existing field" if fields.any? { |f| f.name == str }
|
|
64
71
|
@filters << Filter.new(name: str, required: required)
|
|
65
72
|
end
|
|
66
73
|
|
|
@@ -84,6 +91,14 @@ module FullSearch
|
|
|
84
91
|
@highlight_config = {open_tag: open_tag, close_tag: close_tag}
|
|
85
92
|
end
|
|
86
93
|
|
|
94
|
+
def index_if(sql: nil)
|
|
95
|
+
@index_if_sql = sql&.to_s
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def conditional_index?
|
|
99
|
+
@index_if_sql.present?
|
|
100
|
+
end
|
|
101
|
+
|
|
87
102
|
def typo_tolerance(enabled = true, min_term_length: nil)
|
|
88
103
|
@typo_tolerance = enabled
|
|
89
104
|
@typo_tolerance_min_term_length = min_term_length || 3
|
|
@@ -105,8 +120,9 @@ module FullSearch
|
|
|
105
120
|
soft_delete_column,
|
|
106
121
|
typo_tolerance?,
|
|
107
122
|
typo_tolerance_min_term_length,
|
|
108
|
-
|
|
109
|
-
|
|
123
|
+
index_if_sql,
|
|
124
|
+
fields.map { |f| [f.name, f.weight, f.source.nil? ? "column" : "proc:#{f.version}", f.reindex_on, f.async, f.async_source, f.as] },
|
|
125
|
+
exact_matches.map { |e| [e.name, "proc:#{e.version}", e.sql] },
|
|
110
126
|
filters.map { |f| [f.name, f.required] },
|
|
111
127
|
rank_bys.map { |r| [r.column, r.direction] }
|
|
112
128
|
].inspect)
|
|
@@ -9,16 +9,34 @@ module FullSearch
|
|
|
9
9
|
cleaned = query.to_s.strip
|
|
10
10
|
return [] if cleaned.empty?
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
sql_matches = dsl.exact_matches.select(&:sql)
|
|
13
|
+
ruby_matches = dsl.exact_matches.reject(&:sql)
|
|
14
14
|
|
|
15
15
|
ids = []
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
|
|
17
|
+
if sql_matches.any?
|
|
18
|
+
base = model.all
|
|
19
|
+
filters.each { |name, value| base = base.where(name => value) }
|
|
20
|
+
|
|
21
|
+
conditions = sql_matches.map do |em|
|
|
22
|
+
"(#{em.sql}) = #{model.connection.quote(cleaned)}"
|
|
23
|
+
end.join(" OR ")
|
|
24
|
+
|
|
25
|
+
ids += base.where(conditions).pluck(:id)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
if ruby_matches.any?
|
|
29
|
+
base = model.all
|
|
30
|
+
filters.each { |name, value| base = base.where(name => value) }
|
|
31
|
+
|
|
32
|
+
base.find_each do |record|
|
|
33
|
+
ruby_matches.each do |em|
|
|
34
|
+
value = record.instance_exec(&em.source)
|
|
35
|
+
ids << record.id if value.to_s.casecmp?(cleaned)
|
|
36
|
+
end
|
|
20
37
|
end
|
|
21
38
|
end
|
|
39
|
+
|
|
22
40
|
ids.uniq
|
|
23
41
|
end
|
|
24
42
|
end
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
module FullSearch
|
|
4
4
|
class Highlighter
|
|
5
5
|
class << self
|
|
6
|
-
def apply!(records, model, query)
|
|
7
|
-
snippets = build_snippets(model, query)
|
|
6
|
+
def apply!(records, model, query, record_ids: nil)
|
|
7
|
+
snippets = build_snippets(model, query, record_ids: record_ids)
|
|
8
8
|
if snippets.values.all?(&:nil?) && records.any?
|
|
9
9
|
snippets = manual_snippets(records, model, query)
|
|
10
10
|
end
|
|
@@ -12,8 +12,8 @@ module FullSearch
|
|
|
12
12
|
records
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
-
def apply_fields!(records, model, query)
|
|
16
|
-
fields = build_field_snippets(model, query)
|
|
15
|
+
def apply_fields!(records, model, query, record_ids: nil)
|
|
16
|
+
fields = build_field_snippets(model, query, record_ids: record_ids)
|
|
17
17
|
if fields.values.all?(&:empty?) && records.any?
|
|
18
18
|
fields = manual_field_snippets(records, model, query)
|
|
19
19
|
end
|
|
@@ -28,8 +28,8 @@ module FullSearch
|
|
|
28
28
|
|
|
29
29
|
private
|
|
30
30
|
|
|
31
|
-
def build_snippets(model, query)
|
|
32
|
-
rows = highlight_rows(model, query)
|
|
31
|
+
def build_snippets(model, query, record_ids: nil)
|
|
32
|
+
rows = highlight_rows(model, query, record_ids: record_ids)
|
|
33
33
|
cols = model.full_search_dsl.fields.map(&:name)
|
|
34
34
|
|
|
35
35
|
rows.to_h do |row|
|
|
@@ -38,8 +38,8 @@ module FullSearch
|
|
|
38
38
|
end
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
-
def build_field_snippets(model, query)
|
|
42
|
-
rows = highlight_rows(model, query)
|
|
41
|
+
def build_field_snippets(model, query, record_ids: nil)
|
|
42
|
+
rows = highlight_rows(model, query, record_ids: record_ids)
|
|
43
43
|
dsl = model.full_search_dsl
|
|
44
44
|
fields = dsl.fields
|
|
45
45
|
open_tag = (dsl.highlight_config || {open_tag: "<mark>"})[:open_tag]
|
|
@@ -175,7 +175,7 @@ module FullSearch
|
|
|
175
175
|
1
|
|
176
176
|
end
|
|
177
177
|
|
|
178
|
-
def highlight_rows(model, query)
|
|
178
|
+
def highlight_rows(model, query, record_ids: nil)
|
|
179
179
|
dsl = model.full_search_dsl
|
|
180
180
|
config = dsl.highlight_config || {open_tag: "<mark>", close_tag: "</mark>"}
|
|
181
181
|
match_expr = QueryParser.to_match_expression(QueryParser.parse(query))
|
|
@@ -195,6 +195,10 @@ module FullSearch
|
|
|
195
195
|
WHERE #{table} MATCH #{q(match_expr)}
|
|
196
196
|
SQL
|
|
197
197
|
|
|
198
|
+
if record_ids.present?
|
|
199
|
+
sql += " AND rowid IN (#{record_ids.map { |id| connection.quote(id) }.join(", ")})"
|
|
200
|
+
end
|
|
201
|
+
|
|
198
202
|
connection.execute(sql)
|
|
199
203
|
end
|
|
200
204
|
|
data/lib/full_search/index.rb
CHANGED
|
@@ -106,8 +106,33 @@ module FullSearch
|
|
|
106
106
|
end
|
|
107
107
|
|
|
108
108
|
def reindex_source_fields!(model)
|
|
109
|
+
dsl = model.full_search_dsl
|
|
110
|
+
return unless dsl
|
|
111
|
+
|
|
109
112
|
model.find_each do |record|
|
|
110
|
-
|
|
113
|
+
dsl.fields.each do |field|
|
|
114
|
+
FullSearch::Callbacks.reindex_field!(record, field.name) if field.source
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def create_triggers!(model)
|
|
120
|
+
connection.execute(insert_trigger_sql(model))
|
|
121
|
+
connection.execute(delete_trigger_sql(model))
|
|
122
|
+
connection.execute(update_trigger_sql(model))
|
|
123
|
+
if model.full_search_dsl.soft_delete_column
|
|
124
|
+
connection.execute(soft_delete_removal_trigger_sql(model))
|
|
125
|
+
end
|
|
126
|
+
if model.full_search_dsl.typo_tolerance?
|
|
127
|
+
connection.execute(insert_trigram_trigger_sql(model))
|
|
128
|
+
connection.execute(delete_trigram_trigger_sql(model))
|
|
129
|
+
connection.execute(update_trigram_trigger_sql(model))
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def drop_triggers!(model)
|
|
134
|
+
(trigger_names(model) + trigram_trigger_names(model)).each do |name|
|
|
135
|
+
connection.execute("DROP TRIGGER IF EXISTS #{qt(name)};")
|
|
111
136
|
end
|
|
112
137
|
end
|
|
113
138
|
|
|
@@ -197,7 +222,7 @@ module FullSearch
|
|
|
197
222
|
|
|
198
223
|
def create_virtual_table_sql(model)
|
|
199
224
|
dsl = model.full_search_dsl
|
|
200
|
-
columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) })
|
|
225
|
+
columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) } + extra_columns(model))
|
|
201
226
|
column_list = columns.map { |c| (c.respond_to?(:unindexed?) && c.unindexed?) ? "#{qc(c.name)} UNINDEXED" : qc(c.name) }.join(", ")
|
|
202
227
|
|
|
203
228
|
"CREATE VIRTUAL TABLE #{qt(fts_table_name(model))} USING fts5(#{column_list}, tokenize='#{dsl.tokenize}');"
|
|
@@ -205,18 +230,22 @@ module FullSearch
|
|
|
205
230
|
|
|
206
231
|
def backfill_sql(model)
|
|
207
232
|
dsl = model.full_search_dsl
|
|
208
|
-
cols = (dsl.fields + dsl.filters)
|
|
233
|
+
cols = (dsl.fields + dsl.filters + extra_columns(model))
|
|
209
234
|
select = cols.map do |c|
|
|
210
|
-
if c.respond_to?(:
|
|
235
|
+
if c.respond_to?(:unindexed?) && c.name == "indexed"
|
|
236
|
+
"'1'"
|
|
237
|
+
elsif c.respond_to?(:source) && c.source
|
|
211
238
|
source_value_sql(c.source)
|
|
212
239
|
else
|
|
213
240
|
"#{qt(model.table_name)}.#{qc(c.name)}"
|
|
214
241
|
end
|
|
215
242
|
end.join(", ")
|
|
216
243
|
|
|
244
|
+
where_clause = dsl.conditional_index? ? " WHERE (#{dsl.index_if_sql})" : ""
|
|
245
|
+
|
|
217
246
|
<<~SQL
|
|
218
247
|
INSERT INTO #{qt(fts_table_name(model))}(rowid, #{cols.map { |c| qc(c.name) }.join(", ")})
|
|
219
|
-
SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)};
|
|
248
|
+
SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)}#{where_clause};
|
|
220
249
|
SQL
|
|
221
250
|
end
|
|
222
251
|
|
|
@@ -225,6 +254,8 @@ module FullSearch
|
|
|
225
254
|
end
|
|
226
255
|
|
|
227
256
|
def ensure_triggers!(model)
|
|
257
|
+
return if FullSearch.bulk_importing?(model)
|
|
258
|
+
|
|
228
259
|
existing = connection.execute(
|
|
229
260
|
"SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=#{q(model.table_name)}"
|
|
230
261
|
).map { |r| r["name"] }
|
|
@@ -236,26 +267,6 @@ module FullSearch
|
|
|
236
267
|
rebuild!(model)
|
|
237
268
|
end
|
|
238
269
|
|
|
239
|
-
def create_triggers!(model)
|
|
240
|
-
connection.execute(insert_trigger_sql(model))
|
|
241
|
-
connection.execute(delete_trigger_sql(model))
|
|
242
|
-
connection.execute(update_trigger_sql(model))
|
|
243
|
-
if model.full_search_dsl.soft_delete_column
|
|
244
|
-
connection.execute(soft_delete_removal_trigger_sql(model))
|
|
245
|
-
end
|
|
246
|
-
if model.full_search_dsl.typo_tolerance?
|
|
247
|
-
connection.execute(insert_trigram_trigger_sql(model))
|
|
248
|
-
connection.execute(delete_trigram_trigger_sql(model))
|
|
249
|
-
connection.execute(update_trigram_trigger_sql(model))
|
|
250
|
-
end
|
|
251
|
-
end
|
|
252
|
-
|
|
253
|
-
def drop_triggers!(model)
|
|
254
|
-
(trigger_names(model) + trigram_trigger_names(model)).each do |name|
|
|
255
|
-
connection.execute("DROP TRIGGER IF EXISTS #{qt(name)};")
|
|
256
|
-
end
|
|
257
|
-
end
|
|
258
|
-
|
|
259
270
|
def trigger_names(model)
|
|
260
271
|
base = fts_table_name(model)
|
|
261
272
|
names = %W[#{base}_ai #{base}_ad #{base}_au]
|
|
@@ -270,15 +281,30 @@ module FullSearch
|
|
|
270
281
|
|
|
271
282
|
def insert_trigger_sql(model)
|
|
272
283
|
dsl = model.full_search_dsl
|
|
273
|
-
cols = dsl.fields + dsl.filters
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
284
|
+
cols = dsl.fields + dsl.filters + extra_columns(model)
|
|
285
|
+
cols_str = col_names(cols)
|
|
286
|
+
fts_table = qt(fts_table_name(model))
|
|
287
|
+
tbl = qt(model.table_name)
|
|
288
|
+
|
|
289
|
+
if dsl.conditional_index?
|
|
290
|
+
select_parts = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
291
|
+
<<~SQL
|
|
292
|
+
CREATE TRIGGER #{qt(trigger_names(model).first)} AFTER INSERT ON #{tbl}
|
|
293
|
+
BEGIN
|
|
294
|
+
INSERT INTO #{fts_table}(rowid, #{cols_str})
|
|
295
|
+
SELECT new.id, #{select_parts} FROM #{tbl} WHERE #{tbl}.id = new.id AND (#{dsl.index_if_sql});
|
|
296
|
+
END;
|
|
297
|
+
SQL
|
|
298
|
+
else
|
|
299
|
+
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
300
|
+
<<~SQL
|
|
301
|
+
CREATE TRIGGER #{qt(trigger_names(model).first)} AFTER INSERT ON #{tbl}
|
|
302
|
+
BEGIN
|
|
303
|
+
INSERT INTO #{fts_table}(rowid, #{col_names(cols)})
|
|
304
|
+
VALUES (new.id, #{values});
|
|
305
|
+
END;
|
|
306
|
+
SQL
|
|
307
|
+
end
|
|
282
308
|
end
|
|
283
309
|
|
|
284
310
|
def delete_trigger_sql(model)
|
|
@@ -291,21 +317,34 @@ module FullSearch
|
|
|
291
317
|
|
|
292
318
|
def update_trigger_sql(model)
|
|
293
319
|
dsl = model.full_search_dsl
|
|
294
|
-
cols = dsl.fields + dsl.filters
|
|
295
|
-
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
320
|
+
cols = dsl.fields + dsl.filters + extra_columns(model)
|
|
296
321
|
cols_str = col_names(cols)
|
|
297
322
|
fts_table = qt(fts_table_name(model))
|
|
323
|
+
tbl = qt(model.table_name)
|
|
298
324
|
|
|
299
325
|
when_clause = SoftDelete.active_update_clause(model)
|
|
300
326
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
327
|
+
if dsl.conditional_index?
|
|
328
|
+
select_parts = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
329
|
+
<<~SQL
|
|
330
|
+
CREATE TRIGGER #{qt(trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
|
|
331
|
+
BEGIN
|
|
332
|
+
DELETE FROM #{fts_table} WHERE rowid = old.id;
|
|
333
|
+
INSERT INTO #{fts_table}(rowid, #{cols_str})
|
|
334
|
+
SELECT new.id, #{select_parts} FROM #{tbl} WHERE #{tbl}.id = new.id AND (#{dsl.index_if_sql});
|
|
335
|
+
END;
|
|
336
|
+
SQL
|
|
337
|
+
else
|
|
338
|
+
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
339
|
+
<<~SQL
|
|
340
|
+
CREATE TRIGGER #{qt(trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
|
|
341
|
+
BEGIN
|
|
342
|
+
DELETE FROM #{fts_table} WHERE rowid = old.id;
|
|
343
|
+
INSERT INTO #{fts_table}(rowid, #{cols_str})
|
|
344
|
+
VALUES (new.id, #{values});
|
|
345
|
+
END;
|
|
346
|
+
SQL
|
|
347
|
+
end
|
|
309
348
|
end
|
|
310
349
|
|
|
311
350
|
def soft_delete_removal_trigger_sql(model)
|
|
@@ -329,7 +368,7 @@ module FullSearch
|
|
|
329
368
|
|
|
330
369
|
def create_trigram_virtual_table_sql(model)
|
|
331
370
|
dsl = model.full_search_dsl
|
|
332
|
-
columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) })
|
|
371
|
+
columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) } + extra_columns(model))
|
|
333
372
|
column_list = columns.map { |c| (c.respond_to?(:unindexed?) && c.unindexed?) ? "#{qc(c.name)} UNINDEXED" : qc(c.name) }.join(", ")
|
|
334
373
|
|
|
335
374
|
"CREATE VIRTUAL TABLE #{qt(trigram_table_name(model))} USING fts5(#{column_list}, tokenize='trigram');"
|
|
@@ -337,32 +376,51 @@ module FullSearch
|
|
|
337
376
|
|
|
338
377
|
def backfill_trigram_sql(model)
|
|
339
378
|
dsl = model.full_search_dsl
|
|
340
|
-
cols = (dsl.fields + dsl.filters)
|
|
379
|
+
cols = (dsl.fields + dsl.filters + extra_columns(model))
|
|
341
380
|
select = cols.map do |c|
|
|
342
|
-
if c.respond_to?(:
|
|
381
|
+
if c.respond_to?(:unindexed?) && c.name == "indexed"
|
|
382
|
+
"'1'"
|
|
383
|
+
elsif c.respond_to?(:source) && c.source
|
|
343
384
|
source_value_sql(c.source)
|
|
344
385
|
else
|
|
345
386
|
"#{qt(model.table_name)}.#{qc(c.name)}"
|
|
346
387
|
end
|
|
347
388
|
end.join(", ")
|
|
348
389
|
|
|
390
|
+
where_clause = dsl.conditional_index? ? " WHERE (#{dsl.index_if_sql})" : ""
|
|
391
|
+
|
|
349
392
|
<<~SQL
|
|
350
393
|
INSERT INTO #{qt(trigram_table_name(model))}(rowid, #{cols.map { |c| qc(c.name) }.join(", ")})
|
|
351
|
-
SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)};
|
|
394
|
+
SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)}#{where_clause};
|
|
352
395
|
SQL
|
|
353
396
|
end
|
|
354
397
|
|
|
355
398
|
def insert_trigram_trigger_sql(model)
|
|
356
399
|
dsl = model.full_search_dsl
|
|
357
|
-
cols = dsl.fields + dsl.filters
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
400
|
+
cols = dsl.fields + dsl.filters + extra_columns(model)
|
|
401
|
+
cols_str = col_names(cols)
|
|
402
|
+
trigram_table = qt(trigram_table_name(model))
|
|
403
|
+
tbl = qt(model.table_name)
|
|
404
|
+
|
|
405
|
+
if dsl.conditional_index?
|
|
406
|
+
select_parts = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
407
|
+
<<~SQL
|
|
408
|
+
CREATE TRIGGER #{qt(trigram_trigger_names(model).first)} AFTER INSERT ON #{tbl}
|
|
409
|
+
BEGIN
|
|
410
|
+
INSERT INTO #{trigram_table}(rowid, #{cols_str})
|
|
411
|
+
SELECT new.id, #{select_parts} FROM #{tbl} WHERE #{tbl}.id = new.id AND (#{dsl.index_if_sql});
|
|
412
|
+
END;
|
|
413
|
+
SQL
|
|
414
|
+
else
|
|
415
|
+
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
416
|
+
<<~SQL
|
|
417
|
+
CREATE TRIGGER #{qt(trigram_trigger_names(model).first)} AFTER INSERT ON #{tbl}
|
|
418
|
+
BEGIN
|
|
419
|
+
INSERT INTO #{trigram_table}(rowid, #{col_names(cols)})
|
|
420
|
+
VALUES (new.id, #{values});
|
|
421
|
+
END;
|
|
422
|
+
SQL
|
|
423
|
+
end
|
|
366
424
|
end
|
|
367
425
|
|
|
368
426
|
def delete_trigram_trigger_sql(model)
|
|
@@ -375,21 +433,39 @@ module FullSearch
|
|
|
375
433
|
|
|
376
434
|
def update_trigram_trigger_sql(model)
|
|
377
435
|
dsl = model.full_search_dsl
|
|
378
|
-
cols = dsl.fields + dsl.filters
|
|
379
|
-
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
436
|
+
cols = dsl.fields + dsl.filters + extra_columns(model)
|
|
380
437
|
cols_str = col_names(cols)
|
|
381
438
|
trigram_table = qt(trigram_table_name(model))
|
|
439
|
+
tbl = qt(model.table_name)
|
|
382
440
|
|
|
383
441
|
when_clause = SoftDelete.active_update_clause(model)
|
|
384
442
|
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
443
|
+
if dsl.conditional_index?
|
|
444
|
+
select_parts = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
445
|
+
<<~SQL
|
|
446
|
+
CREATE TRIGGER #{qt(trigram_trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
|
|
447
|
+
BEGIN
|
|
448
|
+
DELETE FROM #{trigram_table} WHERE rowid = old.id;
|
|
449
|
+
INSERT INTO #{trigram_table}(rowid, #{cols_str})
|
|
450
|
+
SELECT new.id, #{select_parts} FROM #{tbl} WHERE #{tbl}.id = new.id AND (#{dsl.index_if_sql});
|
|
451
|
+
END;
|
|
452
|
+
SQL
|
|
453
|
+
else
|
|
454
|
+
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
455
|
+
<<~SQL
|
|
456
|
+
CREATE TRIGGER #{qt(trigram_trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
|
|
457
|
+
BEGIN
|
|
458
|
+
DELETE FROM #{trigram_table} WHERE rowid = old.id;
|
|
459
|
+
INSERT INTO #{trigram_table}(rowid, #{cols_str})
|
|
460
|
+
VALUES (new.id, #{values});
|
|
461
|
+
END;
|
|
462
|
+
SQL
|
|
463
|
+
end
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
def extra_columns(model)
|
|
467
|
+
dsl = model.full_search_dsl
|
|
468
|
+
dsl&.conditional_index? ? [FilterColumnPlaceholder.new(name: "indexed")] : []
|
|
393
469
|
end
|
|
394
470
|
|
|
395
471
|
def col_names(cols)
|
|
@@ -397,7 +473,9 @@ module FullSearch
|
|
|
397
473
|
end
|
|
398
474
|
|
|
399
475
|
def column_ref(col, prefix:)
|
|
400
|
-
if col.respond_to?(:
|
|
476
|
+
if col.respond_to?(:unindexed?) && col.name == "indexed"
|
|
477
|
+
"'1'"
|
|
478
|
+
elsif col.respond_to?(:source) && col.source
|
|
401
479
|
"''"
|
|
402
480
|
else
|
|
403
481
|
"#{prefix}.#{qc(col.name)}"
|
data/lib/full_search/model.rb
CHANGED
|
@@ -5,19 +5,18 @@ module FullSearch
|
|
|
5
5
|
extend ActiveSupport::Concern
|
|
6
6
|
|
|
7
7
|
class_methods do
|
|
8
|
-
def full_search(query_or_options = nil, filters: {}, include_soft_deleted: false, limit: nil, offset: nil, highlight: false, highlight_fields: false, matching_strategy: nil, &block)
|
|
8
|
+
def full_search(query_or_options = nil, filters: {}, include_soft_deleted: false, limit: nil, offset: nil, highlight: false, highlight_fields: false, matching_strategy: nil, per_strategy_limit: nil, &block)
|
|
9
9
|
if block_given? || query_or_options.is_a?(Hash)
|
|
10
10
|
@full_search_dsl ||= FullSearch::Dsl.new(self)
|
|
11
11
|
@full_search_dsl.tokenize(query_or_options[:tokenize]) if query_or_options.is_a?(Hash) && query_or_options.key?(:tokenize)
|
|
12
12
|
@full_search_dsl.instance_eval(&block) if block_given?
|
|
13
13
|
FullSearch::Callbacks.install!(self)
|
|
14
|
-
FullSearch::Index.ensure_table!(self)
|
|
15
14
|
include InstanceMethods
|
|
16
15
|
|
|
17
16
|
FullSearch.register_model(self)
|
|
18
17
|
@full_search_dsl
|
|
19
18
|
else
|
|
20
|
-
FullSearch::Search.new(self, query_or_options, filters: filters, include_soft_deleted: include_soft_deleted, limit: limit, offset: offset, highlight: highlight, highlight_fields: highlight_fields, matching_strategy: matching_strategy).relation
|
|
19
|
+
FullSearch::Search.new(self, query_or_options, filters: filters, include_soft_deleted: include_soft_deleted, limit: limit, offset: offset, highlight: highlight, highlight_fields: highlight_fields, matching_strategy: matching_strategy, per_strategy_limit: per_strategy_limit).relation
|
|
21
20
|
end
|
|
22
21
|
end
|
|
23
22
|
|
|
@@ -40,6 +39,14 @@ module FullSearch
|
|
|
40
39
|
def reindex!
|
|
41
40
|
FullSearch::Index.reindex_source_fields!(self)
|
|
42
41
|
end
|
|
42
|
+
|
|
43
|
+
def bulk_import(&block)
|
|
44
|
+
FullSearch.bulk_import(self, &block)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def backfill_search_index!
|
|
48
|
+
FullSearch::BackfillJob.perform_later(name)
|
|
49
|
+
end
|
|
43
50
|
end
|
|
44
51
|
|
|
45
52
|
included do |base|
|
|
@@ -48,6 +55,16 @@ module FullSearch
|
|
|
48
55
|
end
|
|
49
56
|
end
|
|
50
57
|
|
|
58
|
+
def self.evaluate_source(record, field)
|
|
59
|
+
if field.source
|
|
60
|
+
record.instance_exec(&field.source)
|
|
61
|
+
else
|
|
62
|
+
record.public_send(field.name)
|
|
63
|
+
end
|
|
64
|
+
rescue NameError => e
|
|
65
|
+
raise e.class, "#{record.class.name}: full_search source block for field #{field.name.inspect} failed: #{e.message}"
|
|
66
|
+
end
|
|
67
|
+
|
|
51
68
|
module InstanceMethods
|
|
52
69
|
attr_accessor :full_search_snippet, :full_search_highlight_fields
|
|
53
70
|
|
|
@@ -56,7 +73,7 @@ module FullSearch
|
|
|
56
73
|
field = dsl.fields.find { |f| f.name == field_name.to_s || f.as == field_name.to_s }
|
|
57
74
|
return nil unless field
|
|
58
75
|
|
|
59
|
-
|
|
76
|
+
FullSearch::Model.evaluate_source(self, field)
|
|
60
77
|
end
|
|
61
78
|
end
|
|
62
79
|
end
|
|
@@ -27,7 +27,9 @@ module FullSearch
|
|
|
27
27
|
limit: raw_limit,
|
|
28
28
|
offset: offset,
|
|
29
29
|
highlight: group[:highlight],
|
|
30
|
-
highlight_fields: group[:highlight_fields]
|
|
30
|
+
highlight_fields: group[:highlight_fields],
|
|
31
|
+
matching_strategy: group[:matching_strategy],
|
|
32
|
+
per_strategy_limit: group[:per_strategy_limit]
|
|
31
33
|
)
|
|
32
34
|
|
|
33
35
|
relation = group[:scope].call(relation) if group[:scope]
|
|
@@ -4,12 +4,18 @@ require "active_job"
|
|
|
4
4
|
|
|
5
5
|
module FullSearch
|
|
6
6
|
class ReindexJob < ActiveJob::Base
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
queue_as :low
|
|
8
|
+
|
|
9
|
+
def perform(model_name, record_id, field_name = nil)
|
|
10
|
+
model = model_name.is_a?(Class) ? model_name : model_name.to_s.constantize
|
|
9
11
|
record = model.find_by(id: record_id)
|
|
10
12
|
return unless record
|
|
11
13
|
|
|
12
|
-
|
|
14
|
+
if field_name
|
|
15
|
+
FullSearch::Callbacks.reindex_field!(record, field_name)
|
|
16
|
+
else
|
|
17
|
+
FullSearch::Callbacks.reindex_record!(record)
|
|
18
|
+
end
|
|
13
19
|
end
|
|
14
20
|
end
|
|
15
21
|
end
|
data/lib/full_search/search.rb
CHANGED
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
module FullSearch
|
|
4
4
|
class Search
|
|
5
|
-
attr_reader :model, :query, :filters, :include_soft_deleted, :limit, :offset, :highlight, :highlight_fields, :matching_strategy
|
|
5
|
+
attr_reader :model, :query, :filters, :include_soft_deleted, :limit, :offset, :highlight, :highlight_fields, :matching_strategy, :per_strategy_limit
|
|
6
6
|
|
|
7
|
-
def initialize(model, query, filters:, include_soft_deleted:, limit:, offset:, highlight: false, highlight_fields: false, matching_strategy: nil)
|
|
7
|
+
def initialize(model, query, filters:, include_soft_deleted:, limit:, offset:, highlight: false, highlight_fields: false, matching_strategy: nil, per_strategy_limit: nil)
|
|
8
8
|
@model = model
|
|
9
9
|
@query = query.to_s.strip
|
|
10
10
|
@filters = filters
|
|
@@ -14,6 +14,7 @@ module FullSearch
|
|
|
14
14
|
@highlight = highlight
|
|
15
15
|
@highlight_fields = highlight_fields
|
|
16
16
|
@matching_strategy = matching_strategy
|
|
17
|
+
@per_strategy_limit = per_strategy_limit
|
|
17
18
|
end
|
|
18
19
|
|
|
19
20
|
MIN_TERM_LENGTH = 3
|
|
@@ -28,8 +29,8 @@ module FullSearch
|
|
|
28
29
|
parsed = QueryParser.parse(query)
|
|
29
30
|
exact_ids = ExactMatch.ids_for(model, query, filters)
|
|
30
31
|
primary_ids = fts_match_ids(parsed)
|
|
31
|
-
fallback_ids = (dsl.typo_tolerance? && matching_strategy != "all") ? trigram_match_ids(parsed, primary_ids) : []
|
|
32
|
-
fuzzy_ids = (dsl.typo_tolerance? && matching_strategy != "all" && primary_ids.empty? && fallback_ids.empty?) ? fuzzy_match_ids(parsed) : []
|
|
32
|
+
fallback_ids = (dsl.typo_tolerance? && matching_strategy != "all") ? trigram_match_ids(parsed, primary_ids, candidate_limit: per_strategy_limit) : []
|
|
33
|
+
fuzzy_ids = (dsl.typo_tolerance? && matching_strategy != "all" && primary_ids.empty? && fallback_ids.empty?) ? fuzzy_match_ids(parsed, candidate_limit: per_strategy_limit) : []
|
|
33
34
|
|
|
34
35
|
all_ids = (exact_ids + primary_ids + fallback_ids + fuzzy_ids).uniq
|
|
35
36
|
return model.none if all_ids.empty?
|
|
@@ -42,9 +43,13 @@ module FullSearch
|
|
|
42
43
|
rel = apply_ranking(rel, all_ids, exact_ids)
|
|
43
44
|
|
|
44
45
|
if highlight
|
|
45
|
-
|
|
46
|
+
records = rel.to_a
|
|
47
|
+
Highlighter.apply!(records, model, query, record_ids: records.map(&:id))
|
|
48
|
+
records
|
|
46
49
|
elsif highlight_fields
|
|
47
|
-
|
|
50
|
+
records = rel.to_a
|
|
51
|
+
Highlighter.apply_fields!(records, model, query, record_ids: records.map(&:id))
|
|
52
|
+
records
|
|
48
53
|
else
|
|
49
54
|
rel
|
|
50
55
|
end
|
|
@@ -76,13 +81,18 @@ module FullSearch
|
|
|
76
81
|
stored = FullSearch::Index.stored_config_hash(model)
|
|
77
82
|
return unless stored
|
|
78
83
|
|
|
79
|
-
if stored
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
return if stored == dsl.config_hash
|
|
85
|
+
|
|
86
|
+
if FullSearch.config.auto_rebuild_on_stale_query
|
|
87
|
+
FullSearch::Index.rebuild_if_needed!(model)
|
|
88
|
+
return
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
case FullSearch.config.stale_query_behavior
|
|
92
|
+
when :raise
|
|
93
|
+
raise ConfigChangedError, "FTS index for #{model.table_name} is stale; run full_search:rebuild"
|
|
94
|
+
when :log_and_fallback
|
|
95
|
+
Rails.logger.warn("[full_search] FTS index for #{model.table_name} is stale; results may be incomplete")
|
|
86
96
|
end
|
|
87
97
|
end
|
|
88
98
|
|
|
@@ -104,10 +114,12 @@ module FullSearch
|
|
|
104
114
|
"AND #{fts_table}.#{qc(name)} = #{q(value)}"
|
|
105
115
|
end.join(" ")
|
|
106
116
|
|
|
107
|
-
|
|
117
|
+
indexed_filter = dsl.conditional_index? ? " AND #{fts_table}.indexed = '1'" : ""
|
|
118
|
+
|
|
119
|
+
connection.execute("#{sql} #{filter_conditions}#{indexed_filter}").map { |r| r["id"] }
|
|
108
120
|
end
|
|
109
121
|
|
|
110
|
-
def trigram_match_ids(parsed, primary_ids)
|
|
122
|
+
def trigram_match_ids(parsed, primary_ids, candidate_limit: nil)
|
|
111
123
|
return [] if primary_ids.any?
|
|
112
124
|
|
|
113
125
|
match_expr = QueryParser.to_match_expression(parsed)
|
|
@@ -117,7 +129,7 @@ module FullSearch
|
|
|
117
129
|
return [] if term.nil?
|
|
118
130
|
|
|
119
131
|
if term.length < dsl.typo_tolerance_min_term_length.to_i
|
|
120
|
-
return like_prefix_ids(term)
|
|
132
|
+
return like_prefix_ids(term, candidate_limit: candidate_limit)
|
|
121
133
|
end
|
|
122
134
|
|
|
123
135
|
trigram_table = qt(FullSearch::Index.trigram_table_name(model))
|
|
@@ -134,10 +146,14 @@ module FullSearch
|
|
|
134
146
|
"AND #{trigram_table}.#{qc(name)} = #{q(value)}"
|
|
135
147
|
end.join(" ")
|
|
136
148
|
|
|
137
|
-
|
|
149
|
+
indexed_filter = dsl.conditional_index? ? " AND #{trigram_table}.indexed = '1'" : ""
|
|
150
|
+
|
|
151
|
+
sql += filter_conditions + indexed_filter
|
|
152
|
+
order_and_limit!(sql, tbl, candidate_limit)
|
|
153
|
+
connection.execute(sql).map { |r| r["id"] }
|
|
138
154
|
end
|
|
139
155
|
|
|
140
|
-
def like_prefix_ids(term)
|
|
156
|
+
def like_prefix_ids(term, candidate_limit: nil)
|
|
141
157
|
column_fields = dsl.fields.select { |f| f.source.nil? }
|
|
142
158
|
source_fields = dsl.fields.select { |f| f.source }
|
|
143
159
|
tbl = qt(model.table_name)
|
|
@@ -147,6 +163,8 @@ module FullSearch
|
|
|
147
163
|
soft_delete_clause = "AND #{tbl}.#{qc(dsl.soft_delete_column)} IS NULL"
|
|
148
164
|
end
|
|
149
165
|
|
|
166
|
+
indexed_clause = dsl.conditional_index? ? "AND (#{dsl.index_if_sql}) " : ""
|
|
167
|
+
|
|
150
168
|
ids = []
|
|
151
169
|
|
|
152
170
|
if column_fields.any?
|
|
@@ -161,9 +179,10 @@ module FullSearch
|
|
|
161
179
|
sql = <<~SQL
|
|
162
180
|
SELECT #{tbl}.id
|
|
163
181
|
FROM #{tbl}
|
|
164
|
-
WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause}
|
|
182
|
+
WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause} #{indexed_clause}
|
|
165
183
|
SQL
|
|
166
184
|
|
|
185
|
+
order_and_limit!(sql, tbl, candidate_limit)
|
|
167
186
|
ids = connection.execute(sql).map { |r| r["id"] }
|
|
168
187
|
return ids if ids.any?
|
|
169
188
|
end
|
|
@@ -178,20 +197,23 @@ module FullSearch
|
|
|
178
197
|
"AND #{fts_table}.#{qc(name)} = #{q(value)}"
|
|
179
198
|
end.join(" ")
|
|
180
199
|
|
|
200
|
+
fts_indexed = dsl.conditional_index? ? " AND #{fts_table}.indexed = '1'" : ""
|
|
201
|
+
|
|
181
202
|
sql = <<~SQL
|
|
182
203
|
SELECT #{tbl}.id
|
|
183
204
|
FROM #{fts_table}
|
|
184
205
|
JOIN #{tbl} ON #{tbl}.id = #{fts_table}.rowid
|
|
185
|
-
WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause}
|
|
206
|
+
WHERE (#{like_conditions}) #{filter_conditions} #{soft_delete_clause}#{fts_indexed}
|
|
186
207
|
SQL
|
|
187
208
|
|
|
209
|
+
order_and_limit!(sql, tbl, candidate_limit)
|
|
188
210
|
ids = connection.execute(sql).map { |r| r["id"] }
|
|
189
211
|
end
|
|
190
212
|
|
|
191
213
|
ids
|
|
192
214
|
end
|
|
193
215
|
|
|
194
|
-
def fuzzy_match_ids(parsed)
|
|
216
|
+
def fuzzy_match_ids(parsed, candidate_limit: nil)
|
|
195
217
|
term = parsed.last
|
|
196
218
|
return [] if term.nil?
|
|
197
219
|
|
|
@@ -212,6 +234,8 @@ module FullSearch
|
|
|
212
234
|
soft_delete_clause = "AND #{tbl}.#{qc(dsl.soft_delete_column)} IS NULL"
|
|
213
235
|
end
|
|
214
236
|
|
|
237
|
+
indexed_clause = dsl.conditional_index? ? "AND (#{dsl.index_if_sql}) " : ""
|
|
238
|
+
|
|
215
239
|
filter_conditions = filters.map do |name, value|
|
|
216
240
|
"AND #{tbl}.#{qc(name)} = #{q(value)}"
|
|
217
241
|
end.join(" ")
|
|
@@ -223,12 +247,21 @@ module FullSearch
|
|
|
223
247
|
sql = <<~SQL
|
|
224
248
|
SELECT #{tbl}.id
|
|
225
249
|
FROM #{tbl}
|
|
226
|
-
WHERE (#{conditions}) #{filter_conditions} #{soft_delete_clause}
|
|
250
|
+
WHERE (#{conditions}) #{filter_conditions} #{soft_delete_clause} #{indexed_clause}
|
|
227
251
|
SQL
|
|
228
252
|
|
|
253
|
+
order_and_limit!(sql, tbl, candidate_limit)
|
|
229
254
|
connection.execute(sql).map { |r| r["id"] }
|
|
230
255
|
end
|
|
231
256
|
|
|
257
|
+
def order_and_limit!(sql, tbl, candidate_limit)
|
|
258
|
+
order_parts = dsl.rank_bys.map do |rank_by|
|
|
259
|
+
"#{tbl}.#{qc(rank_by.column)} #{rank_by.direction.to_s.upcase} NULLS LAST"
|
|
260
|
+
end
|
|
261
|
+
sql << " ORDER BY #{order_parts.join(", ")}" if order_parts.any?
|
|
262
|
+
sql << " LIMIT #{candidate_limit.to_i}" if candidate_limit
|
|
263
|
+
end
|
|
264
|
+
|
|
232
265
|
def max_allowed_typos(length)
|
|
233
266
|
min_length = dsl.typo_tolerance_min_term_length.to_i
|
|
234
267
|
return -1 if length < min_length
|
data/lib/full_search/version.rb
CHANGED
data/lib/full_search.rb
CHANGED
|
@@ -22,6 +22,8 @@ require "full_search/soft_delete"
|
|
|
22
22
|
require "full_search/callbacks"
|
|
23
23
|
require "full_search/reindex_job"
|
|
24
24
|
require "full_search/optimize_job"
|
|
25
|
+
require "full_search/bulk_import"
|
|
26
|
+
require "full_search/backfill_job"
|
|
25
27
|
require "full_search/test_helpers"
|
|
26
28
|
require "full_search/multi_search"
|
|
27
29
|
require "full_search/schema_dumper_patch"
|
|
@@ -54,6 +56,14 @@ module FullSearch
|
|
|
54
56
|
Callbacks.install!(model)
|
|
55
57
|
end
|
|
56
58
|
end
|
|
59
|
+
|
|
60
|
+
def bulk_import(model, &block)
|
|
61
|
+
BulkImport.bulk_import(model, &block)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def bulk_importing?(model)
|
|
65
|
+
BulkImport.bulk_importing?(model)
|
|
66
|
+
end
|
|
57
67
|
end
|
|
58
68
|
end
|
|
59
69
|
|
|
@@ -3,7 +3,13 @@
|
|
|
3
3
|
FullSearch.configure do |config|
|
|
4
4
|
config.auto_rebuild_schema = Rails.env.local?
|
|
5
5
|
|
|
6
|
+
config.auto_rebuild_on_stale_query = Rails.env.local?
|
|
7
|
+
|
|
6
8
|
config.stale_query_behavior = Rails.env.production? ? :log_and_fallback : :raise
|
|
7
9
|
|
|
8
10
|
config.lock_rebuilds = true
|
|
11
|
+
|
|
12
|
+
# Reindex computed source: fields synchronously (false) or via background job (true).
|
|
13
|
+
# Bulk imports should use FullSearch.bulk_import(Model) { ... } to defer reindexing.
|
|
14
|
+
config.default_async_source_reindex = true
|
|
9
15
|
end
|
data/lib/tasks/full_search.rake
CHANGED
|
@@ -40,6 +40,16 @@ namespace :full_search do
|
|
|
40
40
|
end
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
+
desc "Backfill FTS indexes for given models (or all)"
|
|
44
|
+
task :backfill, [:models] => :environment do |_t, args|
|
|
45
|
+
Rails.application.eager_load!
|
|
46
|
+
|
|
47
|
+
resolve_full_search_models(args).each do |model|
|
|
48
|
+
FullSearch::BackfillJob.perform_now(model.name)
|
|
49
|
+
puts "Backfilled #{FullSearch::Index.fts_table_name(model)}"
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
43
53
|
desc "Optimize full_search indexes"
|
|
44
54
|
task optimize: :environment do
|
|
45
55
|
Rails.application.eager_load!
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: full_search
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ben D'Angelo
|
|
@@ -140,6 +140,8 @@ extra_rdoc_files: []
|
|
|
140
140
|
files:
|
|
141
141
|
- README.md
|
|
142
142
|
- lib/full_search.rb
|
|
143
|
+
- lib/full_search/backfill_job.rb
|
|
144
|
+
- lib/full_search/bulk_import.rb
|
|
143
145
|
- lib/full_search/callbacks.rb
|
|
144
146
|
- lib/full_search/config.rb
|
|
145
147
|
- lib/full_search/distance.rb
|