full_search 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +115 -9
- data/lib/full_search/callbacks.rb +26 -8
- data/lib/full_search/config.rb +2 -2
- data/lib/full_search/distance.rb +33 -0
- data/lib/full_search/dsl.rb +29 -12
- data/lib/full_search/errors.rb +2 -0
- data/lib/full_search/exact_match.rb +7 -4
- data/lib/full_search/highlighter.rb +175 -80
- data/lib/full_search/index.rb +134 -44
- data/lib/full_search/model.rb +23 -4
- data/lib/full_search/multi_search.rb +4 -1
- data/lib/full_search/optimize_job.rb +13 -0
- data/lib/full_search/query_parser.rb +101 -88
- data/lib/full_search/quoting.rb +11 -0
- data/lib/full_search/railtie.rb +2 -8
- data/lib/full_search/search.rb +161 -30
- data/lib/full_search/soft_delete.rb +15 -2
- data/lib/full_search/typo.rb +10 -8
- data/lib/full_search/version.rb +1 -1
- data/lib/full_search.rb +16 -1
- data/lib/generators/full_search/install/templates/full_search.rb +1 -1
- data/lib/tasks/full_search.rake +50 -14
- metadata +55 -12
|
@@ -2,112 +2,207 @@
|
|
|
2
2
|
|
|
3
3
|
module FullSearch
|
|
4
4
|
class Highlighter
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
snippets
|
|
5
|
+
class << self
|
|
6
|
+
def apply!(records, model, query)
|
|
7
|
+
snippets = build_snippets(model, query)
|
|
8
|
+
if snippets.values.all?(&:nil?) && records.any?
|
|
9
|
+
snippets = manual_snippets(records, model, query)
|
|
10
|
+
end
|
|
11
|
+
records.each { |record| record.full_search_snippet = snippets[record.id] }
|
|
12
|
+
records
|
|
9
13
|
end
|
|
10
|
-
records.each { |record| record.full_search_snippet = snippets[record.id] }
|
|
11
|
-
records
|
|
12
|
-
end
|
|
13
14
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
def apply_fields!(records, model, query)
|
|
16
|
+
fields = build_field_snippets(model, query)
|
|
17
|
+
if fields.values.all?(&:empty?) && records.any?
|
|
18
|
+
fields = manual_field_snippets(records, model, query)
|
|
19
|
+
end
|
|
20
|
+
exact_fields = exact_match_field_snippets(records, model, query)
|
|
21
|
+
records.each do |record|
|
|
22
|
+
merged = fields[record.id] || {}
|
|
23
|
+
exact_fields.fetch(record.id, {}).each { |k, v| merged[k] ||= v }
|
|
24
|
+
record.full_search_highlight_fields = merged
|
|
25
|
+
end
|
|
26
|
+
records
|
|
18
27
|
end
|
|
19
|
-
records.each { |record| record.full_search_highlight_fields = fields[record.id] || {} }
|
|
20
|
-
records
|
|
21
|
-
end
|
|
22
28
|
|
|
23
|
-
|
|
29
|
+
private
|
|
24
30
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
31
|
+
def build_snippets(model, query)
|
|
32
|
+
rows = highlight_rows(model, query)
|
|
33
|
+
cols = model.full_search_dsl.fields.map(&:name)
|
|
28
34
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
35
|
+
rows.to_h do |row|
|
|
36
|
+
text = cols.map { |col| row["#{col}_snippet"] }.compact.join(" ").strip
|
|
37
|
+
[row["rowid"], text.presence]
|
|
38
|
+
end
|
|
32
39
|
end
|
|
33
|
-
end
|
|
34
40
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
41
|
+
def build_field_snippets(model, query)
|
|
42
|
+
rows = highlight_rows(model, query)
|
|
43
|
+
dsl = model.full_search_dsl
|
|
44
|
+
fields = dsl.fields
|
|
45
|
+
open_tag = (dsl.highlight_config || {open_tag: "<mark>"})[:open_tag]
|
|
46
|
+
|
|
47
|
+
rows.to_h do |row|
|
|
48
|
+
snippets = fields.each_with_object({}) do |field, hash|
|
|
49
|
+
snippet = row["#{field.name}_snippet"].to_s.strip
|
|
50
|
+
key = field.as || field.name
|
|
51
|
+
hash[key] = snippet if snippet.include?(open_tag)
|
|
52
|
+
end
|
|
53
|
+
[row["rowid"], snippets]
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def compute_max_typos(query, dsl)
|
|
58
|
+
return nil unless dsl.typo_tolerance?
|
|
59
|
+
length = query.length
|
|
60
|
+
min_length = dsl.typo_tolerance_min_term_length.to_i
|
|
61
|
+
return nil if length < min_length
|
|
62
|
+
return 2 if length >= 9
|
|
63
|
+
1
|
|
64
|
+
end
|
|
40
65
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
66
|
+
def manual_snippets(records, model, query)
|
|
67
|
+
dsl = model.full_search_dsl
|
|
68
|
+
config = dsl.highlight_config || {open_tag: "<mark>", close_tag: "</mark>"}
|
|
69
|
+
cols = dsl.fields.map(&:name)
|
|
70
|
+
max_typos = compute_max_typos(query, dsl)
|
|
71
|
+
|
|
72
|
+
records.to_h do |record|
|
|
73
|
+
text = cols.map { |col| record.full_search_text_for(col).to_s }.join(" ").strip
|
|
74
|
+
highlighted = manual_highlight(text, query, config, max_typos: max_typos)
|
|
75
|
+
[record.id, highlighted.presence]
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def manual_field_snippets(records, model, query)
|
|
80
|
+
dsl = model.full_search_dsl
|
|
81
|
+
config = dsl.highlight_config || {open_tag: "<mark>", close_tag: "</mark>"}
|
|
82
|
+
fields = dsl.fields
|
|
83
|
+
max_typos = compute_max_typos(query, dsl)
|
|
84
|
+
|
|
85
|
+
records.to_h do |record|
|
|
86
|
+
snippets = fields.each_with_object({}) do |field, hash|
|
|
87
|
+
value = record.full_search_text_for(field.name).to_s
|
|
88
|
+
highlighted = manual_highlight(value, query, config, max_typos: max_typos)
|
|
89
|
+
key = field.as || field.name
|
|
90
|
+
hash[key] = highlighted if highlighted.include?(config[:open_tag])
|
|
91
|
+
end
|
|
92
|
+
[record.id, snippets]
|
|
45
93
|
end
|
|
46
|
-
[row["rowid"], snippets]
|
|
47
94
|
end
|
|
48
|
-
end
|
|
49
95
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
cols = dsl.fields.map(&:name)
|
|
96
|
+
def exact_match_field_snippets(records, model, query)
|
|
97
|
+
dsl = model.full_search_dsl
|
|
98
|
+
return {} if dsl.exact_matches.empty?
|
|
54
99
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
100
|
+
config = dsl.highlight_config || {open_tag: "<mark>", close_tag: "</mark>"}
|
|
101
|
+
max_typos = compute_max_typos(query, dsl)
|
|
102
|
+
|
|
103
|
+
records.to_h do |record|
|
|
104
|
+
snippets = dsl.exact_matches.each_with_object({}) do |em, hash|
|
|
105
|
+
value = exact_match_field_value(em, record)
|
|
106
|
+
highlighted = manual_highlight(value.to_s, query, config, max_typos: max_typos)
|
|
107
|
+
hash[em.name.to_s] = highlighted if highlighted.include?(config[:open_tag])
|
|
108
|
+
end
|
|
109
|
+
[record.id, snippets]
|
|
110
|
+
end
|
|
59
111
|
end
|
|
60
|
-
end
|
|
61
112
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
113
|
+
def exact_match_field_value(em, record)
|
|
114
|
+
if em.source
|
|
115
|
+
record.instance_exec(&em.source)
|
|
116
|
+
else
|
|
117
|
+
record.public_send(em.name)
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def manual_highlight(text, query, config, max_typos: nil)
|
|
122
|
+
return text if text.empty? || query.empty?
|
|
123
|
+
|
|
124
|
+
open_tag = config[:open_tag]
|
|
125
|
+
close_tag = config[:close_tag]
|
|
126
|
+
escaped_query = Regexp.escape(query)
|
|
127
|
+
|
|
128
|
+
if text.match?(/#{escaped_query}/i)
|
|
129
|
+
return text.gsub(/#{escaped_query}/i, "#{open_tag}\\0#{close_tag}")
|
|
130
|
+
end
|
|
66
131
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
hash[col.to_s] = highlighted if highlighted.include?(config[:open_tag])
|
|
132
|
+
best = best_fuzzy_match(text, query, max_typos: max_typos)
|
|
133
|
+
if best
|
|
134
|
+
start_pos, end_pos = best
|
|
135
|
+
return "#{text[0...start_pos]}#{open_tag}#{text[start_pos...end_pos]}#{close_tag}#{text[end_pos..]}"
|
|
72
136
|
end
|
|
73
|
-
|
|
137
|
+
|
|
138
|
+
text
|
|
74
139
|
end
|
|
75
|
-
end
|
|
76
140
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
141
|
+
def best_fuzzy_match(text, query, max_typos: nil)
|
|
142
|
+
query_len = query.length
|
|
143
|
+
text_len = text.length
|
|
144
|
+
return nil if query_len == 0 || text_len == 0
|
|
81
145
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
config = dsl.highlight_config || { open_tag: "<mark>", close_tag: "</mark>" }
|
|
85
|
-
match_expr = QueryParser.to_match_expression(QueryParser.parse(query))
|
|
86
|
-
return [] if match_expr.empty?
|
|
146
|
+
max_typos ||= max_allowed_typos(query_len)
|
|
147
|
+
return nil if max_typos < 0
|
|
87
148
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
return [] if cols.empty?
|
|
149
|
+
best_score = max_typos + 1
|
|
150
|
+
best_range = nil
|
|
91
151
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
152
|
+
min_window = [1, query_len - max_typos].max
|
|
153
|
+
max_window = [text_len, query_len + max_typos].min
|
|
154
|
+
return nil if min_window > max_window
|
|
95
155
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
156
|
+
(min_window..max_window).each do |window_len|
|
|
157
|
+
(0..(text_len - window_len)).each do |start|
|
|
158
|
+
substr = text[start, window_len]
|
|
159
|
+
distance = Distance.damerau_levenshtein(query.downcase, substr.downcase)
|
|
160
|
+
next if distance > max_typos
|
|
101
161
|
|
|
102
|
-
|
|
103
|
-
|
|
162
|
+
if best_range.nil? || distance < best_score || (distance == best_score && window_len > (best_range[1] - best_range[0]))
|
|
163
|
+
best_score = distance
|
|
164
|
+
best_range = [start, start + window_len]
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
104
168
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
end
|
|
169
|
+
best_range
|
|
170
|
+
end
|
|
108
171
|
|
|
109
|
-
|
|
110
|
-
|
|
172
|
+
def max_allowed_typos(length)
|
|
173
|
+
return -1 if length < 3
|
|
174
|
+
return 2 if length >= 9
|
|
175
|
+
1
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def highlight_rows(model, query)
|
|
179
|
+
dsl = model.full_search_dsl
|
|
180
|
+
config = dsl.highlight_config || {open_tag: "<mark>", close_tag: "</mark>"}
|
|
181
|
+
match_expr = QueryParser.to_match_expression(QueryParser.parse(query))
|
|
182
|
+
return [] if match_expr.empty?
|
|
183
|
+
|
|
184
|
+
table = qt(FullSearch::Index.fts_table_name(model))
|
|
185
|
+
cols = dsl.fields.map(&:name)
|
|
186
|
+
return [] if cols.empty?
|
|
187
|
+
|
|
188
|
+
highlight_parts = cols.each_with_index.map do |col, idx|
|
|
189
|
+
"highlight(#{table}, #{idx}, #{q(config[:open_tag])}, #{q(config[:close_tag])}) AS #{qc("#{col}_snippet")}"
|
|
190
|
+
end.join(", ")
|
|
191
|
+
|
|
192
|
+
sql = <<~SQL
|
|
193
|
+
SELECT rowid, #{highlight_parts}
|
|
194
|
+
FROM #{table}
|
|
195
|
+
WHERE #{table} MATCH #{q(match_expr)}
|
|
196
|
+
SQL
|
|
197
|
+
|
|
198
|
+
connection.execute(sql)
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
include Quoting
|
|
202
|
+
|
|
203
|
+
def connection
|
|
204
|
+
ActiveRecord::Base.connection
|
|
205
|
+
end
|
|
111
206
|
end
|
|
112
207
|
end
|
|
113
208
|
end
|
data/lib/full_search/index.rb
CHANGED
|
@@ -1,12 +1,21 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
class FilterColumnPlaceholder
|
|
4
|
+
attr_reader :name
|
|
5
|
+
def initialize(name:)
|
|
6
|
+
@name = name.to_s
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def unindexed?
|
|
10
|
+
true
|
|
11
|
+
end
|
|
12
|
+
end
|
|
4
13
|
|
|
5
14
|
module FullSearch
|
|
6
15
|
class Index
|
|
7
16
|
class << self
|
|
8
17
|
def ensure_table!(model)
|
|
9
|
-
|
|
18
|
+
sqlite!(model)
|
|
10
19
|
|
|
11
20
|
conn = connection
|
|
12
21
|
dsl = model.full_search_dsl
|
|
@@ -14,21 +23,31 @@ module FullSearch
|
|
|
14
23
|
|
|
15
24
|
create_metadata_table!
|
|
16
25
|
|
|
26
|
+
fts_was_created = false
|
|
17
27
|
unless table_exists?(model)
|
|
18
28
|
conn.execute(create_virtual_table_sql(model))
|
|
29
|
+
fts_was_created = true
|
|
19
30
|
end
|
|
20
31
|
|
|
32
|
+
trigram_was_created = false
|
|
21
33
|
if dsl.typo_tolerance? && !trigram_table_exists?(model)
|
|
22
|
-
FullSearch::Typo.warn_unsupported! unless FullSearch::Typo.supported?
|
|
34
|
+
FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
|
|
23
35
|
conn.execute(create_trigram_virtual_table_sql(model))
|
|
36
|
+
trigram_was_created = true
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
if fts_was_created || trigram_was_created
|
|
40
|
+
conn.execute(backfill_sql(model)) if fts_was_created
|
|
41
|
+
conn.execute(backfill_trigram_sql(model)) if trigram_was_created
|
|
42
|
+
reindex_source_fields!(model) if dsl.fields.any?(&:source)
|
|
24
43
|
end
|
|
25
44
|
|
|
26
|
-
ensure_triggers!(model) if model_table_exists?(model)
|
|
27
45
|
store_config_hash!(model)
|
|
46
|
+
ensure_triggers!(model) if model_table_exists?(model)
|
|
28
47
|
end
|
|
29
48
|
|
|
30
49
|
def rebuild!(model)
|
|
31
|
-
|
|
50
|
+
sqlite!(model)
|
|
32
51
|
|
|
33
52
|
dsl = model.full_search_dsl
|
|
34
53
|
return unless dsl
|
|
@@ -38,29 +57,56 @@ module FullSearch
|
|
|
38
57
|
|
|
39
58
|
with_rebuild_lock(model) do
|
|
40
59
|
drop_triggers!(model)
|
|
41
|
-
conn.execute("DROP TABLE IF EXISTS #{fts_table_name(model)};")
|
|
42
|
-
conn.execute("DROP TABLE IF EXISTS #{trigram_table_name(model)};")
|
|
60
|
+
conn.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
|
|
61
|
+
conn.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
|
|
43
62
|
conn.execute(create_virtual_table_sql(model))
|
|
44
63
|
if dsl.typo_tolerance?
|
|
45
|
-
FullSearch::Typo.warn_unsupported! unless FullSearch::Typo.supported?
|
|
64
|
+
FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
|
|
46
65
|
conn.execute(create_trigram_virtual_table_sql(model))
|
|
47
66
|
end
|
|
48
67
|
conn.execute(backfill_sql(model))
|
|
49
68
|
conn.execute(backfill_trigram_sql(model)) if dsl.typo_tolerance?
|
|
69
|
+
reindex_source_fields!(model) if dsl.fields.any?(&:source)
|
|
50
70
|
create_triggers!(model)
|
|
51
71
|
optimize!(model)
|
|
52
72
|
store_config_hash!(model, rebuilt_at: Time.current)
|
|
53
73
|
end
|
|
54
74
|
end
|
|
55
75
|
|
|
76
|
+
def rebuild_if_needed!(model)
|
|
77
|
+
sqlite!(model)
|
|
78
|
+
|
|
79
|
+
dsl = model.full_search_dsl
|
|
80
|
+
return false unless dsl
|
|
81
|
+
|
|
82
|
+
create_metadata_table!
|
|
83
|
+
ensure_table!(model)
|
|
84
|
+
|
|
85
|
+
stored = stored_config_hash(model)
|
|
86
|
+
if stored && stored == dsl.config_hash
|
|
87
|
+
false
|
|
88
|
+
else
|
|
89
|
+
rebuild!(model)
|
|
90
|
+
true
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
56
94
|
def optimize!(model)
|
|
57
|
-
|
|
95
|
+
sqlite!(model)
|
|
96
|
+
connection.execute("INSERT INTO #{qt(fts_table_name(model))}(#{qt(fts_table_name(model))}) VALUES('optimize');")
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def reindex_source_fields!(model)
|
|
100
|
+
model.find_each do |record|
|
|
101
|
+
FullSearch::Callbacks.reindex_record!(record)
|
|
102
|
+
end
|
|
58
103
|
end
|
|
59
104
|
|
|
60
105
|
def drop!(model)
|
|
106
|
+
sqlite!(model)
|
|
61
107
|
drop_triggers!(model)
|
|
62
|
-
connection.execute("DROP TABLE IF EXISTS #{fts_table_name(model)};")
|
|
63
|
-
connection.execute("DROP TABLE IF EXISTS #{trigram_table_name(model)};")
|
|
108
|
+
connection.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
|
|
109
|
+
connection.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
|
|
64
110
|
end
|
|
65
111
|
|
|
66
112
|
def fts_table_name(model)
|
|
@@ -71,10 +117,21 @@ module FullSearch
|
|
|
71
117
|
"#{fts_table_name(model)}_trigram"
|
|
72
118
|
end
|
|
73
119
|
|
|
74
|
-
def sqlite?
|
|
120
|
+
def sqlite?(model = nil)
|
|
121
|
+
if model
|
|
122
|
+
model.connection.adapter_name.downcase.include?("sqlite")
|
|
123
|
+
else
|
|
124
|
+
connection.adapter_name.downcase.include?("sqlite")
|
|
125
|
+
end
|
|
126
|
+
rescue
|
|
75
127
|
connection.adapter_name.downcase.include?("sqlite")
|
|
76
128
|
end
|
|
77
129
|
|
|
130
|
+
def sqlite!(model)
|
|
131
|
+
adapter = model ? model.connection.adapter_name : connection.adapter_name
|
|
132
|
+
raise UnsupportedDatabaseError, "full_search requires SQLite, but #{adapter} is configured" unless sqlite?(model)
|
|
133
|
+
end
|
|
134
|
+
|
|
78
135
|
def stored_config_hash(model)
|
|
79
136
|
row = connection.execute(
|
|
80
137
|
"SELECT config_hash FROM full_search_index_versions WHERE table_name=#{q(model.table_name)}"
|
|
@@ -92,6 +149,14 @@ module FullSearch
|
|
|
92
149
|
connection.quote(value)
|
|
93
150
|
end
|
|
94
151
|
|
|
152
|
+
def qt(name)
|
|
153
|
+
connection.quote_table_name(name)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def qc(name)
|
|
157
|
+
connection.quote_column_name(name)
|
|
158
|
+
end
|
|
159
|
+
|
|
95
160
|
def model_table_exists?(model)
|
|
96
161
|
connection.execute(
|
|
97
162
|
"SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(model.table_name)} LIMIT 1"
|
|
@@ -122,11 +187,11 @@ module FullSearch
|
|
|
122
187
|
|
|
123
188
|
def create_virtual_table_sql(model)
|
|
124
189
|
dsl = model.full_search_dsl
|
|
125
|
-
columns = (dsl.fields + dsl.filters.map { |f|
|
|
126
|
-
column_list = columns.map { |c| c.respond_to?(:unindexed?) && c.unindexed? ? "#{c.name} UNINDEXED" : c.name }.join(", ")
|
|
190
|
+
columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) })
|
|
191
|
+
column_list = columns.map { |c| (c.respond_to?(:unindexed?) && c.unindexed?) ? "#{qc(c.name)} UNINDEXED" : qc(c.name) }.join(", ")
|
|
127
192
|
|
|
128
193
|
<<~SQL
|
|
129
|
-
CREATE VIRTUAL TABLE #{fts_table_name(model)} USING fts5(
|
|
194
|
+
CREATE VIRTUAL TABLE #{qt(fts_table_name(model))} USING fts5(
|
|
130
195
|
#{column_list},
|
|
131
196
|
tokenize='#{dsl.tokenize}'
|
|
132
197
|
);
|
|
@@ -140,13 +205,13 @@ module FullSearch
|
|
|
140
205
|
if c.respond_to?(:source) && c.source
|
|
141
206
|
source_value_sql(c.source)
|
|
142
207
|
else
|
|
143
|
-
"#{model.table_name}.#{c.name}"
|
|
208
|
+
"#{qt(model.table_name)}.#{qc(c.name)}"
|
|
144
209
|
end
|
|
145
210
|
end.join(", ")
|
|
146
211
|
|
|
147
212
|
<<~SQL
|
|
148
|
-
INSERT INTO #{fts_table_name(model)}(rowid, #{cols.map(
|
|
149
|
-
SELECT #{model.table_name}.id, #{select} FROM #{model.table_name};
|
|
213
|
+
INSERT INTO #{qt(fts_table_name(model))}(rowid, #{cols.map { |c| qc(c.name) }.join(", ")})
|
|
214
|
+
SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)};
|
|
150
215
|
SQL
|
|
151
216
|
end
|
|
152
217
|
|
|
@@ -159,13 +224,20 @@ module FullSearch
|
|
|
159
224
|
"SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=#{q(model.table_name)}"
|
|
160
225
|
).map { |r| r["name"] }
|
|
161
226
|
|
|
162
|
-
|
|
227
|
+
expected = trigger_names(model)
|
|
228
|
+
expected += trigram_trigger_names(model) if model.full_search_dsl.typo_tolerance?
|
|
229
|
+
return if (expected - existing).empty? && (existing - expected).empty?
|
|
230
|
+
|
|
231
|
+
rebuild!(model)
|
|
163
232
|
end
|
|
164
233
|
|
|
165
234
|
def create_triggers!(model)
|
|
166
235
|
connection.execute(insert_trigger_sql(model))
|
|
167
236
|
connection.execute(delete_trigger_sql(model))
|
|
168
237
|
connection.execute(update_trigger_sql(model))
|
|
238
|
+
if model.full_search_dsl.soft_delete_column
|
|
239
|
+
connection.execute(soft_delete_removal_trigger_sql(model))
|
|
240
|
+
end
|
|
169
241
|
if model.full_search_dsl.typo_tolerance?
|
|
170
242
|
connection.execute(insert_trigram_trigger_sql(model))
|
|
171
243
|
connection.execute(delete_trigram_trigger_sql(model))
|
|
@@ -175,13 +247,15 @@ module FullSearch
|
|
|
175
247
|
|
|
176
248
|
def drop_triggers!(model)
|
|
177
249
|
(trigger_names(model) + trigram_trigger_names(model)).each do |name|
|
|
178
|
-
connection.execute("DROP TRIGGER IF EXISTS #{name};")
|
|
250
|
+
connection.execute("DROP TRIGGER IF EXISTS #{qt(name)};")
|
|
179
251
|
end
|
|
180
252
|
end
|
|
181
253
|
|
|
182
254
|
def trigger_names(model)
|
|
183
255
|
base = fts_table_name(model)
|
|
184
|
-
%W[#{base}_ai #{base}_ad #{base}_au]
|
|
256
|
+
names = %W[#{base}_ai #{base}_ad #{base}_au]
|
|
257
|
+
names << "#{base}_au_soft_delete" if model.full_search_dsl.soft_delete_column
|
|
258
|
+
names
|
|
185
259
|
end
|
|
186
260
|
|
|
187
261
|
def trigram_trigger_names(model)
|
|
@@ -195,8 +269,8 @@ module FullSearch
|
|
|
195
269
|
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
196
270
|
|
|
197
271
|
<<~SQL
|
|
198
|
-
CREATE TRIGGER #{trigger_names(model).first} AFTER INSERT ON #{model.table_name} BEGIN
|
|
199
|
-
INSERT INTO #{fts_table_name(model)}(rowid, #{col_names(cols)})
|
|
272
|
+
CREATE TRIGGER #{qt(trigger_names(model).first)} AFTER INSERT ON #{qt(model.table_name)} BEGIN
|
|
273
|
+
INSERT INTO #{qt(fts_table_name(model))}(rowid, #{col_names(cols)})
|
|
200
274
|
VALUES (new.id, #{values});
|
|
201
275
|
END;
|
|
202
276
|
SQL
|
|
@@ -204,8 +278,8 @@ module FullSearch
|
|
|
204
278
|
|
|
205
279
|
def delete_trigger_sql(model)
|
|
206
280
|
<<~SQL
|
|
207
|
-
CREATE TRIGGER #{trigger_names(model)[1]} AFTER DELETE ON #{model.table_name} BEGIN
|
|
208
|
-
DELETE FROM #{fts_table_name(model)} WHERE rowid = old.id;
|
|
281
|
+
CREATE TRIGGER #{qt(trigger_names(model)[1])} AFTER DELETE ON #{qt(model.table_name)} BEGIN
|
|
282
|
+
DELETE FROM #{qt(fts_table_name(model))} WHERE rowid = old.id;
|
|
209
283
|
END;
|
|
210
284
|
SQL
|
|
211
285
|
end
|
|
@@ -215,12 +289,12 @@ module FullSearch
|
|
|
215
289
|
cols = dsl.fields + dsl.filters
|
|
216
290
|
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
217
291
|
cols_str = col_names(cols)
|
|
218
|
-
fts_table = fts_table_name(model)
|
|
292
|
+
fts_table = qt(fts_table_name(model))
|
|
219
293
|
|
|
220
|
-
when_clause = SoftDelete.
|
|
294
|
+
when_clause = SoftDelete.active_update_clause(model)
|
|
221
295
|
|
|
222
296
|
<<~SQL
|
|
223
|
-
CREATE TRIGGER #{trigger_names(model)[2]} AFTER UPDATE ON #{model.table_name} #{when_clause}
|
|
297
|
+
CREATE TRIGGER #{qt(trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
|
|
224
298
|
BEGIN
|
|
225
299
|
DELETE FROM #{fts_table} WHERE rowid = old.id;
|
|
226
300
|
INSERT INTO #{fts_table}(rowid, #{cols_str})
|
|
@@ -229,6 +303,19 @@ module FullSearch
|
|
|
229
303
|
SQL
|
|
230
304
|
end
|
|
231
305
|
|
|
306
|
+
def soft_delete_removal_trigger_sql(model)
|
|
307
|
+
model.full_search_dsl
|
|
308
|
+
fts_table = qt(fts_table_name(model))
|
|
309
|
+
when_clause = SoftDelete.soft_delete_remove_clause(model)
|
|
310
|
+
|
|
311
|
+
<<~SQL
|
|
312
|
+
CREATE TRIGGER #{qt("#{trigger_names(model)[2]}_soft_delete")} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
|
|
313
|
+
BEGIN
|
|
314
|
+
DELETE FROM #{fts_table} WHERE rowid = old.id;
|
|
315
|
+
END;
|
|
316
|
+
SQL
|
|
317
|
+
end
|
|
318
|
+
|
|
232
319
|
def trigram_table_exists?(model)
|
|
233
320
|
connection.execute(
|
|
234
321
|
"SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(trigram_table_name(model))} LIMIT 1"
|
|
@@ -237,11 +324,11 @@ module FullSearch
|
|
|
237
324
|
|
|
238
325
|
def create_trigram_virtual_table_sql(model)
|
|
239
326
|
dsl = model.full_search_dsl
|
|
240
|
-
columns = (dsl.fields + dsl.filters.map { |f|
|
|
241
|
-
column_list = columns.map { |c| c.respond_to?(:unindexed?) && c.unindexed? ? "#{c.name} UNINDEXED" : c.name }.join(", ")
|
|
327
|
+
columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) })
|
|
328
|
+
column_list = columns.map { |c| (c.respond_to?(:unindexed?) && c.unindexed?) ? "#{qc(c.name)} UNINDEXED" : qc(c.name) }.join(", ")
|
|
242
329
|
|
|
243
330
|
<<~SQL
|
|
244
|
-
CREATE VIRTUAL TABLE #{trigram_table_name(model)} USING fts5(
|
|
331
|
+
CREATE VIRTUAL TABLE #{qt(trigram_table_name(model))} USING fts5(
|
|
245
332
|
#{column_list},
|
|
246
333
|
tokenize='trigram'
|
|
247
334
|
);
|
|
@@ -255,13 +342,13 @@ module FullSearch
|
|
|
255
342
|
if c.respond_to?(:source) && c.source
|
|
256
343
|
source_value_sql(c.source)
|
|
257
344
|
else
|
|
258
|
-
"#{model.table_name}.#{c.name}"
|
|
345
|
+
"#{qt(model.table_name)}.#{qc(c.name)}"
|
|
259
346
|
end
|
|
260
347
|
end.join(", ")
|
|
261
348
|
|
|
262
349
|
<<~SQL
|
|
263
|
-
INSERT INTO #{trigram_table_name(model)}(rowid, #{cols.map(
|
|
264
|
-
SELECT #{model.table_name}.id, #{select} FROM #{model.table_name};
|
|
350
|
+
INSERT INTO #{qt(trigram_table_name(model))}(rowid, #{cols.map { |c| qc(c.name) }.join(", ")})
|
|
351
|
+
SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)};
|
|
265
352
|
SQL
|
|
266
353
|
end
|
|
267
354
|
|
|
@@ -271,8 +358,8 @@ module FullSearch
|
|
|
271
358
|
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
272
359
|
|
|
273
360
|
<<~SQL
|
|
274
|
-
CREATE TRIGGER #{trigram_trigger_names(model).first} AFTER INSERT ON #{model.table_name} BEGIN
|
|
275
|
-
INSERT INTO #{trigram_table_name(model)}(rowid, #{col_names(cols)})
|
|
361
|
+
CREATE TRIGGER #{qt(trigram_trigger_names(model).first)} AFTER INSERT ON #{qt(model.table_name)} BEGIN
|
|
362
|
+
INSERT INTO #{qt(trigram_table_name(model))}(rowid, #{col_names(cols)})
|
|
276
363
|
VALUES (new.id, #{values});
|
|
277
364
|
END;
|
|
278
365
|
SQL
|
|
@@ -280,8 +367,8 @@ module FullSearch
|
|
|
280
367
|
|
|
281
368
|
def delete_trigram_trigger_sql(model)
|
|
282
369
|
<<~SQL
|
|
283
|
-
CREATE TRIGGER #{trigram_trigger_names(model)[1]} AFTER DELETE ON #{model.table_name} BEGIN
|
|
284
|
-
DELETE FROM #{trigram_table_name(model)} WHERE rowid = old.id;
|
|
370
|
+
CREATE TRIGGER #{qt(trigram_trigger_names(model)[1])} AFTER DELETE ON #{qt(model.table_name)} BEGIN
|
|
371
|
+
DELETE FROM #{qt(trigram_table_name(model))} WHERE rowid = old.id;
|
|
285
372
|
END;
|
|
286
373
|
SQL
|
|
287
374
|
end
|
|
@@ -291,12 +378,12 @@ module FullSearch
|
|
|
291
378
|
cols = dsl.fields + dsl.filters
|
|
292
379
|
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
293
380
|
cols_str = col_names(cols)
|
|
294
|
-
trigram_table = trigram_table_name(model)
|
|
381
|
+
trigram_table = qt(trigram_table_name(model))
|
|
295
382
|
|
|
296
|
-
when_clause = SoftDelete.
|
|
383
|
+
when_clause = SoftDelete.active_update_clause(model)
|
|
297
384
|
|
|
298
385
|
<<~SQL
|
|
299
|
-
CREATE TRIGGER #{trigram_trigger_names(model)[2]} AFTER UPDATE ON #{model.table_name} #{when_clause}
|
|
386
|
+
CREATE TRIGGER #{qt(trigram_trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
|
|
300
387
|
BEGIN
|
|
301
388
|
DELETE FROM #{trigram_table} WHERE rowid = old.id;
|
|
302
389
|
INSERT INTO #{trigram_table}(rowid, #{cols_str})
|
|
@@ -306,22 +393,25 @@ module FullSearch
|
|
|
306
393
|
end
|
|
307
394
|
|
|
308
395
|
def col_names(cols)
|
|
309
|
-
cols.map(
|
|
396
|
+
cols.map { |c| qc(c.name) }.join(", ")
|
|
310
397
|
end
|
|
311
398
|
|
|
312
399
|
def column_ref(col, prefix:)
|
|
313
400
|
if col.respond_to?(:source) && col.source
|
|
314
401
|
"''"
|
|
315
402
|
else
|
|
316
|
-
"#{prefix}.#{col.name}"
|
|
403
|
+
"#{prefix}.#{qc(col.name)}"
|
|
317
404
|
end
|
|
318
405
|
end
|
|
319
406
|
|
|
407
|
+
# NOTE: This lock prevents concurrent rebuilds within the same process/connection only.
|
|
408
|
+
# For multi-process or multi-host deployments, run full_search:rebuild from a single
|
|
409
|
+
# deployment step. See README.
|
|
320
410
|
def with_rebuild_lock(model)
|
|
321
411
|
if FullSearch.config.lock_rebuilds
|
|
322
412
|
connection.transaction do
|
|
323
413
|
connection.execute(
|
|
324
|
-
"INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q("")}, datetime('now'))
|
|
414
|
+
"INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q("__rebuilding__")}, datetime('now'))
|
|
325
415
|
ON CONFLICT(table_name) DO UPDATE SET config_hash=excluded.config_hash;"
|
|
326
416
|
)
|
|
327
417
|
yield
|