full_search 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +22 -4
- data/lib/full_search/callbacks.rb +26 -8
- data/lib/full_search/config.rb +0 -1
- data/lib/full_search/distance.rb +33 -0
- data/lib/full_search/dsl.rb +26 -12
- data/lib/full_search/errors.rb +2 -0
- data/lib/full_search/exact_match.rb +7 -13
- data/lib/full_search/highlighter.rb +166 -102
- data/lib/full_search/index.rb +107 -61
- data/lib/full_search/model.rb +2 -1
- data/lib/full_search/multi_search.rb +4 -1
- data/lib/full_search/query_parser.rb +101 -88
- data/lib/full_search/quoting.rb +11 -0
- data/lib/full_search/schema_dumper_patch.rb +45 -0
- data/lib/full_search/search.rb +97 -77
- data/lib/full_search/soft_delete.rb +15 -2
- data/lib/full_search/typo.rb +10 -8
- data/lib/full_search/version.rb +1 -1
- data/lib/full_search.rb +12 -1
- data/lib/tasks/full_search.rake +12 -4
- metadata +46 -17
data/lib/full_search/index.rb
CHANGED
|
@@ -1,12 +1,21 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
class FilterColumnPlaceholder
|
|
4
|
+
attr_reader :name
|
|
5
|
+
def initialize(name:)
|
|
6
|
+
@name = name.to_s
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def unindexed?
|
|
10
|
+
true
|
|
11
|
+
end
|
|
12
|
+
end
|
|
4
13
|
|
|
5
14
|
module FullSearch
|
|
6
15
|
class Index
|
|
7
16
|
class << self
|
|
8
17
|
def ensure_table!(model)
|
|
9
|
-
|
|
18
|
+
sqlite!(model)
|
|
10
19
|
|
|
11
20
|
conn = connection
|
|
12
21
|
dsl = model.full_search_dsl
|
|
@@ -14,30 +23,31 @@ module FullSearch
|
|
|
14
23
|
|
|
15
24
|
create_metadata_table!
|
|
16
25
|
|
|
17
|
-
|
|
26
|
+
fts_was_created = false
|
|
18
27
|
unless table_exists?(model)
|
|
19
28
|
conn.execute(create_virtual_table_sql(model))
|
|
20
|
-
|
|
29
|
+
fts_was_created = true
|
|
21
30
|
end
|
|
22
31
|
|
|
32
|
+
trigram_was_created = false
|
|
23
33
|
if dsl.typo_tolerance? && !trigram_table_exists?(model)
|
|
24
|
-
FullSearch::Typo.warn_unsupported! unless FullSearch::Typo.supported?
|
|
34
|
+
FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
|
|
25
35
|
conn.execute(create_trigram_virtual_table_sql(model))
|
|
26
|
-
|
|
36
|
+
trigram_was_created = true
|
|
27
37
|
end
|
|
28
38
|
|
|
29
|
-
if
|
|
30
|
-
conn.execute(backfill_sql(model))
|
|
31
|
-
conn.execute(backfill_trigram_sql(model)) if
|
|
39
|
+
if fts_was_created || trigram_was_created
|
|
40
|
+
conn.execute(backfill_sql(model)) if fts_was_created
|
|
41
|
+
conn.execute(backfill_trigram_sql(model)) if trigram_was_created
|
|
32
42
|
reindex_source_fields!(model) if dsl.fields.any?(&:source)
|
|
33
|
-
store_config_hash!(model)
|
|
34
43
|
end
|
|
35
44
|
|
|
45
|
+
store_config_hash!(model)
|
|
36
46
|
ensure_triggers!(model) if model_table_exists?(model)
|
|
37
47
|
end
|
|
38
48
|
|
|
39
49
|
def rebuild!(model)
|
|
40
|
-
|
|
50
|
+
sqlite!(model)
|
|
41
51
|
|
|
42
52
|
dsl = model.full_search_dsl
|
|
43
53
|
return unless dsl
|
|
@@ -47,11 +57,11 @@ module FullSearch
|
|
|
47
57
|
|
|
48
58
|
with_rebuild_lock(model) do
|
|
49
59
|
drop_triggers!(model)
|
|
50
|
-
conn.execute("DROP TABLE IF EXISTS #{fts_table_name(model)};")
|
|
51
|
-
conn.execute("DROP TABLE IF EXISTS #{trigram_table_name(model)};")
|
|
60
|
+
conn.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
|
|
61
|
+
conn.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
|
|
52
62
|
conn.execute(create_virtual_table_sql(model))
|
|
53
63
|
if dsl.typo_tolerance?
|
|
54
|
-
FullSearch::Typo.warn_unsupported! unless FullSearch::Typo.supported?
|
|
64
|
+
FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
|
|
55
65
|
conn.execute(create_trigram_virtual_table_sql(model))
|
|
56
66
|
end
|
|
57
67
|
conn.execute(backfill_sql(model))
|
|
@@ -64,7 +74,7 @@ module FullSearch
|
|
|
64
74
|
end
|
|
65
75
|
|
|
66
76
|
def rebuild_if_needed!(model)
|
|
67
|
-
|
|
77
|
+
sqlite!(model)
|
|
68
78
|
|
|
69
79
|
dsl = model.full_search_dsl
|
|
70
80
|
return false unless dsl
|
|
@@ -82,7 +92,8 @@ module FullSearch
|
|
|
82
92
|
end
|
|
83
93
|
|
|
84
94
|
def optimize!(model)
|
|
85
|
-
|
|
95
|
+
sqlite!(model)
|
|
96
|
+
connection.execute("INSERT INTO #{qt(fts_table_name(model))}(#{qt(fts_table_name(model))}) VALUES('optimize');")
|
|
86
97
|
end
|
|
87
98
|
|
|
88
99
|
def reindex_source_fields!(model)
|
|
@@ -92,9 +103,10 @@ module FullSearch
|
|
|
92
103
|
end
|
|
93
104
|
|
|
94
105
|
def drop!(model)
|
|
106
|
+
sqlite!(model)
|
|
95
107
|
drop_triggers!(model)
|
|
96
|
-
connection.execute("DROP TABLE IF EXISTS #{fts_table_name(model)};")
|
|
97
|
-
connection.execute("DROP TABLE IF EXISTS #{trigram_table_name(model)};")
|
|
108
|
+
connection.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
|
|
109
|
+
connection.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
|
|
98
110
|
end
|
|
99
111
|
|
|
100
112
|
def fts_table_name(model)
|
|
@@ -105,10 +117,21 @@ module FullSearch
|
|
|
105
117
|
"#{fts_table_name(model)}_trigram"
|
|
106
118
|
end
|
|
107
119
|
|
|
108
|
-
def sqlite?
|
|
120
|
+
def sqlite?(model = nil)
|
|
121
|
+
if model
|
|
122
|
+
model.connection.adapter_name.downcase.include?("sqlite")
|
|
123
|
+
else
|
|
124
|
+
connection.adapter_name.downcase.include?("sqlite")
|
|
125
|
+
end
|
|
126
|
+
rescue
|
|
109
127
|
connection.adapter_name.downcase.include?("sqlite")
|
|
110
128
|
end
|
|
111
129
|
|
|
130
|
+
def sqlite!(model)
|
|
131
|
+
adapter = model ? model.connection.adapter_name : connection.adapter_name
|
|
132
|
+
raise UnsupportedDatabaseError, "full_search requires SQLite, but #{adapter} is configured" unless sqlite?(model)
|
|
133
|
+
end
|
|
134
|
+
|
|
112
135
|
def stored_config_hash(model)
|
|
113
136
|
row = connection.execute(
|
|
114
137
|
"SELECT config_hash FROM full_search_index_versions WHERE table_name=#{q(model.table_name)}"
|
|
@@ -126,6 +149,14 @@ module FullSearch
|
|
|
126
149
|
connection.quote(value)
|
|
127
150
|
end
|
|
128
151
|
|
|
152
|
+
def qt(name)
|
|
153
|
+
connection.quote_table_name(name)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def qc(name)
|
|
157
|
+
connection.quote_column_name(name)
|
|
158
|
+
end
|
|
159
|
+
|
|
129
160
|
def model_table_exists?(model)
|
|
130
161
|
connection.execute(
|
|
131
162
|
"SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(model.table_name)} LIMIT 1"
|
|
@@ -156,15 +187,10 @@ module FullSearch
|
|
|
156
187
|
|
|
157
188
|
def create_virtual_table_sql(model)
|
|
158
189
|
dsl = model.full_search_dsl
|
|
159
|
-
columns = (dsl.fields + dsl.filters.map { |f|
|
|
160
|
-
column_list = columns.map { |c| c.respond_to?(:unindexed?) && c.unindexed? ? "#{c.name} UNINDEXED" : c.name }.join(", ")
|
|
190
|
+
columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) })
|
|
191
|
+
column_list = columns.map { |c| (c.respond_to?(:unindexed?) && c.unindexed?) ? "#{qc(c.name)} UNINDEXED" : qc(c.name) }.join(", ")
|
|
161
192
|
|
|
162
|
-
|
|
163
|
-
CREATE VIRTUAL TABLE #{fts_table_name(model)} USING fts5(
|
|
164
|
-
#{column_list},
|
|
165
|
-
tokenize='#{dsl.tokenize}'
|
|
166
|
-
);
|
|
167
|
-
SQL
|
|
193
|
+
"CREATE VIRTUAL TABLE #{qt(fts_table_name(model))} USING fts5(#{column_list}, tokenize='#{dsl.tokenize}');"
|
|
168
194
|
end
|
|
169
195
|
|
|
170
196
|
def backfill_sql(model)
|
|
@@ -174,13 +200,13 @@ module FullSearch
|
|
|
174
200
|
if c.respond_to?(:source) && c.source
|
|
175
201
|
source_value_sql(c.source)
|
|
176
202
|
else
|
|
177
|
-
"#{model.table_name}.#{c.name}"
|
|
203
|
+
"#{qt(model.table_name)}.#{qc(c.name)}"
|
|
178
204
|
end
|
|
179
205
|
end.join(", ")
|
|
180
206
|
|
|
181
207
|
<<~SQL
|
|
182
|
-
INSERT INTO #{fts_table_name(model)}(rowid, #{cols.map(
|
|
183
|
-
SELECT #{model.table_name}.id, #{select} FROM #{model.table_name};
|
|
208
|
+
INSERT INTO #{qt(fts_table_name(model))}(rowid, #{cols.map { |c| qc(c.name) }.join(", ")})
|
|
209
|
+
SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)};
|
|
184
210
|
SQL
|
|
185
211
|
end
|
|
186
212
|
|
|
@@ -193,13 +219,20 @@ module FullSearch
|
|
|
193
219
|
"SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name=#{q(model.table_name)}"
|
|
194
220
|
).map { |r| r["name"] }
|
|
195
221
|
|
|
196
|
-
|
|
222
|
+
expected = trigger_names(model)
|
|
223
|
+
expected += trigram_trigger_names(model) if model.full_search_dsl.typo_tolerance?
|
|
224
|
+
return if (expected - existing).empty? && (existing - expected).empty?
|
|
225
|
+
|
|
226
|
+
rebuild!(model)
|
|
197
227
|
end
|
|
198
228
|
|
|
199
229
|
def create_triggers!(model)
|
|
200
230
|
connection.execute(insert_trigger_sql(model))
|
|
201
231
|
connection.execute(delete_trigger_sql(model))
|
|
202
232
|
connection.execute(update_trigger_sql(model))
|
|
233
|
+
if model.full_search_dsl.soft_delete_column
|
|
234
|
+
connection.execute(soft_delete_removal_trigger_sql(model))
|
|
235
|
+
end
|
|
203
236
|
if model.full_search_dsl.typo_tolerance?
|
|
204
237
|
connection.execute(insert_trigram_trigger_sql(model))
|
|
205
238
|
connection.execute(delete_trigram_trigger_sql(model))
|
|
@@ -209,13 +242,15 @@ module FullSearch
|
|
|
209
242
|
|
|
210
243
|
def drop_triggers!(model)
|
|
211
244
|
(trigger_names(model) + trigram_trigger_names(model)).each do |name|
|
|
212
|
-
connection.execute("DROP TRIGGER IF EXISTS #{name};")
|
|
245
|
+
connection.execute("DROP TRIGGER IF EXISTS #{qt(name)};")
|
|
213
246
|
end
|
|
214
247
|
end
|
|
215
248
|
|
|
216
249
|
def trigger_names(model)
|
|
217
250
|
base = fts_table_name(model)
|
|
218
|
-
%W[#{base}_ai #{base}_ad #{base}_au]
|
|
251
|
+
names = %W[#{base}_ai #{base}_ad #{base}_au]
|
|
252
|
+
names << "#{base}_au_soft_delete" if model.full_search_dsl.soft_delete_column
|
|
253
|
+
names
|
|
219
254
|
end
|
|
220
255
|
|
|
221
256
|
def trigram_trigger_names(model)
|
|
@@ -229,8 +264,8 @@ module FullSearch
|
|
|
229
264
|
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
230
265
|
|
|
231
266
|
<<~SQL
|
|
232
|
-
CREATE TRIGGER #{trigger_names(model).first} AFTER INSERT ON #{model.table_name} BEGIN
|
|
233
|
-
INSERT INTO #{fts_table_name(model)}(rowid, #{col_names(cols)})
|
|
267
|
+
CREATE TRIGGER #{qt(trigger_names(model).first)} AFTER INSERT ON #{qt(model.table_name)} BEGIN
|
|
268
|
+
INSERT INTO #{qt(fts_table_name(model))}(rowid, #{col_names(cols)})
|
|
234
269
|
VALUES (new.id, #{values});
|
|
235
270
|
END;
|
|
236
271
|
SQL
|
|
@@ -238,8 +273,8 @@ module FullSearch
|
|
|
238
273
|
|
|
239
274
|
def delete_trigger_sql(model)
|
|
240
275
|
<<~SQL
|
|
241
|
-
CREATE TRIGGER #{trigger_names(model)[1]} AFTER DELETE ON #{model.table_name} BEGIN
|
|
242
|
-
DELETE FROM #{fts_table_name(model)} WHERE rowid = old.id;
|
|
276
|
+
CREATE TRIGGER #{qt(trigger_names(model)[1])} AFTER DELETE ON #{qt(model.table_name)} BEGIN
|
|
277
|
+
DELETE FROM #{qt(fts_table_name(model))} WHERE rowid = old.id;
|
|
243
278
|
END;
|
|
244
279
|
SQL
|
|
245
280
|
end
|
|
@@ -249,12 +284,12 @@ module FullSearch
|
|
|
249
284
|
cols = dsl.fields + dsl.filters
|
|
250
285
|
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
251
286
|
cols_str = col_names(cols)
|
|
252
|
-
fts_table = fts_table_name(model)
|
|
287
|
+
fts_table = qt(fts_table_name(model))
|
|
253
288
|
|
|
254
|
-
when_clause = SoftDelete.
|
|
289
|
+
when_clause = SoftDelete.active_update_clause(model)
|
|
255
290
|
|
|
256
291
|
<<~SQL
|
|
257
|
-
CREATE TRIGGER #{trigger_names(model)[2]} AFTER UPDATE ON #{model.table_name} #{when_clause}
|
|
292
|
+
CREATE TRIGGER #{qt(trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
|
|
258
293
|
BEGIN
|
|
259
294
|
DELETE FROM #{fts_table} WHERE rowid = old.id;
|
|
260
295
|
INSERT INTO #{fts_table}(rowid, #{cols_str})
|
|
@@ -263,6 +298,19 @@ module FullSearch
|
|
|
263
298
|
SQL
|
|
264
299
|
end
|
|
265
300
|
|
|
301
|
+
def soft_delete_removal_trigger_sql(model)
|
|
302
|
+
model.full_search_dsl
|
|
303
|
+
fts_table = qt(fts_table_name(model))
|
|
304
|
+
when_clause = SoftDelete.soft_delete_remove_clause(model)
|
|
305
|
+
|
|
306
|
+
<<~SQL
|
|
307
|
+
CREATE TRIGGER #{qt("#{trigger_names(model)[2]}_soft_delete")} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
|
|
308
|
+
BEGIN
|
|
309
|
+
DELETE FROM #{fts_table} WHERE rowid = old.id;
|
|
310
|
+
END;
|
|
311
|
+
SQL
|
|
312
|
+
end
|
|
313
|
+
|
|
266
314
|
def trigram_table_exists?(model)
|
|
267
315
|
connection.execute(
|
|
268
316
|
"SELECT name FROM sqlite_master WHERE type='table' AND name=#{q(trigram_table_name(model))} LIMIT 1"
|
|
@@ -271,15 +319,10 @@ module FullSearch
|
|
|
271
319
|
|
|
272
320
|
def create_trigram_virtual_table_sql(model)
|
|
273
321
|
dsl = model.full_search_dsl
|
|
274
|
-
columns = (dsl.fields + dsl.filters.map { |f|
|
|
275
|
-
column_list = columns.map { |c| c.respond_to?(:unindexed?) && c.unindexed? ? "#{c.name} UNINDEXED" : c.name }.join(", ")
|
|
322
|
+
columns = (dsl.fields + dsl.filters.map { |f| FilterColumnPlaceholder.new(name: f.name) })
|
|
323
|
+
column_list = columns.map { |c| (c.respond_to?(:unindexed?) && c.unindexed?) ? "#{qc(c.name)} UNINDEXED" : qc(c.name) }.join(", ")
|
|
276
324
|
|
|
277
|
-
|
|
278
|
-
CREATE VIRTUAL TABLE #{trigram_table_name(model)} USING fts5(
|
|
279
|
-
#{column_list},
|
|
280
|
-
tokenize='trigram'
|
|
281
|
-
);
|
|
282
|
-
SQL
|
|
325
|
+
"CREATE VIRTUAL TABLE #{qt(trigram_table_name(model))} USING fts5(#{column_list}, tokenize='trigram');"
|
|
283
326
|
end
|
|
284
327
|
|
|
285
328
|
def backfill_trigram_sql(model)
|
|
@@ -289,13 +332,13 @@ module FullSearch
|
|
|
289
332
|
if c.respond_to?(:source) && c.source
|
|
290
333
|
source_value_sql(c.source)
|
|
291
334
|
else
|
|
292
|
-
"#{model.table_name}.#{c.name}"
|
|
335
|
+
"#{qt(model.table_name)}.#{qc(c.name)}"
|
|
293
336
|
end
|
|
294
337
|
end.join(", ")
|
|
295
338
|
|
|
296
339
|
<<~SQL
|
|
297
|
-
INSERT INTO #{trigram_table_name(model)}(rowid, #{cols.map(
|
|
298
|
-
SELECT #{model.table_name}.id, #{select} FROM #{model.table_name};
|
|
340
|
+
INSERT INTO #{qt(trigram_table_name(model))}(rowid, #{cols.map { |c| qc(c.name) }.join(", ")})
|
|
341
|
+
SELECT #{qt(model.table_name)}.id, #{select} FROM #{qt(model.table_name)};
|
|
299
342
|
SQL
|
|
300
343
|
end
|
|
301
344
|
|
|
@@ -305,8 +348,8 @@ module FullSearch
|
|
|
305
348
|
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
306
349
|
|
|
307
350
|
<<~SQL
|
|
308
|
-
CREATE TRIGGER #{trigram_trigger_names(model).first} AFTER INSERT ON #{model.table_name} BEGIN
|
|
309
|
-
INSERT INTO #{trigram_table_name(model)}(rowid, #{col_names(cols)})
|
|
351
|
+
CREATE TRIGGER #{qt(trigram_trigger_names(model).first)} AFTER INSERT ON #{qt(model.table_name)} BEGIN
|
|
352
|
+
INSERT INTO #{qt(trigram_table_name(model))}(rowid, #{col_names(cols)})
|
|
310
353
|
VALUES (new.id, #{values});
|
|
311
354
|
END;
|
|
312
355
|
SQL
|
|
@@ -314,8 +357,8 @@ module FullSearch
|
|
|
314
357
|
|
|
315
358
|
def delete_trigram_trigger_sql(model)
|
|
316
359
|
<<~SQL
|
|
317
|
-
CREATE TRIGGER #{trigram_trigger_names(model)[1]} AFTER DELETE ON #{model.table_name} BEGIN
|
|
318
|
-
DELETE FROM #{trigram_table_name(model)} WHERE rowid = old.id;
|
|
360
|
+
CREATE TRIGGER #{qt(trigram_trigger_names(model)[1])} AFTER DELETE ON #{qt(model.table_name)} BEGIN
|
|
361
|
+
DELETE FROM #{qt(trigram_table_name(model))} WHERE rowid = old.id;
|
|
319
362
|
END;
|
|
320
363
|
SQL
|
|
321
364
|
end
|
|
@@ -325,12 +368,12 @@ module FullSearch
|
|
|
325
368
|
cols = dsl.fields + dsl.filters
|
|
326
369
|
values = cols.map { |c| column_ref(c, prefix: "new") }.join(", ")
|
|
327
370
|
cols_str = col_names(cols)
|
|
328
|
-
trigram_table = trigram_table_name(model)
|
|
371
|
+
trigram_table = qt(trigram_table_name(model))
|
|
329
372
|
|
|
330
|
-
when_clause = SoftDelete.
|
|
373
|
+
when_clause = SoftDelete.active_update_clause(model)
|
|
331
374
|
|
|
332
375
|
<<~SQL
|
|
333
|
-
CREATE TRIGGER #{trigram_trigger_names(model)[2]} AFTER UPDATE ON #{model.table_name} #{when_clause}
|
|
376
|
+
CREATE TRIGGER #{qt(trigram_trigger_names(model)[2])} AFTER UPDATE ON #{qt(model.table_name)} #{when_clause}
|
|
334
377
|
BEGIN
|
|
335
378
|
DELETE FROM #{trigram_table} WHERE rowid = old.id;
|
|
336
379
|
INSERT INTO #{trigram_table}(rowid, #{cols_str})
|
|
@@ -340,22 +383,25 @@ module FullSearch
|
|
|
340
383
|
end
|
|
341
384
|
|
|
342
385
|
def col_names(cols)
|
|
343
|
-
cols.map(
|
|
386
|
+
cols.map { |c| qc(c.name) }.join(", ")
|
|
344
387
|
end
|
|
345
388
|
|
|
346
389
|
def column_ref(col, prefix:)
|
|
347
390
|
if col.respond_to?(:source) && col.source
|
|
348
391
|
"''"
|
|
349
392
|
else
|
|
350
|
-
"#{prefix}.#{col.name}"
|
|
393
|
+
"#{prefix}.#{qc(col.name)}"
|
|
351
394
|
end
|
|
352
395
|
end
|
|
353
396
|
|
|
397
|
+
# NOTE: This lock prevents concurrent rebuilds within the same process/connection only.
|
|
398
|
+
# For multi-process or multi-host deployments, run full_search:rebuild from a single
|
|
399
|
+
# deployment step. See README.
|
|
354
400
|
def with_rebuild_lock(model)
|
|
355
401
|
if FullSearch.config.lock_rebuilds
|
|
356
402
|
connection.transaction do
|
|
357
403
|
connection.execute(
|
|
358
|
-
"INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q("")}, datetime('now'))
|
|
404
|
+
"INSERT INTO full_search_index_versions (table_name, config_hash, rebuilt_at) VALUES (#{q(model.table_name)}, #{q("__rebuilding__")}, datetime('now'))
|
|
359
405
|
ON CONFLICT(table_name) DO UPDATE SET config_hash=excluded.config_hash;"
|
|
360
406
|
)
|
|
361
407
|
yield
|
data/lib/full_search/model.rb
CHANGED
|
@@ -13,7 +13,8 @@ module FullSearch
|
|
|
13
13
|
FullSearch::Index.ensure_table!(self)
|
|
14
14
|
FullSearch::Callbacks.install!(self)
|
|
15
15
|
include InstanceMethods
|
|
16
|
-
|
|
16
|
+
|
|
17
|
+
FullSearch.register_model(self)
|
|
17
18
|
@full_search_dsl
|
|
18
19
|
else
|
|
19
20
|
FullSearch::Search.new(self, query_or_options, filters: filters, include_soft_deleted: include_soft_deleted, limit: limit, offset: offset, highlight: highlight, highlight_fields: highlight_fields, matching_strategy: matching_strategy).relation
|
|
@@ -14,6 +14,8 @@ module FullSearch
|
|
|
14
14
|
def call
|
|
15
15
|
searched = groups.map do |group|
|
|
16
16
|
model = fetch(group, :model)
|
|
17
|
+
raise FullSearch::NotConfiguredError, "#{model} is not full_search configured" unless model.full_search_dsl
|
|
18
|
+
|
|
17
19
|
filters = group[:filters] || {}
|
|
18
20
|
limit = positive_integer(group[:limit], 8)
|
|
19
21
|
offset = positive_integer(group[:offset], 0)
|
|
@@ -24,6 +26,7 @@ module FullSearch
|
|
|
24
26
|
filters: filters,
|
|
25
27
|
limit: raw_limit,
|
|
26
28
|
offset: offset,
|
|
29
|
+
highlight: group[:highlight],
|
|
27
30
|
highlight_fields: group[:highlight_fields]
|
|
28
31
|
)
|
|
29
32
|
|
|
@@ -40,7 +43,7 @@ module FullSearch
|
|
|
40
43
|
)
|
|
41
44
|
end
|
|
42
45
|
|
|
43
|
-
{
|
|
46
|
+
{groups: searched, total_count: searched.sum { |g| g[:total_count] }}
|
|
44
47
|
end
|
|
45
48
|
|
|
46
49
|
private
|
|
@@ -6,121 +6,134 @@ module FullSearch
|
|
|
6
6
|
module QueryParser
|
|
7
7
|
Token = Data.define(:type, :value)
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
tokens = tokenize(query.to_s.strip)
|
|
11
|
-
return [] if tokens.empty?
|
|
9
|
+
MAX_QUERY_LENGTH = 255
|
|
12
10
|
|
|
13
|
-
|
|
11
|
+
class << self
|
|
12
|
+
def parse(query)
|
|
13
|
+
query = query.to_s.strip
|
|
14
|
+
validate!(query)
|
|
15
|
+
tokens = tokenize(query)
|
|
16
|
+
return [] if tokens.empty?
|
|
14
17
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
18
|
+
or_clauses = split_or(tokens)
|
|
19
|
+
|
|
20
|
+
if or_clauses.size == 1
|
|
21
|
+
parse_and_clause(or_clauses.first)
|
|
22
|
+
else
|
|
23
|
+
[:or, or_clauses.map { |clause| parse_and_clause(clause) }]
|
|
24
|
+
end
|
|
19
25
|
end
|
|
20
|
-
end
|
|
21
26
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
27
|
+
def to_match_expression(parsed)
|
|
28
|
+
return '""' if parsed.empty?
|
|
29
|
+
|
|
30
|
+
type, value = parsed
|
|
31
|
+
case type
|
|
32
|
+
when :and
|
|
33
|
+
build_and_expression(value)
|
|
34
|
+
when :or
|
|
35
|
+
value.map { |sub| to_match_expression(sub) }.join(" OR ")
|
|
36
|
+
else
|
|
37
|
+
node_to_match(parsed)
|
|
38
|
+
end
|
|
33
39
|
end
|
|
34
|
-
end
|
|
35
40
|
|
|
36
|
-
|
|
41
|
+
private
|
|
37
42
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
43
|
+
def build_and_expression(nodes)
|
|
44
|
+
non_excludes = nodes.reject { |n| n.first == :exclude }
|
|
45
|
+
excludes = nodes.select { |n| n.first == :exclude }
|
|
41
46
|
|
|
42
|
-
|
|
43
|
-
|
|
47
|
+
parts = non_excludes.map { |n| node_to_match(n) }
|
|
48
|
+
result = (parts.size == 1) ? parts.first : "(#{parts.join(" AND ")})"
|
|
44
49
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
50
|
+
excludes.each do |node|
|
|
51
|
+
_, value = node
|
|
52
|
+
result = "#{result} NOT \"#{escape(value)}\""
|
|
53
|
+
end
|
|
49
54
|
|
|
50
|
-
|
|
51
|
-
|
|
55
|
+
result
|
|
56
|
+
end
|
|
52
57
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
def node_to_match(node)
|
|
59
|
+
type, value = node
|
|
60
|
+
case type
|
|
61
|
+
when :term
|
|
62
|
+
'"' + escape(value) + '"*'
|
|
63
|
+
when :phrase
|
|
64
|
+
'"' + escape(value) + '"'
|
|
65
|
+
when :exclude
|
|
66
|
+
nil
|
|
67
|
+
end
|
|
62
68
|
end
|
|
63
|
-
end
|
|
64
69
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
70
|
+
def validate!(query)
|
|
71
|
+
if query.bytesize > MAX_QUERY_LENGTH
|
|
72
|
+
raise InvalidQueryError, "Query too long (max #{MAX_QUERY_LENGTH} chars)"
|
|
73
|
+
end
|
|
74
|
+
raise InvalidQueryError, "Query contains invalid characters" if query.include?("\x00")
|
|
75
|
+
end
|
|
68
76
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
77
|
+
def tokenize(query)
|
|
78
|
+
tokens = []
|
|
79
|
+
scanner = StringScanner.new(query)
|
|
72
80
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
if
|
|
76
|
-
|
|
81
|
+
until scanner.eos?
|
|
82
|
+
scanner.skip(/\s+/)
|
|
83
|
+
break if scanner.eos?
|
|
84
|
+
|
|
85
|
+
if scanner.scan('"')
|
|
86
|
+
phrase = scanner.scan_until(/"/)
|
|
87
|
+
if phrase
|
|
88
|
+
tokens << Token.new(:phrase, phrase.chomp('"'))
|
|
89
|
+
else
|
|
90
|
+
tokens << Token.new(:term, scanner.rest)
|
|
91
|
+
scanner.terminate
|
|
92
|
+
end
|
|
93
|
+
elsif scanner.scan("-")
|
|
94
|
+
scanner.skip(/\s+/)
|
|
95
|
+
tokens << Token.new(:exclude, scanner.scan(/\S+/))
|
|
96
|
+
elsif scanner.scan(/OR|or/i)
|
|
97
|
+
tokens << Token.new(:or, nil)
|
|
77
98
|
else
|
|
78
|
-
tokens << Token.new(:term, scanner.
|
|
79
|
-
scanner.terminate
|
|
99
|
+
tokens << Token.new(:term, scanner.scan(/\S+/))
|
|
80
100
|
end
|
|
81
|
-
elsif scanner.scan(/-/)
|
|
82
|
-
scanner.skip(/\s+/)
|
|
83
|
-
tokens << Token.new(:exclude, scanner.scan(/\S+/))
|
|
84
|
-
elsif scanner.scan(/OR|or/i)
|
|
85
|
-
tokens << Token.new(:or, nil)
|
|
86
|
-
else
|
|
87
|
-
tokens << Token.new(:term, scanner.scan(/\S+/))
|
|
88
101
|
end
|
|
89
|
-
end
|
|
90
102
|
|
|
91
|
-
|
|
92
|
-
|
|
103
|
+
tokens
|
|
104
|
+
end
|
|
93
105
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
106
|
+
def split_or(tokens)
|
|
107
|
+
clauses = []
|
|
108
|
+
current = []
|
|
97
109
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
110
|
+
tokens.each do |token|
|
|
111
|
+
if token.type == :or
|
|
112
|
+
clauses << current unless current.empty?
|
|
113
|
+
current = []
|
|
114
|
+
else
|
|
115
|
+
current << token
|
|
116
|
+
end
|
|
104
117
|
end
|
|
118
|
+
clauses << current unless current.empty?
|
|
119
|
+
clauses
|
|
105
120
|
end
|
|
106
|
-
clauses << current unless current.empty?
|
|
107
|
-
clauses
|
|
108
|
-
end
|
|
109
121
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
122
|
+
def parse_and_clause(tokens)
|
|
123
|
+
nodes = tokens.map do |token|
|
|
124
|
+
case token.type
|
|
125
|
+
when :term then [:term, token.value]
|
|
126
|
+
when :phrase then [:phrase, token.value]
|
|
127
|
+
when :exclude then [:exclude, token.value]
|
|
128
|
+
end
|
|
116
129
|
end
|
|
117
|
-
end
|
|
118
130
|
|
|
119
|
-
|
|
120
|
-
|
|
131
|
+
(nodes.size == 1) ? nodes.first : [:and, nodes]
|
|
132
|
+
end
|
|
121
133
|
|
|
122
|
-
|
|
123
|
-
|
|
134
|
+
def escape(value)
|
|
135
|
+
value.to_s.gsub('"', '""')
|
|
136
|
+
end
|
|
124
137
|
end
|
|
125
138
|
end
|
|
126
139
|
end
|