huginn_datatable 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +14 -1
- data/README.pt-BR.md +14 -1
- data/lib/generators/huginn/fts_indexes/fts_indexes_generator.rb +49 -0
- data/lib/generators/huginn/fts_indexes/templates/add_huginn_fts_indexes.rb.tt +24 -0
- data/lib/generators/huginn/searchable_columns.rb +69 -0
- data/lib/generators/huginn/trigram_indexes/trigram_indexes_generator.rb +2 -48
- data/lib/huginn/configuration.rb +27 -8
- data/lib/huginn/datatable/association_path.rb +6 -0
- data/lib/huginn/searchable/fuzzy.rb +45 -1
- data/lib/huginn/searchable/query.rb +122 -19
- data/lib/huginn/searchable/searchable.rb +9 -5
- data/lib/huginn/version.rb +1 -1
- metadata +4 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e68c742da97818c79b41681099972a3e4c3cc4613b8500831226abe09b2a87ed
|
|
4
|
+
data.tar.gz: 855168e5b6bbc9df1bceb381484bf20f550fcf57c0d0a35d6aba15c50b20a775
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f345b93074fec47d95589bdc06078b5c2585a79e0fd2a7c02c519d3ca35baa286a885f94278a64048004891a139f4c97a293848cf094523cbcff85ccc2276ce5
|
|
7
|
+
data.tar.gz: d2bfa72d92cc096dba420c20e2a9bc0b26f466e0b981b5d9e653d6f5f2dc33c105c5fd12a1c3e3269691ecafd0704efabaec80943f166b610798066420c3fd81
|
data/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
### Unreleased
|
|
4
4
|
|
|
5
|
+
#### 0.3.0
|
|
6
|
+
|
|
7
|
+
- **New `:full_text` search strategy** — PostgreSQL full text search over lexemes (`public.f_tsvector(col) @@ plainto_tsquery('portuguese'::regconfig, 'term')`), so morphological variants match (e.g. "compras" finds "compra"). Blind to typos, unlike `:pg_trgm`. Uses a GIN tsvector index when the `fts_function` wrapper (IMMUTABLE, dictionary pinned) exists; degrades gracefully to unaccent+ILIKE otherwise.
|
|
8
|
+
- `search_strategy:` now accepts **a single symbol or an Array of them**; multiple strategies are combined with **OR** (a row matches if any strategy matches), e.g. `[ :pg_trgm, :full_text ]` to get typo tolerance plus stemming.
|
|
9
|
+
- New `fts_dictionary:` (default `"portuguese"`) and `fts_function:` (default `"public.f_tsvector"`) configurations, mirroring `unaccent_function`.
|
|
10
|
+
- New generator **`rails g huginn:fts_indexes [Model...]`** scaffolds a migration that creates the IMMUTABLE `public.f_tsvector` wrapper (with `--dictionary`) and GIN tsvector indexes on every `:string`/`:text` column of the Searchable models, including association-scoped ones.
|
|
11
|
+
- `searchable_columns columns: []` is now honored — an empty array means "search only the declared associations", no longer falling back to the model's own string/text columns.
|
|
12
|
+
- Specs for the `Configuration` normalization surface (defaults + `search_strategy` symbol/array shapes).
|
|
13
|
+
|
|
5
14
|
#### 0.2.1
|
|
6
15
|
|
|
7
16
|
- Association-scoped `search` now matches through **primary-key semi-join subqueries** (`pk IN (SELECT DISTINCT pk FROM <base> JOIN <chain> WHERE <fuzzy>)`) instead of `left_joins` — one subquery per distinct association chain, combined with OR. The base relation never joins, so the count is a plain `COUNT(*)`.
|
data/README.md
CHANGED
|
@@ -66,9 +66,13 @@ gem "huginn"
|
|
|
66
66
|
```ruby
|
|
67
67
|
# config/initializers/huginn.rb
|
|
68
68
|
Huginn.configure do |config|
|
|
69
|
-
# :pg_trgm
|
|
69
|
+
# :pg_trgm (recommended) — trigram similarity (%) OR unaccent+ILIKE
|
|
70
|
+
# :full_text — PostgreSQL full text search over lexemes
|
|
71
|
+
# (to_tsvector @@ plainto_tsquery)
|
|
70
72
|
# :unaccent — unaccent + ILIKE only
|
|
71
73
|
# :simple — plain LIKE
|
|
74
|
+
# Pass an Array to combine strategies with OR, e.g.
|
|
75
|
+
# config.search_strategy = [:pg_trgm, :full_text]
|
|
72
76
|
config.search_strategy = :pg_trgm
|
|
73
77
|
|
|
74
78
|
# Unaccent function used around search terms/columns. Defaults to the pg
|
|
@@ -77,6 +81,13 @@ Huginn.configure do |config|
|
|
|
77
81
|
# indexes can actually be used.
|
|
78
82
|
config.unaccent_function = "unaccent"
|
|
79
83
|
|
|
84
|
+
# Full text search (strategy :full_text). The dictionary and the IMMUTABLE
|
|
85
|
+
# tsvector wrapper (`rails g huginn:fts_indexes`) pinned to that dictionary;
|
|
86
|
+
# they only apply when the wrapper exists, otherwise search degrades to the
|
|
87
|
+
# unaccent fallback.
|
|
88
|
+
# config.fts_dictionary = "portuguese"
|
|
89
|
+
# config.fts_function = "public.f_tsvector"
|
|
90
|
+
|
|
80
91
|
config.pagy_items = 10 # default page size
|
|
81
92
|
config.pagy_max_items = 500 # hard cap for per_page
|
|
82
93
|
end
|
|
@@ -84,6 +95,8 @@ end
|
|
|
84
95
|
|
|
85
96
|
> **Threshold:** under `:pg_trgm` the similarity cutoff is the PostgreSQL GUC `pg_trgm.similarity_threshold` (default `0.3`), **not** a gem-level setting. Tune it with `SET pg_trgm.similarity_threshold = 0.4` / `SELECT set_limit(0.4)` on the database.
|
|
86
97
|
|
|
98
|
+
> **Strategy tradeoffs:** `:pg_trgm` tolerates typos and substrings but has no notion of morphology. `:full_text` matches morphological variants (e.g. "correndo"/"correr" → same stem) but is blind to typos, and shines on longer text columns — for name/keyword lookups it mostly overlaps with trigram. Combining both (`[:pg_trgm, :full_text]`) unions the two result sets and requires both index sets (`rails g huginn:trigram_indexes` + `rails g huginn:fts_indexes`).
|
|
99
|
+
|
|
87
100
|
## Railtie (automatic include)
|
|
88
101
|
|
|
89
102
|
By default the Railtie includes `Huginn::Datatable` and `Huginn::Searchable` into every `ActiveRecord::Base` model. You do **not** need `include` statements unless you opt in selectively:
|
data/README.pt-BR.md
CHANGED
|
@@ -66,9 +66,13 @@ gem "huginn"
|
|
|
66
66
|
```ruby
|
|
67
67
|
# config/initializers/huginn.rb
|
|
68
68
|
Huginn.configure do |config|
|
|
69
|
-
# :pg_trgm
|
|
69
|
+
# :pg_trgm (recomendado) — similaridade trigram (%) OU unaccent+ILIKE
|
|
70
|
+
# :full_text — full text search do PostgreSQL sobre lexemas
|
|
71
|
+
# (to_tsvector @@ plainto_tsquery)
|
|
70
72
|
# :unaccent — somente unaccent + ILIKE
|
|
71
73
|
# :simple — LIKE simples
|
|
74
|
+
# Para combinar estratégias com OR, passe um Array:
|
|
75
|
+
# config.search_strategy = [:pg_trgm, :full_text]
|
|
72
76
|
config.search_strategy = :pg_trgm
|
|
73
77
|
|
|
74
78
|
# Função unaccent usada em torno de termos/colunas. O default é o UNACCENT()
|
|
@@ -77,6 +81,13 @@ Huginn.configure do |config|
|
|
|
77
81
|
# GIN trigram possam ser usados de fato.
|
|
78
82
|
config.unaccent_function = "unaccent"
|
|
79
83
|
|
|
84
|
+
# Full text search (estratégia :full_text). O dicionário e o wrapper
|
|
85
|
+
# tsvector IMMUTABLE (`rails g huginn:fts_indexes`) fixado nesse dicionário;
|
|
86
|
+
# só têm efeito quando o wrapper existe — caso contrário a busca degrada
|
|
87
|
+
# para o fallback unaccent.
|
|
88
|
+
# config.fts_dictionary = "portuguese"
|
|
89
|
+
# config.fts_function = "public.f_tsvector"
|
|
90
|
+
|
|
80
91
|
config.pagy_items = 10 # tamanho de página padrão
|
|
81
92
|
config.pagy_max_items = 500 # teto máximo de per_page
|
|
82
93
|
end
|
|
@@ -84,6 +95,8 @@ end
|
|
|
84
95
|
|
|
85
96
|
> **Limiar:** sob `:pg_trgm`, o corte de similaridade é o GUC do PostgreSQL `pg_trgm.similarity_threshold` (default `0.3`), **não** uma configuração da gem. Ajuste com `SET pg_trgm.similarity_threshold = 0.4` / `SELECT set_limit(0.4)` no banco.
|
|
86
97
|
|
|
98
|
+
> **Tradeoffs de estratégia:** `:pg_trgm` tolera typos e substrings, mas não tem noção de morfologia. `:full_text` captura variações morfológicas (ex. "correndo"/"correr" → mesmo lexema), mas é cego a typos, e brilha em colunas de texto mais longo — para busca por nome/palavra-chave ele em grande parte se sobrepõe ao trigram. Combinar os dois (`[:pg_trgm, :full_text]`) une os dois conjuntos de resultado e exige os dois conjuntos de índices (`rails g huginn:trigram_indexes` + `rails g huginn:fts_indexes`).
|
|
99
|
+
|
|
87
100
|
## Railtie (include automático)
|
|
88
101
|
|
|
89
102
|
Por padrão, o Railtie inclui `Huginn::Datatable` e `Huginn::Searchable` em toda `ActiveRecord::Base`. Você **não** precisa de `include`, a menos que opte seletivamente:
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/migration"
|
|
5
|
+
require "generators/huginn/searchable_columns"
|
|
6
|
+
|
|
7
|
+
module Huginn
|
|
8
|
+
module Generators
|
|
9
|
+
# Scaffolds a migration that creates the PostgreSQL GIN tsvector indexes
|
|
10
|
+
# (on an IMMUTABLE wrapper around to_tsvector) backing Huginn::Searchable's
|
|
11
|
+
# :full_text strategy for every model that includes Huginn::Searchable.
|
|
12
|
+
#
|
|
13
|
+
# rails g huginn:fts_indexes # all Searchable models
|
|
14
|
+
# rails g huginn:fts_indexes Person Product # specific models
|
|
15
|
+
# rails g huginn:fts_indexes --dictionary english # pick the dictionary
|
|
16
|
+
#
|
|
17
|
+
# The migration creates an IMMUTABLE public.f_tsvector(text) wrapper that
|
|
18
|
+
# pins the configured dictionary and the GIN indexes. Point the gem at the
|
|
19
|
+
# wrapper (to_tsvector itself is STABLE and cannot be indexed directly):
|
|
20
|
+
#
|
|
21
|
+
# Huginn.configure { |c| c.search_strategy = :full_text;
|
|
22
|
+
# c.fts_dictionary = "portuguese";
|
|
23
|
+
# c.fts_function = "public.f_tsvector" }
|
|
24
|
+
#
|
|
25
|
+
# Apply the same searchable model/association resolution as the trigram
|
|
26
|
+
# generator, so both index sets cover identical columns.
|
|
27
|
+
class FtsIndexesGenerator < Rails::Generators::Base
|
|
28
|
+
include SearchableColumns
|
|
29
|
+
|
|
30
|
+
source_root File.expand_path("templates", __dir__)
|
|
31
|
+
|
|
32
|
+
argument :model_names, type: :array, default: [], desc: "Models to index (default: all Searchable models)"
|
|
33
|
+
|
|
34
|
+
class_option :dictionary, type: :string, default: "portuguese",
|
|
35
|
+
desc: "tsvector dictionary pinned by the IMMUTABLE wrapper"
|
|
36
|
+
|
|
37
|
+
def copy_migration
|
|
38
|
+
@entries = resolve_entries
|
|
39
|
+
@dictionary = options[:dictionary]
|
|
40
|
+
return if @entries.empty?
|
|
41
|
+
|
|
42
|
+
migration_template(
|
|
43
|
+
"add_huginn_fts_indexes.rb.tt",
|
|
44
|
+
"db/migrate/add_huginn_fts_indexes.rb"
|
|
45
|
+
)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
class AddHuginnFtsIndexes < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
2
|
+
def up
|
|
3
|
+
# to_tsvector(regconfig, text) is STABLE (not IMMUTABLE), so it cannot be
|
|
4
|
+
# used in an index expression as-is. Wrap it in an IMMUTABLE function that
|
|
5
|
+
# always pins the configured dictionary.
|
|
6
|
+
execute <<~SQL
|
|
7
|
+
CREATE OR REPLACE FUNCTION public.f_tsvector(text)
|
|
8
|
+
RETURNS tsvector AS $$
|
|
9
|
+
SELECT to_tsvector('<%= @dictionary %>', $1);
|
|
10
|
+
$$
|
|
11
|
+
LANGUAGE sql IMMUTABLE PARALLEL SAFE;
|
|
12
|
+
SQL
|
|
13
|
+
<% @entries.each do |entry| %>
|
|
14
|
+
add_index :<%= entry[:table] %>, "public.f_tsvector(<%= entry[:column] %>)", using: :gin, name: "index_<%= entry[:table] %>_<%= entry[:column] %>_fts"
|
|
15
|
+
<% end %>
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def down
|
|
19
|
+
<% @entries.each do |entry| %>
|
|
20
|
+
remove_index :<%= entry[:table] %>, name: "index_<%= entry[:table] %>_<%= entry[:column] %>_fts"
|
|
21
|
+
<% end %>
|
|
22
|
+
execute "DROP FUNCTION IF EXISTS public.f_tsvector(text)"
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/migration"
|
|
5
|
+
|
|
6
|
+
module Huginn
|
|
7
|
+
module Generators
|
|
8
|
+
# Shared logic between the index-scaffold generators
|
|
9
|
+
# (huginn:trigram_indexes / huginn:fts_indexes): resolving the Searchable
|
|
10
|
+
# models (or the explicit ones given as arguments) into the table/column
|
|
11
|
+
# pairs that need an index, and the migration filename machinery.
|
|
12
|
+
module SearchableColumns
|
|
13
|
+
extend ActiveSupport::Concern
|
|
14
|
+
|
|
15
|
+
included do
|
|
16
|
+
include Rails::Generators::Migration
|
|
17
|
+
|
|
18
|
+
# Rails::Generators::Migration leaves this as NotImplementedError; it
|
|
19
|
+
# is expected to be overridden to produce the next migration number.
|
|
20
|
+
def self.next_migration_number(dirname)
|
|
21
|
+
number = current_migration_number(dirname) + 1
|
|
22
|
+
ActiveRecord::Migration.next_migration_number(number)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def resolve_entries
|
|
29
|
+
models = if model_names.any?
|
|
30
|
+
model_names.map { |name| constantize(name) }
|
|
31
|
+
else
|
|
32
|
+
ActiveRecord::Base.descendants
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
models.filter_map do |model|
|
|
36
|
+
next unless model.respond_to?(:searchable_columns_config_resolved)
|
|
37
|
+
|
|
38
|
+
columns_for(model)
|
|
39
|
+
end.reject(&:empty?).flatten.uniq
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def columns_for(model)
|
|
43
|
+
scope = []
|
|
44
|
+
resolved = model.searchable_columns_config_resolved
|
|
45
|
+
|
|
46
|
+
Array(resolved[:columns]).map(&:to_s).reject(&:blank?).each do |field|
|
|
47
|
+
scope << { table: model.table_name, column: field } if model.columns_hash.key?(field)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
resolved[:associations].each do |assoc, cols|
|
|
51
|
+
reflection = model.reflect_on_association(assoc.to_sym)
|
|
52
|
+
next unless reflection
|
|
53
|
+
|
|
54
|
+
Array(cols).map(&:to_s).each do |col|
|
|
55
|
+
scope << { table: reflection.klass.table_name, column: col } if reflection.klass.columns_hash.key?(col)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
scope
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def constantize(klass_name)
|
|
63
|
+
klass_name.constantize
|
|
64
|
+
rescue NameError
|
|
65
|
+
raise Thor::Error, "Could not find model #{klass_name.inspect}"
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "rails/generators"
|
|
4
4
|
require "rails/generators/migration"
|
|
5
|
+
require "generators/huginn/searchable_columns"
|
|
5
6
|
|
|
6
7
|
module Huginn
|
|
7
8
|
module Generators
|
|
@@ -23,17 +24,12 @@ module Huginn
|
|
|
23
24
|
# Indexes are only used when the :pg_trgm strategy is configured and the
|
|
24
25
|
# pg_trgm/unaccent extensions are installed.
|
|
25
26
|
class TrigramIndexesGenerator < Rails::Generators::Base
|
|
26
|
-
include
|
|
27
|
+
include SearchableColumns
|
|
27
28
|
|
|
28
29
|
source_root File.expand_path("templates", __dir__)
|
|
29
30
|
|
|
30
31
|
argument :model_names, type: :array, default: [], desc: "Models to index (default: all Searchable models)"
|
|
31
32
|
|
|
32
|
-
def self.next_migration_number(dirname)
|
|
33
|
-
next_migration_number = current_migration_number(dirname) + 1
|
|
34
|
-
ActiveRecord::Migration.next_migration_number(next_migration_number)
|
|
35
|
-
end
|
|
36
|
-
|
|
37
33
|
def copy_migration
|
|
38
34
|
@entries = resolve_entries
|
|
39
35
|
return if @entries.empty?
|
|
@@ -43,48 +39,6 @@ module Huginn
|
|
|
43
39
|
"db/migrate/add_huginn_trigram_indexes.rb"
|
|
44
40
|
)
|
|
45
41
|
end
|
|
46
|
-
|
|
47
|
-
private
|
|
48
|
-
|
|
49
|
-
def resolve_entries
|
|
50
|
-
models = if model_names.any?
|
|
51
|
-
model_names.map { |name| constantize(name) }
|
|
52
|
-
else
|
|
53
|
-
ActiveRecord::Base.descendants
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
models.filter_map do |model|
|
|
57
|
-
next unless model.respond_to?(:searchable_columns_config_resolved)
|
|
58
|
-
|
|
59
|
-
columns_for(model)
|
|
60
|
-
end.reject(&:empty?).flatten.uniq
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
def columns_for(model)
|
|
64
|
-
scope = []
|
|
65
|
-
resolved = model.searchable_columns_config_resolved
|
|
66
|
-
|
|
67
|
-
Array(resolved[:columns]).map(&:to_s).reject(&:blank?).each do |field|
|
|
68
|
-
scope << { table: model.table_name, column: field } if model.columns_hash.key?(field)
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
resolved[:associations].each do |assoc, cols|
|
|
72
|
-
reflection = model.reflect_on_association(assoc.to_sym)
|
|
73
|
-
next unless reflection
|
|
74
|
-
|
|
75
|
-
Array(cols).map(&:to_s).each do |col|
|
|
76
|
-
scope << { table: reflection.klass.table_name, column: col } if reflection.klass.columns_hash.key?(col)
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
scope
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
def constantize(klass_name)
|
|
84
|
-
klass_name.constantize
|
|
85
|
-
rescue NameError
|
|
86
|
-
raise Thor::Error, "Could not find model #{klass_name.inspect}"
|
|
87
|
-
end
|
|
88
42
|
end
|
|
89
43
|
end
|
|
90
44
|
end
|
data/lib/huginn/configuration.rb
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module Huginn
|
|
4
4
|
class Configuration
|
|
5
|
-
attr_accessor :pagy_items, :pagy_max_items, :unaccent_function
|
|
5
|
+
attr_accessor :pagy_items, :pagy_max_items, :unaccent_function,
|
|
6
|
+
:fts_dictionary, :fts_function
|
|
6
7
|
attr_writer :search_strategy, :auto_include_datatable, :auto_include_searchable
|
|
7
8
|
|
|
8
9
|
def initialize
|
|
@@ -13,19 +14,37 @@ module Huginn
|
|
|
13
14
|
# this at an IMMUTABLE wrapper (see `rails g huginn:trigram_indexes`),
|
|
14
15
|
# e.g. "public.f_unaccent".
|
|
15
16
|
@unaccent_function = "unaccent"
|
|
17
|
+
# The tsvector wrapper and dictionary used by the :full_text strategy. The
|
|
18
|
+
# wrapper (see `rails g huginn:fts_indexes`) pins the dictionary and is
|
|
19
|
+
# IMMUTABLE so GIN tsvector indexes are usable.
|
|
20
|
+
@fts_dictionary = "portuguese"
|
|
21
|
+
@fts_function = "public.f_tsvector"
|
|
16
22
|
@search_strategy = :pg_trgm
|
|
17
23
|
@auto_include_datatable = true
|
|
18
24
|
@auto_include_searchable = true
|
|
19
25
|
end
|
|
20
26
|
|
|
21
|
-
# :pg_trgm
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
# :
|
|
27
|
+
# :pg_trgm -> trigram similarity (%) OR unaccent+ILIKE. The
|
|
28
|
+
# similarity cutoff is the PostgreSQL GUC
|
|
29
|
+
# pg_trgm.similarity_threshold (tune with
|
|
30
|
+
# SELECT set_limit(...)); a gin_trgm_ops GIN index on
|
|
31
|
+
# UNACCENT(col) is used when present.
|
|
32
|
+
# :full_text -> PostgreSQL full text search (lexemes): the column is
|
|
33
|
+
# matched as to_tsvector(DICTIONARY, col) vs
|
|
34
|
+
# plainto_tsquery(DICTIONARY, term). Needs the
|
|
35
|
+
# fts_function wrapper and a GIN tsvector index.
|
|
36
|
+
# :unaccent -> unaccent + ILIKE only
|
|
37
|
+
# :simple -> plain LIKE
|
|
38
|
+
#
|
|
39
|
+
# Accepts a single symbol or an Array of symbols; when more than one is
|
|
40
|
+
# given the generated predicates are combined with OR (a row matches if
|
|
41
|
+
# any strategy matches).
|
|
27
42
|
def search_strategy
|
|
28
|
-
@search_strategy.
|
|
43
|
+
if @search_strategy.is_a?(Array)
|
|
44
|
+
@search_strategy.map(&:to_sym)
|
|
45
|
+
else
|
|
46
|
+
@search_strategy.to_sym
|
|
47
|
+
end
|
|
29
48
|
end
|
|
30
49
|
|
|
31
50
|
def auto_include_datatable?
|
|
@@ -77,6 +77,12 @@ module Huginn
|
|
|
77
77
|
nil
|
|
78
78
|
end
|
|
79
79
|
|
|
80
|
+
# The Arel predicates folded in by the association scopes of every step.
|
|
81
|
+
# Public so subquery builders can replicate ActiveRecord's scoped joins.
|
|
82
|
+
def scope_constraints
|
|
83
|
+
scope_predicates
|
|
84
|
+
end
|
|
85
|
+
|
|
80
86
|
# Reverse engineers a join spec usable by `ActiveRecord::Relation#joins`
|
|
81
87
|
# for filter subqueries: `.joins(:people: { company: ... })`.
|
|
82
88
|
def self.join_spec_for(names)
|
|
@@ -9,6 +9,11 @@ module Huginn
|
|
|
9
9
|
# :pg_trgm -> trigram similarity (%) OR unaccent+ILIKE (best for typos).
|
|
10
10
|
# Both branches are supported by a gin_trgm_ops GIN index
|
|
11
11
|
# on UNACCENT(col); the index is used only when it exists.
|
|
12
|
+
# :full_text -> PostgreSQL full text search: to_tsvector(col) @@
|
|
13
|
+
# plainto_tsquery(term), so morphological variants match;
|
|
14
|
+
# needs the IMMUTABLE fts_function wrapper and a GIN
|
|
15
|
+
# tsvector index. Effective with a single strategy or
|
|
16
|
+
# combined with :pg_trgm (OR).
|
|
12
17
|
# :unaccent -> unaccent + ILIKE (case/accents insensitive)
|
|
13
18
|
# :simple -> plain LIKE
|
|
14
19
|
module Fuzzy
|
|
@@ -90,6 +95,45 @@ module Huginn
|
|
|
90
95
|
Unaccent.new(@column, @value).predicate
|
|
91
96
|
end
|
|
92
97
|
end
|
|
98
|
+
|
|
99
|
+
# PostgreSQL full text search over lexemes:
|
|
100
|
+
#
|
|
101
|
+
# public.f_tsvector(col) @@ plainto_tsquery('portuguese'::regconfig, term)
|
|
102
|
+
#
|
|
103
|
+
# The column side runs through the configured fts_function (an IMMUTABLE
|
|
104
|
+
# wrapper around to_tsvector that pins the dictionary, created by
|
|
105
|
+
# `rails g huginn:fts_indexes`) so a GIN tsvector index is usable. The
|
|
106
|
+
# query side uses plainto_tsquery, which ANDs the lexemes of the term and
|
|
107
|
+
# applies stemming — tolerant to morphological variants, blind to typos.
|
|
108
|
+
class FullText
|
|
109
|
+
def initialize(column, value, dictionary: nil)
|
|
110
|
+
@column = column
|
|
111
|
+
@value = value.to_s
|
|
112
|
+
@dictionary = dictionary || Huginn.configuration.fts_dictionary
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def predicate
|
|
116
|
+
Arel::Nodes::InfixOperation.new("@@", tsvector, query)
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
private
|
|
120
|
+
|
|
121
|
+
def tsvector
|
|
122
|
+
Arel::Nodes::NamedFunction.new(Huginn.configuration.fts_function, [@column])
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def query
|
|
126
|
+
Arel::Nodes::NamedFunction.new("plainto_tsquery", [regconfig, quoted_value])
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def regconfig
|
|
130
|
+
Arel.sql("'#{@dictionary}'::regconfig")
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def quoted_value
|
|
134
|
+
Arel::Nodes.build_quoted(@value, @column)
|
|
135
|
+
end
|
|
136
|
+
end
|
|
93
137
|
end
|
|
94
138
|
end
|
|
95
|
-
end
|
|
139
|
+
end
|
|
@@ -6,14 +6,22 @@ module Huginn
|
|
|
6
6
|
module Searchable
|
|
7
7
|
AssociationPath = Datatable::AssociationPath unless const_defined?(:AssociationPath)
|
|
8
8
|
# Builds a tolerant full-text search relation over a model's searchable
|
|
9
|
-
# columns.
|
|
9
|
+
# columns. The matching predicate per column is driven by
|
|
10
|
+
# Huginn.configuration.search_strategy (:pg_trgm, :full_text, :unaccent,
|
|
11
|
+
# :simple — or an Array of them combined with OR).
|
|
10
12
|
#
|
|
11
13
|
# Columns are split in two groups following the datatable pattern:
|
|
12
14
|
#
|
|
13
15
|
# * the model's own columns -> direct fuzzy predicates on the base WHERE
|
|
14
|
-
# * association columns -> semi-join subqueries on
|
|
15
|
-
#
|
|
16
|
-
#
|
|
16
|
+
# * association columns -> reverse semi-join subqueries anchored on
|
|
17
|
+
# the first FK/has_many hop, so the base
|
|
18
|
+
# table is never re-scanned:
|
|
19
|
+
#
|
|
20
|
+
# belongs_to -> base[fk] IN (SELECT DISTINCT t1[pk] FROM t1 ...)
|
|
21
|
+
# has_many -> base[pk] IN (SELECT DISTINCT t1[fk] FROM t1 ...)
|
|
22
|
+
#
|
|
23
|
+
# Through/hand-joined chains keep the generic fallback:
|
|
24
|
+
# base[pk] IN (SELECT DISTINCT base[pk] FROM base JOIN <chain> ...)
|
|
17
25
|
#
|
|
18
26
|
# The base relation never carries joins, so every match group stays one
|
|
19
27
|
# query and the count remains a plain COUNT(*). Multiple association
|
|
@@ -36,9 +44,9 @@ module Huginn
|
|
|
36
44
|
|
|
37
45
|
resolution = resolve_columns
|
|
38
46
|
predicates = fuzzy_predicates(resolution.fetch(:own))
|
|
39
|
-
resolution.fetch(:associations).each do |
|
|
40
|
-
|
|
41
|
-
predicates <<
|
|
47
|
+
resolution.fetch(:associations).each do |_chain, group|
|
|
48
|
+
predicate = association_predicate(group)
|
|
49
|
+
predicates << predicate if predicate
|
|
42
50
|
end
|
|
43
51
|
|
|
44
52
|
return apply_distinct(relation) if predicates.empty?
|
|
@@ -49,13 +57,13 @@ module Huginn
|
|
|
49
57
|
private
|
|
50
58
|
|
|
51
59
|
def fuzzy_predicates(attrs)
|
|
52
|
-
attrs.map { |attr| fuzzy_predicate(attr) }
|
|
60
|
+
attrs.map { |attr| fuzzy_predicate(attr) }.compact
|
|
53
61
|
end
|
|
54
62
|
|
|
55
63
|
def resolve_columns
|
|
56
64
|
columns = @model.searchable_columns_config_resolved
|
|
57
65
|
own = []
|
|
58
|
-
associations = Hash.new { |hash, key| hash[key] = [] }
|
|
66
|
+
associations = Hash.new { |hash, key| hash[key] = { path: nil, attrs: [] } }
|
|
59
67
|
|
|
60
68
|
Array(columns[:columns]).map(&:to_s).reject(&:blank?).each do |field|
|
|
61
69
|
if field.include?(".")
|
|
@@ -78,35 +86,109 @@ module Huginn
|
|
|
78
86
|
return unless path.valid?
|
|
79
87
|
return unless path.target_klass.columns_hash.key?(path.column)
|
|
80
88
|
|
|
81
|
-
groups[path.association_names]
|
|
89
|
+
group = groups[path.association_names]
|
|
90
|
+
group[:path] ||= path
|
|
91
|
+
group[:attrs] << path.target_table[path.column]
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Prefers the FK-anchored reverse semi-join; falls back to the generic
|
|
95
|
+
# pk subquery for through chains or anything the reverse plan rejects.
|
|
96
|
+
def association_predicate(group)
|
|
97
|
+
path = group[:path]
|
|
98
|
+
attrs = group[:attrs]
|
|
99
|
+
return nil if path.nil? || attrs.empty?
|
|
100
|
+
|
|
101
|
+
anchored_predicate(path, attrs)
|
|
102
|
+
rescue StandardError
|
|
103
|
+
generic_predicate(path, attrs)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Reverse semi-join:
|
|
107
|
+
#
|
|
108
|
+
# belongs_to -> base[fk] IN (SELECT DISTINCT t1[join_pk] FROM t1 ...)
|
|
109
|
+
# has_many -> base[pk] IN (SELECT DISTINCT t1[fk] FROM t1 ...)
|
|
110
|
+
#
|
|
111
|
+
# Walks the chain from the first target table outwards so the base table
|
|
112
|
+
# is never scanned inside the subquery.
|
|
113
|
+
def anchored_predicate(path, attrs)
|
|
114
|
+
if through_association?(path)
|
|
115
|
+
return generic_predicate(path, attrs)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
steps = path.resolved_steps
|
|
119
|
+
first = steps.first
|
|
120
|
+
reflection = first.reflection
|
|
121
|
+
|
|
122
|
+
outer = if reflection.belongs_to?
|
|
123
|
+
arel_table[reflection.foreign_key]
|
|
124
|
+
else
|
|
125
|
+
arel_table[reflection.active_record_primary_key]
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
projection = if reflection.belongs_to?
|
|
129
|
+
first.table[reflection.join_primary_key]
|
|
130
|
+
else
|
|
131
|
+
first.table[reflection.foreign_key]
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
manager = Arel::SelectManager.new
|
|
135
|
+
manager.from(first.table)
|
|
136
|
+
steps.drop(1).each { |step| manager.join(step.table).on(step.link_to_previous) }
|
|
137
|
+
manager.where(fuzzy_predicates(attrs).reduce(&:or))
|
|
138
|
+
path.scope_constraints.each { |constraint| manager.where(constraint) }
|
|
139
|
+
manager.project(projection)
|
|
140
|
+
manager.distinct
|
|
141
|
+
|
|
142
|
+
outer.in(manager)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Whether the first hop goes through a join model; the FK anchoring is
|
|
146
|
+
# ambiguous there so the generic pk subquery is used instead.
|
|
147
|
+
def through_association?(path)
|
|
148
|
+
reflection = @model.reflect_on_association(path.segments.first.to_sym)
|
|
149
|
+
reflection.is_a?(ActiveRecord::Reflection::ThroughReflection)
|
|
82
150
|
end
|
|
83
151
|
|
|
84
|
-
#
|
|
85
|
-
#
|
|
86
|
-
#
|
|
87
|
-
def
|
|
88
|
-
return nil
|
|
152
|
+
# Generic fallback semi-join subquery over the base table, mirroring
|
|
153
|
+
# datatable's association_ids_subquery:
|
|
154
|
+
# base[pk] IN (SELECT DISTINCT base[pk] FROM base JOIN <chain> ...).
|
|
155
|
+
def generic_predicate(path, attrs)
|
|
156
|
+
return nil unless path
|
|
89
157
|
|
|
90
|
-
spec = AssociationPath.join_spec_for(
|
|
158
|
+
spec = AssociationPath.join_spec_for(path.association_names.map(&:to_sym))
|
|
91
159
|
base = @model.all.except(:select, :order, :offset, :limit)
|
|
92
160
|
.select(arel_table[primary_key])
|
|
93
161
|
.distinct
|
|
94
162
|
|
|
95
163
|
predicate = fuzzy_predicates(attrs).reduce(&:or)
|
|
96
|
-
base.joins(spec).where(predicate)
|
|
164
|
+
arel_table[primary_key].in(base.joins(spec).where(predicate).arel)
|
|
97
165
|
end
|
|
98
166
|
|
|
99
167
|
def fuzzy_predicate(attr)
|
|
100
|
-
|
|
168
|
+
strategies = Array(strategy)
|
|
169
|
+
predicates = strategies.map { |strat| predicate_for_strategy(attr, strat) }.compact
|
|
170
|
+
return nil if predicates.empty?
|
|
171
|
+
|
|
172
|
+
predicates.reduce(&:or)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def predicate_for_strategy(attr, strat)
|
|
176
|
+
case strat
|
|
101
177
|
when :pg_trgm
|
|
102
178
|
if pg_trgm_available?
|
|
103
179
|
Fuzzy::Trigram.new(attr, @value).predicate
|
|
104
180
|
else
|
|
105
181
|
fallback_fuzzy(attr)
|
|
106
182
|
end
|
|
183
|
+
when :full_text
|
|
184
|
+
if fts_available?
|
|
185
|
+
Fuzzy::FullText.new(attr, @value).predicate
|
|
186
|
+
else
|
|
187
|
+
fallback_fuzzy(attr)
|
|
188
|
+
end
|
|
107
189
|
when :unaccent
|
|
108
190
|
Fuzzy::Unaccent.new(attr, @value).predicate
|
|
109
|
-
|
|
191
|
+
when :simple
|
|
110
192
|
Fuzzy::Simple.new(attr, @value).predicate
|
|
111
193
|
end
|
|
112
194
|
end
|
|
@@ -135,6 +217,27 @@ module Huginn
|
|
|
135
217
|
@unaccent_available ||= extension_installed?("unaccent")
|
|
136
218
|
end
|
|
137
219
|
|
|
220
|
+
# Whether the IMMUTABLE tsvector wrapper behind the :full_text strategy
|
|
221
|
+
# exists (created by `rails g huginn:fts_indexes`). FTS is core
|
|
222
|
+
# PostgreSQL, but without the wrapper the predicate would raise, so we
|
|
223
|
+
# degrade to the unaccent fallback instead.
|
|
224
|
+
def fts_available?
|
|
225
|
+
return false unless postgresql?
|
|
226
|
+
@fts_available ||= begin
|
|
227
|
+
schema, name = split_function_name(Huginn.configuration.fts_function)
|
|
228
|
+
@model.connection.select_value(
|
|
229
|
+
ActiveRecord::Base.sanitize_sql_array(
|
|
230
|
+
["SELECT 1 FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname = ? AND p.proname = ?", schema, name]
|
|
231
|
+
)
|
|
232
|
+
).present?
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def split_function_name(function)
|
|
237
|
+
schema, _, name = function.to_s.rpartition(".")
|
|
238
|
+
[schema.presence || "public", name]
|
|
239
|
+
end
|
|
240
|
+
|
|
138
241
|
def extension_installed?(name)
|
|
139
242
|
@model.connection
|
|
140
243
|
.select_all(sanitize_extension_query(name))
|
|
@@ -36,11 +36,15 @@ module Huginn
|
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
def searchable_columns_config_resolved
|
|
39
|
-
config = searchable_columns_config
|
|
40
|
-
|
|
41
|
-
columns:
|
|
42
|
-
|
|
43
|
-
|
|
39
|
+
config = searchable_columns_config
|
|
40
|
+
if config.nil?
|
|
41
|
+
{ columns: default_searchable_columns, associations: {} }
|
|
42
|
+
else
|
|
43
|
+
{
|
|
44
|
+
columns: Array(config[:columns]),
|
|
45
|
+
associations: config[:associations] || {}
|
|
46
|
+
}
|
|
47
|
+
end
|
|
44
48
|
end
|
|
45
49
|
|
|
46
50
|
private
|
data/lib/huginn/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: huginn_datatable
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kayky Marcelo
|
|
@@ -139,6 +139,9 @@ files:
|
|
|
139
139
|
- LICENSE.txt
|
|
140
140
|
- README.md
|
|
141
141
|
- README.pt-BR.md
|
|
142
|
+
- lib/generators/huginn/fts_indexes/fts_indexes_generator.rb
|
|
143
|
+
- lib/generators/huginn/fts_indexes/templates/add_huginn_fts_indexes.rb.tt
|
|
144
|
+
- lib/generators/huginn/searchable_columns.rb
|
|
142
145
|
- lib/generators/huginn/trigram_indexes/templates/add_huginn_trigram_indexes.rb.tt
|
|
143
146
|
- lib/generators/huginn/trigram_indexes/trigram_indexes_generator.rb
|
|
144
147
|
- lib/huginn.rb
|