runestone 1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,11 @@
1
+ class Runestone::WebSearch::Or
2
+ attr_accessor :values
3
+ def initialize(values, negative: false)
4
+ @values = values
5
+ @negative = negative
6
+ end
7
+
8
+ def to_s
9
+ "(#{values.map(&:to_s).join(' | ')})"
10
+ end
11
+ end
@@ -0,0 +1,19 @@
1
+ class Runestone::WebSearch::Phrase
2
+ attr_accessor :values, :prefix, :negative, :distance
3
+ def initialize(values, prefix: false, negative: false, distance: nil)
4
+ @values = values
5
+ @prefix = prefix
6
+ @negative = negative
7
+ @distance = distance
8
+ end
9
+
10
+ def to_s
11
+ v = if values.size == 1
12
+ values.first.to_s
13
+ else
14
+ seperator = distance ? " <#{distance}> " : ' <-> '
15
+ "(#{values.map(&:to_s).join(seperator)})"
16
+ end
17
+ negative ? "!#{v}" : v
18
+ end
19
+ end
@@ -0,0 +1,27 @@
1
+ class Runestone::WebSearch::Token
2
+ attr_accessor :value, :prefix, :negative, :alts
3
+ def initialize(value, prefix: false, negative: false, alts: nil)
4
+ @value = value
5
+ @prefix = prefix
6
+ @negative = negative
7
+ @alts = alts || []
8
+ end
9
+
10
+ def to_s
11
+ if negative
12
+ "!#{value}"
13
+ elsif prefix
14
+ if alts.empty?
15
+ "#{value}:*"
16
+ else
17
+ "(#{value}:* | #{alts.join(' | ')})"
18
+ end
19
+ else
20
+ if alts.empty?
21
+ value
22
+ else
23
+ "(#{value} | #{alts.join(' | ')})"
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,32 @@
1
+ require File.expand_path("../lib/runestone/version", __FILE__)
2
+
3
+ Gem::Specification.new do |s|
4
+ s.name = "runestone"
5
+ s.version = Runestone::VERSION
6
+ s.authors = ["Jon Bracy"]
7
+ s.email = ["jonbracy@gmail.com"]
8
+ s.homepage = "https://github.com/malomalo/runestone"
9
+ s.summary = %q{Full Text Search for Active Record / Rails}
10
+ s.description = %q{PostgreSQL Full Text Search for Active Record and Rails}
11
+ s.license = 'MIT'
12
+
13
+ s.files = `git ls-files`.split("\n")
14
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
15
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
16
+ s.require_paths = ["lib"]
17
+
18
+ # Developoment
19
+ s.add_development_dependency 'rake'
20
+ s.add_development_dependency 'bundler'
21
+ s.add_development_dependency 'minitest'
22
+ s.add_development_dependency 'minitest-reporters'
23
+ s.add_development_dependency 'pg'
24
+ s.add_development_dependency 'byebug'
25
+ s.add_development_dependency 'faker'
26
+ s.add_development_dependency 'simplecov'
27
+ s.add_development_dependency 'activejob', '>= 6.0'
28
+
29
+ # Runtime
30
+ s.add_runtime_dependency 'arel-extensions', '>= 6.0'
31
+ s.add_runtime_dependency 'activerecord', '>= 6.0'
32
+ end
@@ -0,0 +1,42 @@
1
+ require 'test_helper'
2
+
3
+ class CorpusTest < ActiveSupport::TestCase
4
+
5
+ test 'similar_words' do
6
+ Address.create(name: 'Address name broccolini')
7
+
8
+ assert_equal(
9
+ {},
10
+ Runestone::Corpus.similar_words('nam')
11
+ )
12
+
13
+ assert_equal(
14
+ {'addresz' => ['address']},
15
+ Runestone::Corpus.similar_words('addresz')
16
+ )
17
+
18
+ assert_equal(
19
+ {'brockolinl' => ['broccolini']},
20
+ Runestone::Corpus.similar_words('brockolinl')
21
+ )
22
+
23
+ assert_equal(
24
+ {
25
+ 'addresz' => ['address'],
26
+ 'brockolinl' => ['broccolini']
27
+ },
28
+ Runestone::Corpus.similar_words('nam', 'addresz', 'brockolinl')
29
+ )
30
+ end
31
+
32
+ test 'adding words to corpus downcases them' do
33
+ Runestone::Corpus.add('Allée')
34
+ assert_equal(
35
+ {
36
+ "Allee" => ["allée"]
37
+ },
38
+ Runestone::Corpus.similar_words('Allee')
39
+ )
40
+ end
41
+
42
+ end
@@ -0,0 +1,119 @@
1
+ GlobalID.app = 'TestApp'
2
+
3
+ task = ActiveRecord::Tasks::PostgreSQLDatabaseTasks.new({
4
+ 'adapter' => 'postgresql',
5
+ 'database' => "arel-extensions-test"
6
+ })
7
+ task.drop
8
+ task.create
9
+
10
+ ActiveRecord::Base.establish_connection({
11
+ adapter: "postgresql",
12
+ database: "arel-extensions-test",
13
+ encoding: "utf8"
14
+ })
15
+
16
+ ActiveRecord::Migration.suppress_messages do
17
+ ActiveRecord::Schema.define do
18
+ enable_extension 'pgcrypto'
19
+ enable_extension 'pg_trgm'
20
+ enable_extension 'unaccent'
21
+ enable_extension 'fuzzystrmatch'
22
+
23
+ create_table :addresses, id: :uuid, force: :cascade do |t|
24
+ t.string "name"
25
+ t.uuid "property_id"
26
+ end
27
+
28
+ create_table :properties, id: :uuid, force: :cascade do |t|
29
+ t.string "name", limit: 255
30
+ end
31
+
32
+ create_table :regions, id: :uuid, force: :cascade do |t|
33
+ t.string "name", limit: 255
34
+ end
35
+
36
+ create_table :buildings, id: :uuid, force: :cascade do |t|
37
+ t.string "name_en", limit: 255
38
+ t.string "name_ru", limit: 255
39
+ end
40
+
41
+ create_table :runestones, id: :uuid, force: :cascade do |t|
42
+ t.string :record_type, null: false
43
+ t.uuid :record_id, null: false
44
+ t.string :name
45
+ t.string :dictionary
46
+ t.jsonb :data, null: false
47
+ t.tsvector :vector, null: false
48
+ end
49
+
50
+ add_index :runestones, [:record_type, :record_id, :name, :dictionary], unique: true, name: 'index_runestones_for_uniqueness'
51
+ add_index :runestones, :vector, using: :gin
52
+
53
+ execute <<-SQL
54
+ CREATE TABLE runestone_corpus ( word varchar, CONSTRAINT word UNIQUE(word) );
55
+
56
+ CREATE INDEX runestone_corpus_trgm_idx ON runestone_corpus USING GIN (word gin_trgm_ops);
57
+
58
+
59
+ CREATE TEXT SEARCH CONFIGURATION simple_unaccent (COPY = simple);
60
+ ALTER TEXT SEARCH CONFIGURATION simple_unaccent
61
+ ALTER MAPPING FOR hword, hword_part, word
62
+ WITH unaccent, simple;
63
+ SQL
64
+ end
65
+ end
66
+
67
+ class Address < ActiveRecord::Base
68
+
69
+ belongs_to :property
70
+
71
+ runestone do
72
+ index 'name'
73
+
74
+ attribute(:name)
75
+ end
76
+
77
+ end
78
+
79
+ class Region < ActiveRecord::Base
80
+
81
+ include GlobalID::Identification
82
+
83
+ runestone runner: :active_job do
84
+ index 'name'
85
+
86
+ attribute(:name)
87
+ end
88
+
89
+ end
90
+
91
+ class Property < ActiveRecord::Base
92
+
93
+ has_many :addresses
94
+
95
+ runestone do
96
+ index :name
97
+ index 'addresses.name', weight: 3
98
+
99
+ attribute(:name)
100
+ attribute(:addresses) { addresses.map{ |a| a&.attributes&.slice('id', 'name') } }
101
+ end
102
+
103
+ end
104
+
105
+ class Building < ActiveRecord::Base
106
+
107
+ runestone dictionary: 'english' do
108
+ index :name
109
+
110
+ attribute(:name) { name_en }
111
+ end
112
+
113
+ runestone dictionary: 'russian' do
114
+ index :name
115
+
116
+ attribute(:name) { name_ru }
117
+ end
118
+
119
+ end
@@ -0,0 +1,34 @@
1
+ require 'test_helper'
2
+
3
+ class DelayedIndexingTest < ActiveSupport::TestCase
4
+
5
+ test 'simple_unaccent index' do
6
+ region = assert_no_difference 'Runestone::Model.count' do
7
+ assert_no_sql(/setweight\(to_tsvector\('simple_unaccent', 'address name'\), 'A'\)/) do
8
+ Region.create(name: 'Region name')
9
+ end
10
+ end
11
+
12
+ job = assert_enqueued_with(
13
+ job: Runestone::IndexingJob,
14
+ args: [region, :create_runestones!],
15
+ queue: 'runestone_indexing'
16
+ )
17
+
18
+ job.perform_now
19
+
20
+ assert_equal([[
21
+ 'Region', region.id,
22
+ {"name" => "Region name"},
23
+ "'name':2A 'region':1A"
24
+ ]], region.runestones.map { |runestone|
25
+ [
26
+ runestone.record_type,
27
+ runestone.record_id,
28
+ runestone.data,
29
+ runestone.vector
30
+ ]
31
+ })
32
+ end
33
+
34
+ end
@@ -0,0 +1,40 @@
1
+ require 'test_helper'
2
+
3
+ class WebSearchTest < ActiveSupport::TestCase
4
+
5
+ test '::parse' do
6
+ assert_equal "the & fat & rats:*", Runestone::WebSearch.parse('The fat rats').to_s
7
+ assert_equal "(supernovae <-> stars) & !crab", Runestone::WebSearch.parse('"supernovae stars" -crab').to_s
8
+ # assert_equal "(sad <-> cat) | (fat <-> rat)", TS.websearch_to_tsquery('"sad cat" || "fat rat"')
9
+ assert_equal "signal & !(segmentation <-> fault)", Runestone::WebSearch.parse('signal -"segmentation fault"').to_s
10
+ end
11
+
12
+ test '::parse(query, prefix: :all)' do
13
+ assert_equal "the:* & fat:* & rats:*", Runestone::WebSearch.parse('The fat rats', prefix: :all).to_s
14
+ assert_equal "(supernovae <-> stars) & !crab", Runestone::WebSearch.parse('"supernovae stars" -crab', prefix: :all).to_s
15
+ # assert_equal "(sad <-> cat) | (fat <-> rat)", TS.websearch_to_tsquery('"sad cat" || "fat rat"')
16
+ assert_equal "signal:* & !(segmentation <-> fault)", Runestone::WebSearch.parse('signal -"segmentation fault"', prefix: :all).to_s
17
+ end
18
+
19
+ test '::parse(query, prefix: :last)' do
20
+ assert_equal "the & fat & rats:*", Runestone::WebSearch.parse('The fat rats', prefix: :last).to_s
21
+ assert_equal "(supernovae <-> stars) & !crab", Runestone::WebSearch.parse('"supernovae stars" -crab', prefix: :last).to_s
22
+ # assert_equal "(sad <-> cat) | (fat <-> rat)", TS.websearch_to_tsquery('"sad cat" || "fat rat"')
23
+ assert_equal "signal & !(segmentation <-> fault)", Runestone::WebSearch.parse('signal -"segmentation fault"', prefix: :last).to_s
24
+ end
25
+
26
+ test '::parse(query, prefix: :none)' do
27
+ assert_equal "the & fat & rats", Runestone::WebSearch.parse('The fat rats', prefix: :none).to_s
28
+ assert_equal "(supernovae <-> stars) & !crab", Runestone::WebSearch.parse('"supernovae stars" -crab', prefix: :none).to_s
29
+ # assert_equal "(sad <-> cat) | (fat <-> rat)", TS.websearch_to_tsquery('"sad cat" || "fat rat"')
30
+ assert_equal "signal & !(segmentation <-> fault)", Runestone::WebSearch.parse('signal -"segmentation fault"', prefix: :none).to_s
31
+ end
32
+
33
+ test '::parse(weird query, prefix: :none)' do
34
+ assert_equal "(supernovae <-> stars)", Runestone::WebSearch.parse('"supernovae stars', prefix: :none).to_s
35
+ assert_equal "signal", Runestone::WebSearch.parse('signal -', prefix: :none).to_s
36
+ assert_equal "signal", Runestone::WebSearch.parse('signal -"', prefix: :none).to_s
37
+ assert_equal "signal", Runestone::WebSearch.parse('signal -""', prefix: :none).to_s
38
+ end
39
+
40
+ end
@@ -0,0 +1,26 @@
1
+ require 'test_helper'
2
+
3
+ class HighlightTest < ActiveSupport::TestCase
4
+
5
+ test '::highlights(query)' do
6
+ Property.create(name: 'Empire state building', addresses: [Address.create(name: 'address uno')])
7
+ Property.create(name: 'Big state building', addresses: [Address.create(name: 'address of state duo')])
8
+
9
+ tsmodels = Runestone::Model.search('state')
10
+ Runestone::Model.highlight(tsmodels, 'state')
11
+ assert_equal([
12
+ {
13
+ "name"=>"Big <b>state</b> building",
14
+ "addresses"=> [{"name"=>"address of <b>state</b> duo"}]
15
+ },
16
+ {
17
+ "name"=>"Empire <b>state</b> building",
18
+ "addresses"=> [{"name"=>"address uno"}]
19
+ },
20
+ {
21
+ "name"=>"address of <b>state</b> duo"
22
+ }
23
+ ], tsmodels.map(&:highlights))
24
+ end
25
+
26
+ end
@@ -0,0 +1,151 @@
1
+ require 'test_helper'
2
+
3
+ class IndexingTest < ActiveSupport::TestCase
4
+
5
+ test 'simple_unaccent index' do
6
+ address = assert_difference 'Runestone::Model.count', 1 do
7
+ assert_sql(/setweight\(to_tsvector\('simple_unaccent', 'address name'\), 'A'\)/) do
8
+ Address.create(name: 'Address name')
9
+ end
10
+ end
11
+
12
+ assert_equal([[
13
+ 'Address', address.id,
14
+ {"name" => "Address name"},
15
+ "'address':1A 'name':2A"
16
+ ]], address.runestones.map { |runestone|
17
+ [
18
+ runestone.record_type,
19
+ runestone.record_id,
20
+ runestone.data,
21
+ runestone.vector
22
+ ]
23
+ })
24
+ end
25
+
26
+ test 'empty index' do
27
+ address = assert_difference 'Runestone::Model.count', 1 do
28
+ Address.create(name: nil)
29
+ end
30
+
31
+ assert_equal([[
32
+ 'Address', address.id,
33
+ {},
34
+ ""
35
+ ]], address.runestones.map { |runestone|
36
+ [
37
+ runestone.record_type,
38
+ runestone.record_id,
39
+ runestone.data,
40
+ runestone.vector
41
+ ]
42
+ })
43
+ end
44
+
45
+ test 'complex index' do
46
+ property = assert_difference 'Runestone::Model.count', 3 do
47
+ Property.create(name: 'Property name', addresses: [
48
+ Address.create(name: 'Address 1'),
49
+ Address.create(name: 'Address 2')
50
+ ])
51
+ end
52
+
53
+ assert_equal([[
54
+ 'Property', property.id,
55
+ {
56
+ "name"=>"Property name",
57
+ "addresses"=>[
58
+ { "id" => property.addresses.first.id, "name" => "Address 1" },
59
+ { "id" => property.addresses.last.id, "name" => "Address 2" }
60
+ ]
61
+ },
62
+ "'1':4C '2':6C 'address':3C,5C 'name':2A 'property':1A"
63
+ ]], property.runestones.map { |runestone|
64
+ [
65
+ runestone.record_type,
66
+ runestone.record_id,
67
+ runestone.data,
68
+ runestone.vector
69
+ ]
70
+ })
71
+ end
72
+
73
+ test 'index Unicode strings get normalized' do
74
+ address = Address.create(name: "on\u0065\u0301")
75
+
76
+ assert_equal(["'on\u00e9':1A"], address.reload.runestones.map(&:vector))
77
+ end
78
+
79
+ test 'index gets created on Model.create' do
80
+ address = assert_difference 'Runestone::Model.count', 1 do
81
+ Address.create(name: 'Address name')
82
+ end
83
+
84
+ assert_corpus('address', 'name')
85
+ end
86
+
87
+ test 'index gets updated on Model.create' do
88
+ address = Address.create(name: 'Address name')
89
+ assert_no_difference 'Runestone::Model.count' do
90
+ address.update!(name: 'Address name two')
91
+ end
92
+
93
+ assert_equal([[
94
+ 'Address', address.id,
95
+ {"name" => "Address name two"},
96
+ "'address':1A 'name':2A 'two':3A"
97
+ ]], address.runestones.map { |runestone|
98
+ [
99
+ runestone.record_type,
100
+ runestone.record_id,
101
+ runestone.data,
102
+ runestone.vector
103
+ ]
104
+ })
105
+
106
+ assert_corpus('address', 'name', 'two')
107
+ end
108
+
109
+ test 'index gets deleted on Model.destroy' do
110
+ address = Address.create(name: 'Address name')
111
+ assert_difference 'Runestone::Model.count', -1 do
112
+ address.destroy!
113
+ end
114
+ end
115
+
116
+ test 'reindex! deleted removed records' do
117
+ a1 = Address.create(name: 'one')
118
+ a2 = Address.create(name: 'two')
119
+
120
+ assert_no_difference 'Runestone::Model.count' do
121
+ a2.delete
122
+ end
123
+
124
+ assert_difference 'Runestone::Model.count', -1 do
125
+ Address.reindex!
126
+ end
127
+ end
128
+
129
+ test 'reindex! updates runestone on outdated indexes' do
130
+ address = Address.create(name: 'one')
131
+ address.update_columns(name: 'two')
132
+
133
+ assert_equal(["'one':1A"], address.runestones.map(&:vector))
134
+ assert_no_difference 'Runestone::Model.count' do
135
+ Address.reindex!
136
+ end
137
+ assert_equal(["'two':1A"], address.runestones.map { |runestone| runestone.reload.vector })
138
+ end
139
+
140
+ test 'reindex! creates index if not there' do
141
+ address = Address.create(name: 'one')
142
+ address.runestones.each(&:delete)
143
+
144
+ assert_equal 0, address.reload.runestones.size
145
+ assert_difference 'Runestone::Model.count', 1 do
146
+ Address.reindex!
147
+ end
148
+ assert_equal(["'one':1A"], address.reload.runestones.map(&:vector))
149
+ end
150
+
151
+ end