litestack 0.2.6 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/BENCHMARKS.md +11 -0
- data/CHANGELOG.md +19 -0
- data/Gemfile +2 -0
- data/README.md +1 -1
- data/assets/event_page.png +0 -0
- data/assets/index_page.png +0 -0
- data/assets/topic_page.png +0 -0
- data/bench/bench_jobs_rails.rb +1 -1
- data/bench/bench_jobs_raw.rb +1 -1
- data/bench/uljob.rb +1 -1
- data/lib/action_cable/subscription_adapter/litecable.rb +1 -11
- data/lib/active_support/cache/litecache.rb +1 -1
- data/lib/generators/litestack/install/templates/database.yml +5 -1
- data/lib/litestack/liteboard/liteboard.rb +172 -35
- data/lib/litestack/liteboard/views/index.erb +52 -20
- data/lib/litestack/liteboard/views/layout.erb +189 -38
- data/lib/litestack/liteboard/views/litecable.erb +118 -0
- data/lib/litestack/liteboard/views/litecache.erb +144 -0
- data/lib/litestack/liteboard/views/litedb.erb +168 -0
- data/lib/litestack/liteboard/views/litejob.erb +151 -0
- data/lib/litestack/litecable.rb +27 -37
- data/lib/litestack/litecable.sql.yml +1 -1
- data/lib/litestack/litecache.rb +7 -18
- data/lib/litestack/litedb.rb +17 -2
- data/lib/litestack/litejob.rb +2 -3
- data/lib/litestack/litejobqueue.rb +51 -48
- data/lib/litestack/litemetric.rb +46 -69
- data/lib/litestack/litemetric.sql.yml +14 -12
- data/lib/litestack/litemetric_collector.sql.yml +4 -4
- data/lib/litestack/litequeue.rb +9 -20
- data/lib/litestack/litescheduler.rb +84 -0
- data/lib/litestack/litesearch/index.rb +230 -0
- data/lib/litestack/litesearch/model.rb +178 -0
- data/lib/litestack/litesearch/schema.rb +193 -0
- data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +147 -0
- data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +128 -0
- data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +17 -0
- data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +33 -0
- data/lib/litestack/litesearch/schema_adapters.rb +9 -0
- data/lib/litestack/litesearch.rb +37 -0
- data/lib/litestack/litesupport.rb +55 -125
- data/lib/litestack/version.rb +1 -1
- data/lib/litestack.rb +2 -1
- data/lib/sequel/adapters/litedb.rb +3 -2
- metadata +20 -3
@@ -0,0 +1,230 @@
|
|
1
|
+
require 'oj'
|
2
|
+
require_relative './schema.rb'
|
3
|
+
|
4
|
+
class Litesearch::Index
|
5
|
+
|
6
|
+
DEFAULT_SEARCH_OPTIONS = {limit: 25, offset: 0}
|
7
|
+
|
8
|
+
def initialize(db, name)
|
9
|
+
@db = db # this index instance will always belong to this db instance
|
10
|
+
@stmts = {}
|
11
|
+
name = name.to_s.downcase.to_sym
|
12
|
+
# if in the db then put in cache and return if no schema is given
|
13
|
+
# if a schema is given then compare the new and the existing schema
|
14
|
+
# if they are the same put in cache and return
|
15
|
+
# if they differ only in weights then set the new weights, update the schema, put in cache and return
|
16
|
+
# if they differ in fields (added/removed/renamed) then update the structure, then rebuild if auto-rebuild is on
|
17
|
+
# if they differ in tokenizer then rebuild if auto-rebuild is on (error otherwise)
|
18
|
+
# if they differ in both then update the structure and rebuild if auto-rebuild is on (error otherwise)
|
19
|
+
load_index(name) if exists?(name)
|
20
|
+
|
21
|
+
if block_given?
|
22
|
+
schema = Litesearch::Schema.new
|
23
|
+
schema.schema[:name] = name
|
24
|
+
yield schema
|
25
|
+
schema.post_init
|
26
|
+
# now that we have a schema object we need to check if we need to create or modify and existing index
|
27
|
+
@db.transaction(:immediate) do
|
28
|
+
if exists?(name)
|
29
|
+
load_index(name)
|
30
|
+
do_modify(schema)
|
31
|
+
else
|
32
|
+
do_create(schema)
|
33
|
+
end
|
34
|
+
prepare_statements
|
35
|
+
end
|
36
|
+
else
|
37
|
+
if exists?(name)
|
38
|
+
# an index already exists, load it from the database and return the index instance to the caller
|
39
|
+
load_index(name)
|
40
|
+
prepare_statements
|
41
|
+
else
|
42
|
+
raise "index does not exist and no schema was supplied"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def load_index(name)
|
48
|
+
# we cannot use get_config_value here since the schema object is not created yet, should we allow something here?
|
49
|
+
@schema = Litesearch::Schema.new(Oj.load(@db.get_first_value("SELECT v from #{name}_config where k = ?", :litesearch_schema.to_s))) rescue nil
|
50
|
+
raise "index configuration not found, either corrupted or not a litesearch index!" if @schema.nil?
|
51
|
+
self
|
52
|
+
end
|
53
|
+
|
54
|
+
def modify
|
55
|
+
schema = Litesearch::Schema.new
|
56
|
+
yield schema
|
57
|
+
schema.schema[:name] = @schema.schema[:name]
|
58
|
+
do_modify(schema)
|
59
|
+
end
|
60
|
+
|
61
|
+
def rebuild!
|
62
|
+
@db.transaction(:immediate) do
|
63
|
+
do_rebuild
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def add(document)
|
68
|
+
@stmts[:insert].execute!(document)
|
69
|
+
return @db.last_insert_row_id
|
70
|
+
end
|
71
|
+
|
72
|
+
def remove(id)
|
73
|
+
@stmts[:delete].execute!(id)
|
74
|
+
end
|
75
|
+
|
76
|
+
def count(term = nil)
|
77
|
+
if term
|
78
|
+
@stmts[:count].execute!(term)[0][0]
|
79
|
+
else
|
80
|
+
@stmts[:count_all].execute!()[0][0]
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
# search options include
|
85
|
+
# limit: how many records to return
|
86
|
+
# offset: start from which record
|
87
|
+
def search(term, options = {})
|
88
|
+
result = []
|
89
|
+
options = DEFAULT_SEARCH_OPTIONS.merge(options)
|
90
|
+
rs = @stmts[:search].execute(term, options[:limit], options[:offset])
|
91
|
+
if @db.results_as_hash
|
92
|
+
rs.each_hash do |hash|
|
93
|
+
result << hash
|
94
|
+
end
|
95
|
+
else
|
96
|
+
result = rs.to_a
|
97
|
+
end
|
98
|
+
result
|
99
|
+
end
|
100
|
+
|
101
|
+
def clear!
|
102
|
+
@stmts[:delete_all].execute!(id)
|
103
|
+
end
|
104
|
+
|
105
|
+
def drop!
|
106
|
+
if @schema.get(:type) == :backed
|
107
|
+
@db.execute_batch(@schema.sql_for(:drop_primary_triggers))
|
108
|
+
if secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers)
|
109
|
+
@db.execute_batch(@schema.sql_for(:drop_secondary_triggers))
|
110
|
+
end
|
111
|
+
end
|
112
|
+
@db.execute(@schema.sql_for(:drop))
|
113
|
+
end
|
114
|
+
|
115
|
+
|
116
|
+
private
|
117
|
+
|
118
|
+
def exists?(name)
|
119
|
+
@db.get_first_value("SELECT count(*) FROM SQLITE_MASTER WHERE name = ? AND type = 'table' AND (sql like '%fts5%' OR sql like '%FTS5%')", name.to_s) == 1
|
120
|
+
end
|
121
|
+
|
122
|
+
def prepare_statements
|
123
|
+
stmt_names = [:insert, :delete, :delete_all, :drop, :count, :count_all, :search]
|
124
|
+
stmt_names.each do |stmt_name|
|
125
|
+
@stmts[stmt_name] = @db.prepare(@schema.sql_for(stmt_name))
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def do_create(schema)
|
130
|
+
@schema = schema
|
131
|
+
@schema.clean
|
132
|
+
# create index
|
133
|
+
@db.execute(schema.sql_for(:create_index, true))
|
134
|
+
# adjust ranking function
|
135
|
+
@db.execute(schema.sql_for(:ranks, true))
|
136
|
+
# create triggers (if any)
|
137
|
+
if @schema.get(:type) == :backed
|
138
|
+
@db.execute_batch(@schema.sql_for(:create_primary_triggers))
|
139
|
+
if secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers)
|
140
|
+
@db.execute_batch(secondary_triggers_sql)
|
141
|
+
end
|
142
|
+
@db.execute(@schema.sql_for(:rebuild)) if @schema.get(:rebuild_on_create)
|
143
|
+
end
|
144
|
+
set_config_value(:litesearch_schema, @schema.schema)
|
145
|
+
end
|
146
|
+
|
147
|
+
def do_modify(new_schema)
|
148
|
+
changes = @schema.compare(new_schema)
|
149
|
+
# ensure the new schema maintains feild order
|
150
|
+
new_schema.order_fields(@schema)
|
151
|
+
# with the changes object decide what needs to be done to the schema
|
152
|
+
requires_schema_change = false
|
153
|
+
requires_trigger_change = false
|
154
|
+
requires_rebuild = false
|
155
|
+
if changes[:fields] || changes[:table] || changes[:tokenizer] || changes[:filter_column] || changes[:removed_fields_count] > 0# any change here will require a schema change
|
156
|
+
requires_schema_change = true
|
157
|
+
# only a change in tokenizer
|
158
|
+
requires_rebuild = changes[:tokenizer] || new_schema.get(:rebuild_on_modify)
|
159
|
+
requires_trigger_change = (changes[:table] || changes[:fields] || changes[:filter_column]) && @schema.get(:type) == :backed
|
160
|
+
end
|
161
|
+
if requires_schema_change
|
162
|
+
# 1. enable schema editing
|
163
|
+
@db.execute("PRAGMA WRITABLE_SCHEMA = TRUE")
|
164
|
+
# 2. update the index sql
|
165
|
+
@db.execute(new_schema.sql_for(:update_index), new_schema.sql_for(:create_index))
|
166
|
+
# 3. update the content table sql (if it exists)
|
167
|
+
@db.execute(new_schema.sql_for(:update_content_table), new_schema.sql_for(:create_content_table, new_schema.schema[:fields].count))
|
168
|
+
# adjust shadow tables
|
169
|
+
@db.execute(new_schema.sql_for(:expand_data), changes[:extra_fields_count])
|
170
|
+
@db.execute(new_schema.sql_for(:expand_docsize), changes[:extra_fields_count])
|
171
|
+
@db.execute("PRAGMA WRITABLE_SCHEMA = RESET")
|
172
|
+
# need to reprepare statements
|
173
|
+
end
|
174
|
+
if requires_trigger_change
|
175
|
+
@db.execute_batch(new_schema.sql_for(:drop_primary_triggers))
|
176
|
+
@db.execute_batch(new_schema.sql_for(:create_primary_triggers))
|
177
|
+
if secondary_triggers_sql = new_schema.sql_for(:create_secondary_triggers)
|
178
|
+
@db.execute_batch(new_schema.sql_for(:drop_secondary_triggers))
|
179
|
+
@db.execute_batch(secondary_triggers_sql)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
if changes[:fields] || changes[:table] || changes[:tokenizer] || changes[:weights] || changes[:filter_column]
|
183
|
+
@schema = new_schema
|
184
|
+
set_config_value(:litesearch_schema, @schema.schema)
|
185
|
+
prepare_statements
|
186
|
+
#save_schema
|
187
|
+
end
|
188
|
+
do_rebuild if requires_rebuild
|
189
|
+
# update the weights if they changed
|
190
|
+
@db.execute(@schema.sql_for(:ranks)) if changes[:weights]
|
191
|
+
end
|
192
|
+
|
193
|
+
def do_rebuild
|
194
|
+
# remove any zero weight columns
|
195
|
+
if @schema.get(:type) == :backed
|
196
|
+
@db.execute_batch(@schema.sql_for(:drop_primary_triggers))
|
197
|
+
if secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers)
|
198
|
+
@db.execute_batch(@schema.sql_for(:drop_secondary_triggers))
|
199
|
+
end
|
200
|
+
@db.execute(@schema.sql_for(:drop))
|
201
|
+
@db.execute(@schema.sql_for(:create_index, true))
|
202
|
+
@db.execute_batch(@schema.sql_for(:create_primary_triggers))
|
203
|
+
@db.execute_batch(secondary_triggers_sql) if secondary_triggers_sql
|
204
|
+
@db.execute(@schema.sql_for(:rebuild))
|
205
|
+
elsif @schema.get(:type) == :standalone
|
206
|
+
removables = []
|
207
|
+
@schema.get(:fields).each_with_index{|f, i| removables << [f[0], i] if f[1][:weight] == 0 }
|
208
|
+
removables.each do |col|
|
209
|
+
@db.execute(@schema.sql_for(:drop_content_col, col[1]))
|
210
|
+
@schema.get(:fields).delete(col[0])
|
211
|
+
end
|
212
|
+
@db.execute("PRAGMA WRITABLE_SCHEMA = TRUE")
|
213
|
+
@db.execute(@schema.sql_for(:update_index), @schema.sql_for(:create_index, true))
|
214
|
+
@db.execute(@schema.sql_for(:update_content_table), @schema.sql_for(:create_content_table, @schema.schema[:fields].count))
|
215
|
+
@db.execute("PRAGMA WRITABLE_SCHEMA = RESET")
|
216
|
+
@db.execute(@schema.sql_for(:rebuild))
|
217
|
+
end
|
218
|
+
set_config_value(:litesearch_schema, @schema.schema)
|
219
|
+
@db.execute(@schema.sql_for(:ranks, true))
|
220
|
+
end
|
221
|
+
|
222
|
+
def get_config_value(key)
|
223
|
+
Oj.load(@db.get_first_value(@schema.sql_for(:get_config_value), key.to_s)) #rescue nil
|
224
|
+
end
|
225
|
+
|
226
|
+
def set_config_value(key, value)
|
227
|
+
@db.execute(@schema.sql_for(:set_config_value), key.to_s, Oj.dump(value))
|
228
|
+
end
|
229
|
+
|
230
|
+
end
|
@@ -0,0 +1,178 @@
|
|
1
|
+
module Litesearch::Model
|
2
|
+
|
3
|
+
def self.included(klass)
|
4
|
+
klass.include InstanceMethods
|
5
|
+
klass.extend ClassMethods
|
6
|
+
klass.attribute :search_rank, :float if klass.respond_to? :attribute
|
7
|
+
if defined?(Sequel::Model) != nil && klass.ancestors.include?(Sequel::Model)
|
8
|
+
klass.include Litesearch::Model::SequelInstanceMethods
|
9
|
+
klass.extend Litesearch::Model::SequelClassMethods
|
10
|
+
Sequel::Model.extend Litesearch::Model::BaseClassMethods
|
11
|
+
elsif defined?(ActiveRecord::Base) != nil && klass.ancestors.include?(ActiveRecord::Base)
|
12
|
+
klass.include Litesearch::Model::ActiveRecordInstanceMethods
|
13
|
+
klass.extend Litesearch::Model::ActiveRecordClassMethods
|
14
|
+
ActiveRecord::Base.extend Litesearch::Model::BaseClassMethods
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
module BaseClassMethods
|
19
|
+
def search_models
|
20
|
+
@@models ||= {}
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
module InstanceMethods
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
module ClassMethods
|
29
|
+
|
30
|
+
def litesearch
|
31
|
+
idx = get_connection.search_index(index_name) do |schema|
|
32
|
+
schema.type :backed
|
33
|
+
schema.table table_name.to_sym
|
34
|
+
yield schema
|
35
|
+
schema.post_init
|
36
|
+
@schema = schema #save the schema
|
37
|
+
end
|
38
|
+
if defined?(Sequel::Model) != nil && self.ancestors.include?(Sequel::Model)
|
39
|
+
Sequel::Model.search_models[self.name] = self
|
40
|
+
elsif defined?(ActiveRecord::Base) != nil && self.ancestors.include?(ActiveRecord::Base)
|
41
|
+
ActiveRecord::Base.search_models[self.name] = self
|
42
|
+
end
|
43
|
+
idx
|
44
|
+
end
|
45
|
+
|
46
|
+
def rebuild_index!
|
47
|
+
get_connection.search_index(index_name).rebuild!
|
48
|
+
end
|
49
|
+
|
50
|
+
def drop_index!
|
51
|
+
get_connection.search_index(index_name).drop!
|
52
|
+
end
|
53
|
+
|
54
|
+
def search_all(term, options={})
|
55
|
+
options[:offset] ||= 0
|
56
|
+
options[:limit] ||= 25
|
57
|
+
selects = []
|
58
|
+
if models = options[:models]
|
59
|
+
models_hash = {}
|
60
|
+
models.each do |model|
|
61
|
+
models_hash[model.name] = model
|
62
|
+
end
|
63
|
+
else
|
64
|
+
models_hash = search_models
|
65
|
+
end
|
66
|
+
models_hash.each do |name, klass|
|
67
|
+
selects << "SELECT '#{name}' AS model, rowid, -rank AS search_rank FROM #{index_name_for_table(klass.table_name)}('#{term}')"
|
68
|
+
end
|
69
|
+
conn = get_connection
|
70
|
+
sql = selects.join(" UNION ") << " ORDER BY search_rank DESC LIMIT #{options[:limit]} OFFSET #{options[:offset]}"
|
71
|
+
result = []
|
72
|
+
rs = conn.query(sql) #, options[:limit], options[:offset])
|
73
|
+
rs.each_hash do |row|
|
74
|
+
obj = models_hash[row["model"]].fetch_row(row["rowid"])
|
75
|
+
obj.search_rank = row["search_rank"]
|
76
|
+
result << obj
|
77
|
+
end
|
78
|
+
rs.close
|
79
|
+
result
|
80
|
+
end
|
81
|
+
|
82
|
+
# AR specific
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
def index_name
|
87
|
+
"#{table_name}_search_idx"
|
88
|
+
end
|
89
|
+
|
90
|
+
def index_name_for_table(table)
|
91
|
+
"#{table}_search_idx"
|
92
|
+
end
|
93
|
+
|
94
|
+
# create a new instance of self with the row as an argument
|
95
|
+
def create_instance(row)
|
96
|
+
self.new(row)
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
end
|
101
|
+
|
102
|
+
module ActiveRecordInstanceMethods;end
|
103
|
+
|
104
|
+
module ActiveRecordClassMethods
|
105
|
+
|
106
|
+
def get_connection
|
107
|
+
connection.raw_connection
|
108
|
+
end
|
109
|
+
|
110
|
+
def fetch_row(id)
|
111
|
+
find(id)
|
112
|
+
end
|
113
|
+
|
114
|
+
def search(term)
|
115
|
+
self.select(
|
116
|
+
"#{table_name}.*"
|
117
|
+
).joins(
|
118
|
+
"INNER JOIN #{index_name} ON #{table_name}.id = #{index_name}.rowid AND rank != 0 AND #{index_name} MATCH ", Arel.sql("'#{term}'")
|
119
|
+
).select(
|
120
|
+
"-#{index_name}.rank AS search_rank"
|
121
|
+
).order(
|
122
|
+
Arel.sql("#{index_name}.rank")
|
123
|
+
)
|
124
|
+
end
|
125
|
+
|
126
|
+
private
|
127
|
+
|
128
|
+
def create_instance(row)
|
129
|
+
instantiate(row)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
module SequelInstanceMethods
|
134
|
+
|
135
|
+
def search_rank
|
136
|
+
@values[:search_rank]
|
137
|
+
end
|
138
|
+
|
139
|
+
def search_rank=(rank)
|
140
|
+
@values[:search_rank] = rank
|
141
|
+
end
|
142
|
+
|
143
|
+
end
|
144
|
+
|
145
|
+
module SequelClassMethods
|
146
|
+
|
147
|
+
def fetch_row(id)
|
148
|
+
self[id]
|
149
|
+
end
|
150
|
+
|
151
|
+
def get_connection
|
152
|
+
db.instance_variable_get(:@raw_db)
|
153
|
+
end
|
154
|
+
|
155
|
+
def search(term)
|
156
|
+
dataset.select(
|
157
|
+
Sequel.lit("#{table_name}.*, -#{index_name}.rank AS search_rank")
|
158
|
+
).inner_join(
|
159
|
+
Sequel.lit("#{index_name}('#{term}') ON #{table_name}.id = #{index_name}.rowid AND rank != 0")
|
160
|
+
).order(
|
161
|
+
Sequel.lit('rank')
|
162
|
+
)
|
163
|
+
end
|
164
|
+
|
165
|
+
private
|
166
|
+
|
167
|
+
def create_instance(row)
|
168
|
+
# we need to convert keys to symbols first!
|
169
|
+
row.keys.each do |k|
|
170
|
+
next if k.is_a? Symbol
|
171
|
+
row[k.to_sym] = row[k]
|
172
|
+
row.delete(k)
|
173
|
+
end
|
174
|
+
self.call(row)
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
end
|
@@ -0,0 +1,193 @@
|
|
1
|
+
require_relative './schema_adapters.rb'
|
2
|
+
|
3
|
+
class Litesearch::Schema
|
4
|
+
|
5
|
+
TOKENIZERS = {
|
6
|
+
porter: 'porter unicode61 remove_diacritics 2',
|
7
|
+
unicode: 'unicode61 remove_diacritics 2',
|
8
|
+
ascii: 'ascii',
|
9
|
+
trigram: 'trigram'
|
10
|
+
}
|
11
|
+
|
12
|
+
INDEX_TYPES = {
|
13
|
+
standalone: Litesearch::Schema::StandaloneAdapter,
|
14
|
+
contentless: Litesearch::Schema::ContentlessAdapter,
|
15
|
+
backed: Litesearch::Schema::BackedAdapter
|
16
|
+
}
|
17
|
+
|
18
|
+
DEFAULT_SCHEMA = {
|
19
|
+
name: nil,
|
20
|
+
type: :standalone,
|
21
|
+
fields: nil,
|
22
|
+
table: nil,
|
23
|
+
filter_column: nil,
|
24
|
+
tokenizer: :porter,
|
25
|
+
auto_create: true,
|
26
|
+
auto_modify: true,
|
27
|
+
rebuild_on_create: false,
|
28
|
+
rebuild_on_modify: false
|
29
|
+
}
|
30
|
+
|
31
|
+
attr_accessor :schema
|
32
|
+
|
33
|
+
def initialize(schema = {})
|
34
|
+
@schema = schema #DEFAULT_SCHEMA.merge(schema)
|
35
|
+
@schema[:fields] = {} unless @schema[:fields]
|
36
|
+
end
|
37
|
+
|
38
|
+
# schema definition API
|
39
|
+
def name(new_name)
|
40
|
+
@schema[:name] = new_name
|
41
|
+
end
|
42
|
+
|
43
|
+
def type(new_type)
|
44
|
+
raise "Unknown index type" if INDEX_TYPES[new_type].nil?
|
45
|
+
@schema[:type] = new_type
|
46
|
+
end
|
47
|
+
|
48
|
+
def table(table_name)
|
49
|
+
@schema[:table] = table_name
|
50
|
+
end
|
51
|
+
|
52
|
+
def fields(field_names)
|
53
|
+
field_names.each {|f| field f }
|
54
|
+
end
|
55
|
+
|
56
|
+
def field(name, attributes = {})
|
57
|
+
name = name.to_s.downcase.to_sym
|
58
|
+
attributes = {weight: 1}.merge(attributes).select{|k, v| allowed_attributes.include?(k)} # only allow attributes we know, to ease schema comparison later
|
59
|
+
@schema[:fields][name] = attributes
|
60
|
+
end
|
61
|
+
|
62
|
+
def tokenizer(new_tokenizer)
|
63
|
+
raise "Unknown tokenizer" if TOKENIZERS[new_tokenizer].nil?
|
64
|
+
@schema[:tokenizer] = new_tokenizer
|
65
|
+
end
|
66
|
+
|
67
|
+
def filter_column(filter_column)
|
68
|
+
@schema[:filter_column] = filter_column
|
69
|
+
end
|
70
|
+
|
71
|
+
def auto_create(boolean)
|
72
|
+
@schema[:auto_create] = boolean
|
73
|
+
end
|
74
|
+
|
75
|
+
def auto_modify(boolean)
|
76
|
+
@schema[:auto_modify] = boolean
|
77
|
+
end
|
78
|
+
|
79
|
+
def rebuild_on_create(boolean)
|
80
|
+
@schema[:rebuild_on_create] = boolean
|
81
|
+
end
|
82
|
+
|
83
|
+
def rebuild_on_modify(boolean)
|
84
|
+
@schema[:rebuild_on_modify] = boolean
|
85
|
+
end
|
86
|
+
|
87
|
+
def post_init
|
88
|
+
@schema = DEFAULT_SCHEMA.merge(@schema)
|
89
|
+
end
|
90
|
+
|
91
|
+
# schema sql generation API
|
92
|
+
|
93
|
+
def sql_for(method, *args)
|
94
|
+
adapter.sql_for(method, *args)
|
95
|
+
end
|
96
|
+
|
97
|
+
# schema data structure API
|
98
|
+
def get(key)
|
99
|
+
@schema[key]
|
100
|
+
end
|
101
|
+
|
102
|
+
def get_field(name)
|
103
|
+
@schema[:fields][name]
|
104
|
+
end
|
105
|
+
|
106
|
+
def adapter
|
107
|
+
@adapter ||= INDEX_TYPES[@schema[:type]].new(@schema)
|
108
|
+
end
|
109
|
+
|
110
|
+
def reset_sql
|
111
|
+
adapter.generate_sql
|
112
|
+
end
|
113
|
+
|
114
|
+
def order_fields(old_schema)
|
115
|
+
adapter.order_fields(old_schema)
|
116
|
+
end
|
117
|
+
|
118
|
+
# should we do this at the schema objects level?
|
119
|
+
def compare(other_schema)
|
120
|
+
other_schema = other_schema.schema
|
121
|
+
# are the schemas identical?
|
122
|
+
# 1 - same fields?
|
123
|
+
[:type, :tokenizer, :name, :table].each do |key|
|
124
|
+
other_schema[key] = @schema[key] if other_schema[key].nil?
|
125
|
+
end
|
126
|
+
if @schema[:type] != other_schema[:type]
|
127
|
+
raise Litesearch::SchemaChangeException.new "Cannot change the index type, please drop the index before creating it again with the new type"
|
128
|
+
end
|
129
|
+
changes = { tokenizer: @schema[:tokenizer] != other_schema[:tokenizer], table: @schema[:table] != other_schema[:table], removed_fields_count: 0, filter_column: @schema[:filter_column] != other_schema[:filter_column] }
|
130
|
+
#check tokenizer changes
|
131
|
+
if changes[:tokenizer] && !other_schema[:rebuild_on_modify]
|
132
|
+
raise Litesearch::SchemaChangeException.new "Cannot change the tokenizer without an index rebuild!"
|
133
|
+
end
|
134
|
+
|
135
|
+
|
136
|
+
|
137
|
+
# check field changes
|
138
|
+
keys = @schema[:fields].keys.sort
|
139
|
+
other_keys = other_schema[:fields].keys.sort
|
140
|
+
|
141
|
+
extra_keys = other_keys - keys
|
142
|
+
extra_keys.each do |key|
|
143
|
+
if other_schema[:fields][key][:weight] == 0
|
144
|
+
other_schema[:fields].delete(key)
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
other_keys = other_schema[:fields].keys.sort
|
149
|
+
|
150
|
+
changes[:fields] = keys != other_keys # only acceptable change is adding extra fields
|
151
|
+
changes[:extra_fields_count] = other_keys.count - keys.count
|
152
|
+
# check for missing fields (please note that adding fields can work without a rebuild)
|
153
|
+
if keys - other_keys != []
|
154
|
+
raise Litesearch::SchemaChangeException.new "Missing fields from existing schema, they have to exist with weight zero until the next rebuild!"
|
155
|
+
end
|
156
|
+
|
157
|
+
# check field weights
|
158
|
+
weights = keys.collect{|key| @schema[:fields][key][:weight] }
|
159
|
+
other_weights = other_keys.collect{|key| other_schema[:fields][key][:weight] }
|
160
|
+
changes[:weights] = weights != other_weights # will always be true if fields are added
|
161
|
+
if (removed_count = other_weights.select{|w| w == 0}.count) > 0
|
162
|
+
changes[:removed_fields_count] = removed_count
|
163
|
+
end
|
164
|
+
# check field attributes, only backed tables have attributes
|
165
|
+
attrs = keys.collect do |key|
|
166
|
+
f = @schema[:fields][key].dup
|
167
|
+
f.delete(:weight)
|
168
|
+
f.select{|k,v| allowed_attributes.include? k }
|
169
|
+
end
|
170
|
+
other_attrs = other_keys.collect do |key|
|
171
|
+
f = other_schema[:fields][key].dup
|
172
|
+
f.delete(:weight)
|
173
|
+
f.select{|k,v| allowed_attributes.include? k }
|
174
|
+
end
|
175
|
+
changes[:attributes] if other_attrs != attrs # this means that we will need to redefine the triggers if any are there and also the table definition if needed
|
176
|
+
|
177
|
+
# return the changes
|
178
|
+
changes
|
179
|
+
end
|
180
|
+
|
181
|
+
def clean
|
182
|
+
removable = @schema[:fields].select{|name, f| f[:weight] == 0 }.collect{|name, f| name}
|
183
|
+
removable.each{|name| @schema[:fields].delete(name)}
|
184
|
+
end
|
185
|
+
|
186
|
+
def allowed_attributes
|
187
|
+
[:weight, :col, :target]
|
188
|
+
end
|
189
|
+
|
190
|
+
end
|
191
|
+
|
192
|
+
class Litesearch::SchemaException < StandardError; end
|
193
|
+
class Litesearch::SchemaChangeException < StandardError; end
|