litestack 0.4.1 → 0.4.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.standard.yml +3 -0
- data/BENCHMARKS.md +23 -7
- data/CHANGELOG.md +11 -0
- data/Gemfile +1 -7
- data/Gemfile.lock +92 -0
- data/README.md +120 -6
- data/ROADMAP.md +45 -0
- data/Rakefile +3 -1
- data/WHYLITESTACK.md +1 -1
- data/assets/litecache_metrics.png +0 -0
- data/assets/litedb_metrics.png +0 -0
- data/assets/litemetric_logo_teal.png +0 -0
- data/assets/litesearch_logo_teal.png +0 -0
- data/bench/bench.rb +17 -10
- data/bench/bench_cache_rails.rb +10 -13
- data/bench/bench_cache_raw.rb +17 -22
- data/bench/bench_jobs_rails.rb +18 -12
- data/bench/bench_jobs_raw.rb +17 -10
- data/bench/bench_queue.rb +4 -6
- data/bench/rails_job.rb +5 -7
- data/bench/skjob.rb +4 -4
- data/bench/uljob.rb +6 -6
- data/lib/action_cable/subscription_adapter/litecable.rb +5 -8
- data/lib/active_job/queue_adapters/litejob_adapter.rb +6 -8
- data/lib/active_record/connection_adapters/litedb_adapter.rb +65 -75
- data/lib/active_support/cache/litecache.rb +38 -41
- data/lib/generators/litestack/install/install_generator.rb +3 -3
- data/lib/generators/litestack/install/templates/database.yml +7 -1
- data/lib/litestack/liteboard/liteboard.rb +269 -149
- data/lib/litestack/litecable.rb +41 -37
- data/lib/litestack/litecable.sql.yml +22 -11
- data/lib/litestack/litecache.rb +79 -88
- data/lib/litestack/litecache.sql.yml +81 -22
- data/lib/litestack/litecache.yml +1 -1
- data/lib/litestack/litedb.rb +35 -40
- data/lib/litestack/litejob.rb +30 -29
- data/lib/litestack/litejobqueue.rb +63 -65
- data/lib/litestack/litemetric.rb +80 -92
- data/lib/litestack/litemetric.sql.yml +244 -234
- data/lib/litestack/litemetric_collector.sql.yml +38 -41
- data/lib/litestack/litequeue.rb +39 -41
- data/lib/litestack/litequeue.sql.yml +39 -31
- data/lib/litestack/litescheduler.rb +15 -15
- data/lib/litestack/litesearch/index.rb +93 -63
- data/lib/litestack/litesearch/model.rb +66 -65
- data/lib/litestack/litesearch/schema.rb +53 -56
- data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +46 -50
- data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +44 -35
- data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +3 -6
- data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +7 -9
- data/lib/litestack/litesearch/schema_adapters.rb +4 -9
- data/lib/litestack/litesearch.rb +6 -9
- data/lib/litestack/litesupport.rb +76 -86
- data/lib/litestack/railtie.rb +1 -1
- data/lib/litestack/version.rb +2 -2
- data/lib/litestack.rb +6 -4
- data/lib/railties/rails/commands/dbconsole.rb +11 -15
- data/lib/sequel/adapters/litedb.rb +16 -21
- data/lib/sequel/adapters/shared/litedb.rb +168 -168
- data/scripts/build_metrics.rb +91 -0
- data/scripts/test_cable.rb +30 -0
- data/scripts/test_job_retry.rb +33 -0
- data/scripts/test_metrics.rb +60 -0
- data/template.rb +2 -2
- metadata +101 -6
@@ -1,37 +1,36 @@
|
|
1
|
-
require_relative
|
1
|
+
require_relative "./schema_adapters"
|
2
2
|
|
3
3
|
class Litesearch::Schema
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
trigram: 'trigram'
|
4
|
+
TOKENIZERS = {
|
5
|
+
porter: "porter unicode61 remove_diacritics 2",
|
6
|
+
unicode: "unicode61 remove_diacritics 2",
|
7
|
+
ascii: "ascii",
|
8
|
+
trigram: "trigram"
|
10
9
|
}
|
11
|
-
|
10
|
+
|
12
11
|
INDEX_TYPES = {
|
13
|
-
standalone: Litesearch::Schema::StandaloneAdapter,
|
14
|
-
contentless: Litesearch::Schema::ContentlessAdapter,
|
12
|
+
standalone: Litesearch::Schema::StandaloneAdapter,
|
13
|
+
contentless: Litesearch::Schema::ContentlessAdapter,
|
15
14
|
backed: Litesearch::Schema::BackedAdapter
|
16
15
|
}
|
17
16
|
|
18
17
|
DEFAULT_SCHEMA = {
|
19
18
|
name: nil,
|
20
19
|
type: :standalone,
|
21
|
-
fields: nil,
|
20
|
+
fields: nil,
|
22
21
|
table: nil,
|
23
22
|
filter_column: nil,
|
24
23
|
tokenizer: :porter,
|
25
24
|
auto_create: true,
|
26
25
|
auto_modify: true,
|
27
|
-
rebuild_on_create: false,
|
28
|
-
rebuild_on_modify: false
|
26
|
+
rebuild_on_create: false,
|
27
|
+
rebuild_on_modify: false
|
29
28
|
}
|
30
29
|
|
31
30
|
attr_accessor :schema
|
32
31
|
|
33
32
|
def initialize(schema = {})
|
34
|
-
@schema = schema #DEFAULT_SCHEMA.merge(schema)
|
33
|
+
@schema = schema # DEFAULT_SCHEMA.merge(schema)
|
35
34
|
@schema[:fields] = {} unless @schema[:fields]
|
36
35
|
end
|
37
36
|
|
@@ -39,7 +38,7 @@ class Litesearch::Schema
|
|
39
38
|
def name(new_name)
|
40
39
|
@schema[:name] = new_name
|
41
40
|
end
|
42
|
-
|
41
|
+
|
43
42
|
def type(new_type)
|
44
43
|
raise "Unknown index type" if INDEX_TYPES[new_type].nil?
|
45
44
|
@schema[:type] = new_type
|
@@ -50,38 +49,38 @@ class Litesearch::Schema
|
|
50
49
|
end
|
51
50
|
|
52
51
|
def fields(field_names)
|
53
|
-
field_names.each {|f| field f }
|
52
|
+
field_names.each { |f| field f }
|
54
53
|
end
|
55
|
-
|
54
|
+
|
56
55
|
def field(name, attributes = {})
|
57
56
|
name = name.to_s.downcase.to_sym
|
58
|
-
attributes = {weight: 1}.merge(attributes).select{|k, v| allowed_attributes.include?(k)} # only allow attributes we know, to ease schema comparison later
|
57
|
+
attributes = {weight: 1}.merge(attributes).select { |k, v| allowed_attributes.include?(k) } # only allow attributes we know, to ease schema comparison later
|
59
58
|
@schema[:fields][name] = attributes
|
60
59
|
end
|
61
|
-
|
60
|
+
|
62
61
|
def tokenizer(new_tokenizer)
|
63
62
|
raise "Unknown tokenizer" if TOKENIZERS[new_tokenizer].nil?
|
64
|
-
@schema[:tokenizer] = new_tokenizer
|
63
|
+
@schema[:tokenizer] = new_tokenizer
|
65
64
|
end
|
66
|
-
|
65
|
+
|
67
66
|
def filter_column(filter_column)
|
68
67
|
@schema[:filter_column] = filter_column
|
69
68
|
end
|
70
|
-
|
69
|
+
|
71
70
|
def auto_create(boolean)
|
72
71
|
@schema[:auto_create] = boolean
|
73
72
|
end
|
74
|
-
|
73
|
+
|
75
74
|
def auto_modify(boolean)
|
76
75
|
@schema[:auto_modify] = boolean
|
77
76
|
end
|
78
|
-
|
77
|
+
|
79
78
|
def rebuild_on_create(boolean)
|
80
|
-
@schema[:rebuild_on_create] = boolean
|
79
|
+
@schema[:rebuild_on_create] = boolean
|
81
80
|
end
|
82
|
-
|
81
|
+
|
83
82
|
def rebuild_on_modify(boolean)
|
84
|
-
@schema[:rebuild_on_modify] = boolean
|
83
|
+
@schema[:rebuild_on_modify] = boolean
|
85
84
|
end
|
86
85
|
|
87
86
|
def post_init
|
@@ -89,51 +88,49 @@ class Litesearch::Schema
|
|
89
88
|
end
|
90
89
|
|
91
90
|
# schema sql generation API
|
92
|
-
|
91
|
+
|
93
92
|
def sql_for(method, *args)
|
94
93
|
adapter.sql_for(method, *args)
|
95
94
|
end
|
96
|
-
|
95
|
+
|
97
96
|
# schema data structure API
|
98
97
|
def get(key)
|
99
98
|
@schema[key]
|
100
|
-
end
|
101
|
-
|
99
|
+
end
|
100
|
+
|
102
101
|
def get_field(name)
|
103
102
|
@schema[:fields][name]
|
104
103
|
end
|
105
|
-
|
104
|
+
|
106
105
|
def adapter
|
107
106
|
@adapter ||= INDEX_TYPES[@schema[:type]].new(@schema)
|
108
107
|
end
|
109
|
-
|
108
|
+
|
110
109
|
def reset_sql
|
111
110
|
adapter.generate_sql
|
112
111
|
end
|
113
|
-
|
112
|
+
|
114
113
|
def order_fields(old_schema)
|
115
114
|
adapter.order_fields(old_schema)
|
116
115
|
end
|
117
|
-
|
116
|
+
|
118
117
|
# should we do this at the schema objects level?
|
119
118
|
def compare(other_schema)
|
120
119
|
other_schema = other_schema.schema
|
121
120
|
# are the schemas identical?
|
122
121
|
# 1 - same fields?
|
123
122
|
[:type, :tokenizer, :name, :table].each do |key|
|
124
|
-
|
123
|
+
other_schema[key] = @schema[key] if other_schema[key].nil?
|
125
124
|
end
|
126
125
|
if @schema[:type] != other_schema[:type]
|
127
126
|
raise Litesearch::SchemaChangeException.new "Cannot change the index type, please drop the index before creating it again with the new type"
|
128
127
|
end
|
129
|
-
changes = {
|
130
|
-
#check tokenizer changes
|
128
|
+
changes = {tokenizer: @schema[:tokenizer] != other_schema[:tokenizer], table: @schema[:table] != other_schema[:table], removed_fields_count: 0, filter_column: @schema[:filter_column] != other_schema[:filter_column]}
|
129
|
+
# check tokenizer changes
|
131
130
|
if changes[:tokenizer] && !other_schema[:rebuild_on_modify]
|
132
131
|
raise Litesearch::SchemaChangeException.new "Cannot change the tokenizer without an index rebuild!"
|
133
132
|
end
|
134
133
|
|
135
|
-
|
136
|
-
|
137
134
|
# check field changes
|
138
135
|
keys = @schema[:fields].keys.sort
|
139
136
|
other_keys = other_schema[:fields].keys.sort
|
@@ -144,50 +141,50 @@ class Litesearch::Schema
|
|
144
141
|
other_schema[:fields].delete(key)
|
145
142
|
end
|
146
143
|
end
|
147
|
-
|
144
|
+
|
148
145
|
other_keys = other_schema[:fields].keys.sort
|
149
146
|
|
150
147
|
changes[:fields] = keys != other_keys # only acceptable change is adding extra fields
|
151
|
-
changes[:extra_fields_count] = other_keys.count - keys.count
|
148
|
+
changes[:extra_fields_count] = other_keys.count - keys.count
|
152
149
|
# check for missing fields (please note that adding fields can work without a rebuild)
|
153
150
|
if keys - other_keys != []
|
154
|
-
raise Litesearch::SchemaChangeException.new "Missing fields from existing schema, they have to exist with weight zero until the next rebuild!"
|
151
|
+
raise Litesearch::SchemaChangeException.new "Missing fields from existing schema, they have to exist with weight zero until the next rebuild!"
|
155
152
|
end
|
156
|
-
|
153
|
+
|
157
154
|
# check field weights
|
158
|
-
weights = keys.collect{|key| @schema[:fields][key][:weight] }
|
159
|
-
other_weights = other_keys.collect{|key| other_schema[:fields][key][:weight] }
|
155
|
+
weights = keys.collect { |key| @schema[:fields][key][:weight] }
|
156
|
+
other_weights = other_keys.collect { |key| other_schema[:fields][key][:weight] }
|
160
157
|
changes[:weights] = weights != other_weights # will always be true if fields are added
|
161
|
-
if (removed_count = other_weights.
|
158
|
+
if (removed_count = other_weights.count { |w| w == 0 }) > 0
|
162
159
|
changes[:removed_fields_count] = removed_count
|
163
160
|
end
|
164
161
|
# check field attributes, only backed tables have attributes
|
165
|
-
attrs = keys.collect do |key|
|
162
|
+
attrs = keys.collect do |key|
|
166
163
|
f = @schema[:fields][key].dup
|
167
164
|
f.delete(:weight)
|
168
|
-
f.select{|k,v| allowed_attributes.include? k }
|
165
|
+
f.select { |k, v| allowed_attributes.include? k }
|
169
166
|
end
|
170
167
|
other_attrs = other_keys.collect do |key|
|
171
168
|
f = other_schema[:fields][key].dup
|
172
169
|
f.delete(:weight)
|
173
|
-
f.select{|k,v| allowed_attributes.include? k }
|
170
|
+
f.select { |k, v| allowed_attributes.include? k }
|
174
171
|
end
|
175
172
|
changes[:attributes] if other_attrs != attrs # this means that we will need to redefine the triggers if any are there and also the table definition if needed
|
176
|
-
|
173
|
+
|
177
174
|
# return the changes
|
178
175
|
changes
|
179
176
|
end
|
180
|
-
|
177
|
+
|
181
178
|
def clean
|
182
|
-
removable = @schema[:fields].select{|name, f| f[:weight] == 0 }.collect{|name, f| name}
|
183
|
-
removable.each{|name| @schema[:fields].delete(name)}
|
179
|
+
removable = @schema[:fields].select { |name, f| f[:weight] == 0 }.collect { |name, f| name }
|
180
|
+
removable.each { |name| @schema[:fields].delete(name) }
|
184
181
|
end
|
185
|
-
|
182
|
+
|
186
183
|
def allowed_attributes
|
187
184
|
[:weight, :col, :target]
|
188
185
|
end
|
189
|
-
|
190
186
|
end
|
191
187
|
|
192
188
|
class Litesearch::SchemaException < StandardError; end
|
189
|
+
|
193
190
|
class Litesearch::SchemaChangeException < StandardError; end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
class Litesearch::Schema::BackedAdapter < Litesearch::Schema::ContentlessAdapter
|
2
|
-
|
3
2
|
private
|
4
3
|
|
5
4
|
def table
|
@@ -14,50 +13,50 @@ class Litesearch::Schema::BackedAdapter < Litesearch::Schema::ContentlessAdapter
|
|
14
13
|
@sql[:create_primary_triggers] = :create_primary_triggers_sql
|
15
14
|
@sql[:create_secondary_triggers] = :create_secondary_triggers_sql
|
16
15
|
end
|
17
|
-
|
16
|
+
|
18
17
|
def drop_primary_triggers_sql
|
19
|
-
|
20
|
-
DROP TRIGGER IF EXISTS #{name}_insert;
|
21
|
-
DROP TRIGGER IF EXISTS #{name}_update;
|
22
|
-
DROP TRIGGER IF EXISTS #{name}_update_not;
|
23
|
-
DROP TRIGGER IF EXISTS #{name}_delete;
|
24
|
-
SQL
|
18
|
+
<<~SQL
|
19
|
+
DROP TRIGGER IF EXISTS #{name}_insert;
|
20
|
+
DROP TRIGGER IF EXISTS #{name}_update;
|
21
|
+
DROP TRIGGER IF EXISTS #{name}_update_not;
|
22
|
+
DROP TRIGGER IF EXISTS #{name}_delete;
|
23
|
+
SQL
|
25
24
|
end
|
26
|
-
|
27
|
-
def create_primary_triggers_sql(active=false)
|
25
|
+
|
26
|
+
def create_primary_triggers_sql(active = false)
|
28
27
|
when_stmt = "TRUE"
|
29
28
|
cols = active_cols_names
|
30
|
-
if filter = @schema[:filter_column]
|
29
|
+
if (filter = @schema[:filter_column])
|
31
30
|
when_stmt = "NEW.#{filter} = TRUE"
|
32
31
|
cols << filter
|
33
32
|
end
|
34
33
|
|
35
|
-
|
34
|
+
<<-SQL
|
36
35
|
CREATE TRIGGER #{name}_insert AFTER INSERT ON #{table} WHEN #{when_stmt} BEGIN
|
37
|
-
INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(
|
36
|
+
INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) VALUES (NEW.rowid, #{trigger_cols_sql});
|
38
37
|
END;
|
39
|
-
CREATE TRIGGER #{name}_update AFTER UPDATE OF #{cols.join(
|
40
|
-
INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(
|
38
|
+
CREATE TRIGGER #{name}_update AFTER UPDATE OF #{cols.join(", ")} ON #{table} WHEN #{when_stmt} BEGIN
|
39
|
+
INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) VALUES (NEW.rowid, #{trigger_cols_sql});
|
41
40
|
END;
|
42
|
-
CREATE TRIGGER #{name}_update_not AFTER UPDATE OF #{cols.join(
|
41
|
+
CREATE TRIGGER #{name}_update_not AFTER UPDATE OF #{cols.join(", ")} ON #{table} WHEN NOT #{when_stmt} BEGIN
|
43
42
|
DELETE FROM #{name} WHERE rowid = NEW.rowid;
|
44
43
|
END;
|
45
44
|
CREATE TRIGGER #{name}_delete AFTER DELETE ON #{table} BEGIN
|
46
45
|
DELETE FROM #{name} WHERE rowid = OLD.id;
|
47
46
|
END;
|
48
|
-
SQL
|
47
|
+
SQL
|
49
48
|
end
|
50
|
-
|
49
|
+
|
51
50
|
def drop_secondary_trigger_sql(target_table, target_col, col)
|
52
51
|
"DROP TRIGGER IF EXISTS #{target_table}_#{target_col}_#{col}_#{name}_update;"
|
53
52
|
end
|
54
|
-
|
53
|
+
|
55
54
|
def create_secondary_trigger_sql(target_table, target_col, col)
|
56
|
-
|
57
|
-
CREATE TRIGGER #{target_table}_#{target_col}_#{col}_#{name}_update AFTER UPDATE OF #{target_col} ON #{target_table} BEGIN
|
58
|
-
|
59
|
-
END;
|
60
|
-
SQL
|
55
|
+
<<~SQL
|
56
|
+
CREATE TRIGGER #{target_table}_#{target_col}_#{col}_#{name}_update AFTER UPDATE OF #{target_col} ON #{target_table} BEGIN
|
57
|
+
#{rebuild_sql} AND #{table}.#{col} = NEW.id;
|
58
|
+
END;
|
59
|
+
SQL
|
61
60
|
end
|
62
61
|
|
63
62
|
def drop_secondary_triggers_sql
|
@@ -67,36 +66,34 @@ SQL
|
|
67
66
|
sql << drop_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
|
68
67
|
end
|
69
68
|
end
|
70
|
-
|
69
|
+
sql.empty? ? nil : sql
|
71
70
|
end
|
72
71
|
|
73
|
-
|
74
72
|
def create_secondary_triggers_sql
|
75
|
-
sql = ""
|
73
|
+
sql = ""
|
76
74
|
@schema[:fields].each do |name, field|
|
77
75
|
if field[:trigger_sql]
|
78
76
|
sql << create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
|
79
77
|
end
|
80
78
|
end
|
81
|
-
|
79
|
+
sql.empty? ? nil : sql
|
82
80
|
end
|
83
|
-
|
84
|
-
|
81
|
+
|
85
82
|
def rebuild_sql
|
86
|
-
conditions = ""
|
83
|
+
conditions = ""
|
87
84
|
jcs = join_conditions_sql
|
88
85
|
fs = filter_sql
|
89
86
|
conditions = " ON #{jcs} #{fs}" unless jcs.empty? && fs.empty?
|
90
|
-
"INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(
|
87
|
+
"INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) SELECT #{table}.id, #{select_cols_sql} FROM #{join_tables_sql} #{conditions}"
|
91
88
|
end
|
92
|
-
|
89
|
+
|
93
90
|
def enrich_schema
|
94
|
-
@schema[:fields].each do |name, field|
|
95
|
-
if field[:target] && !
|
91
|
+
@schema[:fields].each do |name, field|
|
92
|
+
if field[:target] && !field[:target].start_with?("#{table}.")
|
96
93
|
field[:target] = field[:target].downcase
|
97
|
-
target_table, target_col = field[:target].split(
|
94
|
+
target_table, target_col = field[:target].split(".")
|
98
95
|
field[:col] = "#{name}_id".to_sym unless field[:col]
|
99
|
-
field[:target_table] = target_table.to_sym
|
96
|
+
field[:target_table] = target_table.to_sym
|
100
97
|
field[:target_col] = target_col.to_sym
|
101
98
|
field[:sql] = "(SELECT #{field[:target_col]} FROM #{field[:target_table]} WHERE id = NEW.#{field[:col]})"
|
102
99
|
field[:trigger_sql] = true # create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
|
@@ -109,9 +106,9 @@ SQL
|
|
109
106
|
end
|
110
107
|
end
|
111
108
|
end
|
112
|
-
|
109
|
+
|
113
110
|
def filter_sql
|
114
|
-
sql = ""
|
111
|
+
sql = ""
|
115
112
|
sql << " AND #{@schema[:filter_column]} = TRUE " if @schema[:filter_column]
|
116
113
|
sql
|
117
114
|
end
|
@@ -121,27 +118,26 @@ SQL
|
|
121
118
|
field[:trigger_sql] ? field[:sql] : "NEW.#{field[:sql]}"
|
122
119
|
end.join(", ")
|
123
120
|
end
|
124
|
-
|
121
|
+
|
125
122
|
def select_cols_sql
|
126
123
|
active_fields.collect do |name, field|
|
127
|
-
field[:trigger_sql]
|
128
|
-
end.join(
|
124
|
+
(!field[:trigger_sql].nil?) ? "#{field[:target_table_alias]}.#{field[:target_col]}" : field[:target]
|
125
|
+
end.join(", ")
|
129
126
|
end
|
130
|
-
|
127
|
+
|
131
128
|
def join_tables_sql
|
132
|
-
tables = [@schema[:table]]
|
129
|
+
tables = [@schema[:table]]
|
133
130
|
active_fields.each do |name, field|
|
134
|
-
tables << "#{field[:target_table]} AS #{field[:target_table_alias]}" if field[:trigger_sql]
|
131
|
+
tables << "#{field[:target_table]} AS #{field[:target_table_alias]}" if field[:trigger_sql]
|
135
132
|
end
|
136
133
|
tables.uniq.join(", ")
|
137
134
|
end
|
138
|
-
|
135
|
+
|
139
136
|
def join_conditions_sql
|
140
137
|
conditions = []
|
141
138
|
active_fields.each do |name, field|
|
142
|
-
conditions << "#{field[:target_table_alias]}.id = #{@schema[:table]}.#{field[:col]}" if field[:trigger_sql]
|
139
|
+
conditions << "#{field[:target_table_alias]}.id = #{@schema[:table]}.#{field[:col]}" if field[:trigger_sql]
|
143
140
|
end
|
144
|
-
conditions.join(" AND ")
|
145
|
-
end
|
146
|
-
|
141
|
+
conditions.join(" AND ")
|
142
|
+
end
|
147
143
|
end
|
@@ -1,16 +1,15 @@
|
|
1
1
|
class Litesearch::Schema::BasicAdapter
|
2
|
-
|
3
2
|
def initialize(schema)
|
4
3
|
@schema = schema
|
5
4
|
@sql = {}
|
6
5
|
enrich_schema
|
7
6
|
generate_sql
|
8
7
|
end
|
9
|
-
|
8
|
+
|
10
9
|
def name
|
11
10
|
@schema[:name]
|
12
11
|
end
|
13
|
-
|
12
|
+
|
14
13
|
def table
|
15
14
|
@schema[:table]
|
16
15
|
end
|
@@ -22,25 +21,25 @@ class Litesearch::Schema::BasicAdapter
|
|
22
21
|
def field_names
|
23
22
|
@schema[:fields].keys
|
24
23
|
end
|
25
|
-
|
24
|
+
|
26
25
|
def active_fields
|
27
|
-
@schema[:fields].select{|k, v| v[:weight] != 0 }
|
26
|
+
@schema[:fields].select { |k, v| v[:weight] != 0 }
|
28
27
|
end
|
29
|
-
|
28
|
+
|
30
29
|
def active_field_names
|
31
30
|
active_fields.keys
|
32
31
|
end
|
33
|
-
|
32
|
+
|
34
33
|
def active_cols_names
|
35
|
-
active_fields.collect{|k, v| v[:col]}
|
34
|
+
active_fields.collect { |k, v| v[:col] }
|
36
35
|
end
|
37
|
-
|
36
|
+
|
38
37
|
def weights
|
39
|
-
@schema[:fields].values.collect{|v| v[:weight].to_f }
|
38
|
+
@schema[:fields].values.collect { |v| v[:weight].to_f }
|
40
39
|
end
|
41
40
|
|
42
41
|
def active_weights
|
43
|
-
active_fields.values.collect{|v| v[:weight].to_f }
|
42
|
+
active_fields.values.collect { |v| v[:weight].to_f }
|
44
43
|
end
|
45
44
|
|
46
45
|
def tokenizer_sql
|
@@ -51,33 +50,34 @@ class Litesearch::Schema::BasicAdapter
|
|
51
50
|
new_fields = {}
|
52
51
|
old_field_names = old_schema.schema[:fields].keys
|
53
52
|
old_field_names.each do |name|
|
54
|
-
new_fields[name] = @schema[:fields].delete(name)
|
53
|
+
new_fields[name] = @schema[:fields].delete(name)
|
55
54
|
end
|
56
55
|
missing_field_names = field_names - old_field_names
|
57
56
|
missing_field_names.each do |name|
|
58
57
|
new_fields[name] = @schema[:fields].delete(name)
|
59
|
-
end
|
58
|
+
end
|
60
59
|
@schema[:fields] = new_fields # this should be in order now
|
61
60
|
generate_sql
|
62
61
|
enrich_schema
|
63
62
|
end
|
64
63
|
|
65
64
|
def sql_for(method, *args)
|
66
|
-
if sql = @sql[method]
|
65
|
+
if (sql = @sql[method])
|
67
66
|
if sql.is_a? String
|
68
|
-
|
67
|
+
sql
|
69
68
|
elsif sql.is_a? Proc
|
70
|
-
|
69
|
+
sql.call(*args)
|
71
70
|
elsif sql.is_a? Symbol
|
72
|
-
|
71
|
+
send(sql, *args)
|
73
72
|
elsif sql.is_a? Litesearch::SchemaChangeException
|
74
73
|
raise sql
|
75
74
|
end
|
76
75
|
end
|
77
76
|
end
|
78
|
-
|
77
|
+
|
79
78
|
def generate_sql
|
80
79
|
@sql[:create_index] = :create_index_sql
|
80
|
+
@sql[:create_vocab_tables] = :create_vocab_tables_sql
|
81
81
|
@sql[:insert] = "INSERT OR REPLACE INTO #{name}(rowid, #{active_col_names_sql}) VALUES (:id, #{active_col_names_var_sql}) RETURNING rowid"
|
82
82
|
@sql[:delete] = "DELETE FROM #{name} WHERE rowid = :id"
|
83
83
|
@sql[:count] = "SELECT count(*) FROM #{name}(:term)"
|
@@ -86,43 +86,52 @@ class Litesearch::Schema::BasicAdapter
|
|
86
86
|
@sql[:drop] = "DROP TABLE #{name}"
|
87
87
|
@sql[:expand_data] = "UPDATE #{name}_data SET block = block || zeroblob(:length) WHERE id = 1"
|
88
88
|
@sql[:expand_docsize] = "UPDATE #{name}_docsize SET sz = sz || zeroblob(:length)"
|
89
|
-
@sql[:ranks] = :ranks_sql
|
89
|
+
@sql[:ranks] = :ranks_sql
|
90
90
|
@sql[:set_config_value] = "INSERT OR REPLACE INTO #{name}_config(k, v) VALUES (:key, :value)"
|
91
91
|
@sql[:get_config_value] = "SELECT v FROM #{name}_config WHERE k = :key"
|
92
92
|
@sql[:search] = "SELECT rowid AS id, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
|
93
|
+
@sql[:similarity_terms] = "SELECT DISTINCT term FROM #{name}_instance WHERE doc = :id AND FLOOR(term) IS NULL AND LENGTH(term) > 2 AND NOT instr(term, ' ') AND NOT instr(term, '-') AND NOT instr(term, ':') AND NOT instr(term, '#') AND NOT instr(term, '_') LIMIT 15"
|
94
|
+
@sql[:similarity_query] = "SELECT group_concat('\"' || term || '\"', ' OR ') FROM #{name}_row WHERE term IN (#{@sql[:similarity_terms]})"
|
95
|
+
@sql[:similarity_search] = "SELECT rowid AS id, -rank AS search_rank FROM #{name}(:term) WHERE rowid != :id ORDER BY rank LIMIT :limit"
|
96
|
+
@sql[:similar] = "SELECT rowid AS id, -rank AS search_rank FROM #{name} WHERE #{name} = (#{@sql[:similarity_query]}) AND rowid != :id ORDER BY rank LIMIT :limit"
|
93
97
|
@sql[:update_index] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}'"
|
94
98
|
@sql[:update_content_table] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}_content'"
|
95
99
|
end
|
96
|
-
|
100
|
+
|
97
101
|
private
|
98
|
-
|
99
|
-
def
|
100
|
-
|
101
|
-
|
102
|
+
|
103
|
+
def create_vocab_tables_sql
|
104
|
+
<<~SQL
|
105
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS #{name}_row USING fts5vocab(#{name}, row);
|
106
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS #{name}_instance USING fts5vocab(#{name}, instance);
|
107
|
+
SQL
|
108
|
+
end
|
109
|
+
|
110
|
+
def ranks_sql(active = false)
|
111
|
+
weights_sql = if active
|
112
|
+
weights.join(", ")
|
102
113
|
else
|
103
|
-
|
114
|
+
active_weights.join(", ")
|
104
115
|
end
|
105
116
|
"INSERT INTO #{name}(#{name}, rank) VALUES ('rank', 'bm25(#{weights_sql})')"
|
106
117
|
end
|
107
|
-
|
118
|
+
|
108
119
|
def active_col_names_sql
|
109
|
-
active_field_names.join(
|
120
|
+
active_field_names.join(", ")
|
110
121
|
end
|
111
|
-
|
122
|
+
|
112
123
|
def active_col_names_var_sql
|
113
|
-
":#{active_field_names.join(
|
124
|
+
":#{active_field_names.join(", :")}"
|
114
125
|
end
|
115
|
-
|
126
|
+
|
116
127
|
def col_names_sql
|
117
|
-
field_names.join(
|
128
|
+
field_names.join(", ")
|
118
129
|
end
|
119
|
-
|
130
|
+
|
120
131
|
def col_names_var_sql
|
121
|
-
":#{field_names.join(
|
132
|
+
":#{field_names.join(", :")}"
|
122
133
|
end
|
123
134
|
|
124
135
|
def enrich_schema
|
125
136
|
end
|
126
|
-
|
127
137
|
end
|
128
|
-
|
@@ -1,17 +1,14 @@
|
|
1
1
|
class Litesearch::Schema::ContentlessAdapter < Litesearch::Schema::BasicAdapter
|
2
|
-
|
3
2
|
private
|
4
3
|
|
5
4
|
def generate_sql
|
6
5
|
super
|
7
|
-
|
8
|
-
|
6
|
+
# @sql[:rebuild_index] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
|
7
|
+
# @sql[:rebuild] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
|
9
8
|
end
|
10
|
-
|
9
|
+
|
11
10
|
def create_index_sql(active = false)
|
12
11
|
col_names = active ? active_col_names_sql : col_names_sql
|
13
12
|
"CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, content='', contentless_delete=1, tokenize='#{tokenizer_sql}')"
|
14
13
|
end
|
15
|
-
|
16
14
|
end
|
17
|
-
|
@@ -1,14 +1,14 @@
|
|
1
1
|
class Litesearch::Schema::StandaloneAdapter < Litesearch::Schema::BasicAdapter
|
2
|
-
|
3
2
|
def generate_sql
|
4
3
|
super
|
5
4
|
@sql[:move_content] = "ALTER TABLE #{name}_content RENAME TO #{name}_content_temp"
|
6
5
|
@sql[:adjust_temp_content] = "UPDATE sqlite_schema SET sql (SELECT sql FROM sqlite_schema WHERE name = '#{name}_content') WHERE name = #{name}_content_temp"
|
7
6
|
@sql[:restore_content] = "ALTER TABLE #{name}_content_temp RENAME TO #{name}_content"
|
8
7
|
@sql[:rebuild] = "INSERT INTO #{name}(#{name}) VALUES ('rebuild')"
|
8
|
+
@sql[:similar] = "SELECT rowid AS id, *, -rank AS search_rank FROM #{name} WHERE #{name} = (#{@sql[:similarity_query]}) AND rowid != :id ORDER BY rank LIMIT :limit"
|
9
9
|
@sql[:drop_content_table] = "DROP TABLE #{name}_content"
|
10
10
|
@sql[:drop_content_col] = :drop_content_col_sql
|
11
|
-
@sql[:create_content_table] = :create_content_table_sql
|
11
|
+
@sql[:create_content_table] = :create_content_table_sql
|
12
12
|
@sql[:search] = "SELECT rowid AS id, *, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
|
13
13
|
end
|
14
14
|
|
@@ -18,16 +18,14 @@ class Litesearch::Schema::StandaloneAdapter < Litesearch::Schema::BasicAdapter
|
|
18
18
|
col_names = active ? active_col_names_sql : col_names_sql
|
19
19
|
"CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, tokenize='#{tokenizer_sql}')"
|
20
20
|
end
|
21
|
-
|
21
|
+
|
22
22
|
def drop_content_col_sql(col_index)
|
23
23
|
"ALTER TABLE #{name}_content DROP COLUMN c#{col_index}"
|
24
24
|
end
|
25
|
-
|
25
|
+
|
26
26
|
def create_content_table_sql(count)
|
27
|
-
cols = []
|
28
|
-
count.times{|i| cols << "c#{i}" }
|
29
|
-
"CREATE TABLE #{name}_content(id INTEGER PRIMARY KEY, #{cols.join(
|
27
|
+
cols = []
|
28
|
+
count.times { |i| cols << "c#{i}" }
|
29
|
+
"CREATE TABLE #{name}_content(id INTEGER PRIMARY KEY, #{cols.join(", ")})"
|
30
30
|
end
|
31
|
-
|
32
31
|
end
|
33
|
-
|
@@ -1,9 +1,4 @@
|
|
1
|
-
require_relative
|
2
|
-
require_relative
|
3
|
-
require_relative
|
4
|
-
require_relative
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
1
|
+
require_relative "./schema_adapters/basic_adapter"
|
2
|
+
require_relative "./schema_adapters/standalone_adapter"
|
3
|
+
require_relative "./schema_adapters/contentless_adapter"
|
4
|
+
require_relative "./schema_adapters/backed_adapter"
|