litestack 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.standard.yml +3 -0
- data/BENCHMARKS.md +23 -7
- data/CHANGELOG.md +11 -0
- data/Gemfile +1 -7
- data/Gemfile.lock +92 -0
- data/README.md +120 -6
- data/ROADMAP.md +45 -0
- data/Rakefile +3 -1
- data/WHYLITESTACK.md +1 -1
- data/assets/litecache_metrics.png +0 -0
- data/assets/litedb_metrics.png +0 -0
- data/assets/litemetric_logo_teal.png +0 -0
- data/assets/litesearch_logo_teal.png +0 -0
- data/bench/bench.rb +17 -10
- data/bench/bench_cache_rails.rb +10 -13
- data/bench/bench_cache_raw.rb +17 -22
- data/bench/bench_jobs_rails.rb +18 -12
- data/bench/bench_jobs_raw.rb +17 -10
- data/bench/bench_queue.rb +4 -6
- data/bench/rails_job.rb +5 -7
- data/bench/skjob.rb +4 -4
- data/bench/uljob.rb +6 -6
- data/lib/action_cable/subscription_adapter/litecable.rb +5 -8
- data/lib/active_job/queue_adapters/litejob_adapter.rb +6 -8
- data/lib/active_record/connection_adapters/litedb_adapter.rb +65 -75
- data/lib/active_support/cache/litecache.rb +38 -41
- data/lib/generators/litestack/install/install_generator.rb +3 -3
- data/lib/generators/litestack/install/templates/database.yml +7 -1
- data/lib/litestack/liteboard/liteboard.rb +269 -149
- data/lib/litestack/litecable.rb +41 -37
- data/lib/litestack/litecable.sql.yml +22 -11
- data/lib/litestack/litecache.rb +79 -88
- data/lib/litestack/litecache.sql.yml +81 -22
- data/lib/litestack/litecache.yml +1 -1
- data/lib/litestack/litedb.rb +35 -40
- data/lib/litestack/litejob.rb +30 -29
- data/lib/litestack/litejobqueue.rb +63 -65
- data/lib/litestack/litemetric.rb +80 -92
- data/lib/litestack/litemetric.sql.yml +244 -234
- data/lib/litestack/litemetric_collector.sql.yml +38 -41
- data/lib/litestack/litequeue.rb +39 -41
- data/lib/litestack/litequeue.sql.yml +39 -31
- data/lib/litestack/litescheduler.rb +15 -15
- data/lib/litestack/litesearch/index.rb +93 -63
- data/lib/litestack/litesearch/model.rb +66 -65
- data/lib/litestack/litesearch/schema.rb +53 -56
- data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +46 -50
- data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +44 -35
- data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +3 -6
- data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +7 -9
- data/lib/litestack/litesearch/schema_adapters.rb +4 -9
- data/lib/litestack/litesearch.rb +6 -9
- data/lib/litestack/litesupport.rb +76 -86
- data/lib/litestack/railtie.rb +1 -1
- data/lib/litestack/version.rb +2 -2
- data/lib/litestack.rb +6 -4
- data/lib/railties/rails/commands/dbconsole.rb +11 -15
- data/lib/sequel/adapters/litedb.rb +16 -21
- data/lib/sequel/adapters/shared/litedb.rb +168 -168
- data/scripts/build_metrics.rb +91 -0
- data/scripts/test_cable.rb +30 -0
- data/scripts/test_job_retry.rb +33 -0
- data/scripts/test_metrics.rb +60 -0
- data/template.rb +2 -2
- metadata +101 -6
@@ -1,37 +1,36 @@
|
|
1
|
-
require_relative
|
1
|
+
require_relative "./schema_adapters"
|
2
2
|
|
3
3
|
class Litesearch::Schema
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
trigram: 'trigram'
|
4
|
+
TOKENIZERS = {
|
5
|
+
porter: "porter unicode61 remove_diacritics 2",
|
6
|
+
unicode: "unicode61 remove_diacritics 2",
|
7
|
+
ascii: "ascii",
|
8
|
+
trigram: "trigram"
|
10
9
|
}
|
11
|
-
|
10
|
+
|
12
11
|
INDEX_TYPES = {
|
13
|
-
standalone: Litesearch::Schema::StandaloneAdapter,
|
14
|
-
contentless: Litesearch::Schema::ContentlessAdapter,
|
12
|
+
standalone: Litesearch::Schema::StandaloneAdapter,
|
13
|
+
contentless: Litesearch::Schema::ContentlessAdapter,
|
15
14
|
backed: Litesearch::Schema::BackedAdapter
|
16
15
|
}
|
17
16
|
|
18
17
|
DEFAULT_SCHEMA = {
|
19
18
|
name: nil,
|
20
19
|
type: :standalone,
|
21
|
-
fields: nil,
|
20
|
+
fields: nil,
|
22
21
|
table: nil,
|
23
22
|
filter_column: nil,
|
24
23
|
tokenizer: :porter,
|
25
24
|
auto_create: true,
|
26
25
|
auto_modify: true,
|
27
|
-
rebuild_on_create: false,
|
28
|
-
rebuild_on_modify: false
|
26
|
+
rebuild_on_create: false,
|
27
|
+
rebuild_on_modify: false
|
29
28
|
}
|
30
29
|
|
31
30
|
attr_accessor :schema
|
32
31
|
|
33
32
|
def initialize(schema = {})
|
34
|
-
@schema = schema #DEFAULT_SCHEMA.merge(schema)
|
33
|
+
@schema = schema # DEFAULT_SCHEMA.merge(schema)
|
35
34
|
@schema[:fields] = {} unless @schema[:fields]
|
36
35
|
end
|
37
36
|
|
@@ -39,7 +38,7 @@ class Litesearch::Schema
|
|
39
38
|
def name(new_name)
|
40
39
|
@schema[:name] = new_name
|
41
40
|
end
|
42
|
-
|
41
|
+
|
43
42
|
def type(new_type)
|
44
43
|
raise "Unknown index type" if INDEX_TYPES[new_type].nil?
|
45
44
|
@schema[:type] = new_type
|
@@ -50,38 +49,38 @@ class Litesearch::Schema
|
|
50
49
|
end
|
51
50
|
|
52
51
|
def fields(field_names)
|
53
|
-
field_names.each {|f| field f }
|
52
|
+
field_names.each { |f| field f }
|
54
53
|
end
|
55
|
-
|
54
|
+
|
56
55
|
def field(name, attributes = {})
|
57
56
|
name = name.to_s.downcase.to_sym
|
58
|
-
attributes = {weight: 1}.merge(attributes).select{|k, v| allowed_attributes.include?(k)} # only allow attributes we know, to ease schema comparison later
|
57
|
+
attributes = {weight: 1}.merge(attributes).select { |k, v| allowed_attributes.include?(k) } # only allow attributes we know, to ease schema comparison later
|
59
58
|
@schema[:fields][name] = attributes
|
60
59
|
end
|
61
|
-
|
60
|
+
|
62
61
|
def tokenizer(new_tokenizer)
|
63
62
|
raise "Unknown tokenizer" if TOKENIZERS[new_tokenizer].nil?
|
64
|
-
@schema[:tokenizer] = new_tokenizer
|
63
|
+
@schema[:tokenizer] = new_tokenizer
|
65
64
|
end
|
66
|
-
|
65
|
+
|
67
66
|
def filter_column(filter_column)
|
68
67
|
@schema[:filter_column] = filter_column
|
69
68
|
end
|
70
|
-
|
69
|
+
|
71
70
|
def auto_create(boolean)
|
72
71
|
@schema[:auto_create] = boolean
|
73
72
|
end
|
74
|
-
|
73
|
+
|
75
74
|
def auto_modify(boolean)
|
76
75
|
@schema[:auto_modify] = boolean
|
77
76
|
end
|
78
|
-
|
77
|
+
|
79
78
|
def rebuild_on_create(boolean)
|
80
|
-
@schema[:rebuild_on_create] = boolean
|
79
|
+
@schema[:rebuild_on_create] = boolean
|
81
80
|
end
|
82
|
-
|
81
|
+
|
83
82
|
def rebuild_on_modify(boolean)
|
84
|
-
@schema[:rebuild_on_modify] = boolean
|
83
|
+
@schema[:rebuild_on_modify] = boolean
|
85
84
|
end
|
86
85
|
|
87
86
|
def post_init
|
@@ -89,51 +88,49 @@ class Litesearch::Schema
|
|
89
88
|
end
|
90
89
|
|
91
90
|
# schema sql generation API
|
92
|
-
|
91
|
+
|
93
92
|
def sql_for(method, *args)
|
94
93
|
adapter.sql_for(method, *args)
|
95
94
|
end
|
96
|
-
|
95
|
+
|
97
96
|
# schema data structure API
|
98
97
|
def get(key)
|
99
98
|
@schema[key]
|
100
|
-
end
|
101
|
-
|
99
|
+
end
|
100
|
+
|
102
101
|
def get_field(name)
|
103
102
|
@schema[:fields][name]
|
104
103
|
end
|
105
|
-
|
104
|
+
|
106
105
|
def adapter
|
107
106
|
@adapter ||= INDEX_TYPES[@schema[:type]].new(@schema)
|
108
107
|
end
|
109
|
-
|
108
|
+
|
110
109
|
def reset_sql
|
111
110
|
adapter.generate_sql
|
112
111
|
end
|
113
|
-
|
112
|
+
|
114
113
|
def order_fields(old_schema)
|
115
114
|
adapter.order_fields(old_schema)
|
116
115
|
end
|
117
|
-
|
116
|
+
|
118
117
|
# should we do this at the schema objects level?
|
119
118
|
def compare(other_schema)
|
120
119
|
other_schema = other_schema.schema
|
121
120
|
# are the schemas identical?
|
122
121
|
# 1 - same fields?
|
123
122
|
[:type, :tokenizer, :name, :table].each do |key|
|
124
|
-
|
123
|
+
other_schema[key] = @schema[key] if other_schema[key].nil?
|
125
124
|
end
|
126
125
|
if @schema[:type] != other_schema[:type]
|
127
126
|
raise Litesearch::SchemaChangeException.new "Cannot change the index type, please drop the index before creating it again with the new type"
|
128
127
|
end
|
129
|
-
changes = {
|
130
|
-
#check tokenizer changes
|
128
|
+
changes = {tokenizer: @schema[:tokenizer] != other_schema[:tokenizer], table: @schema[:table] != other_schema[:table], removed_fields_count: 0, filter_column: @schema[:filter_column] != other_schema[:filter_column]}
|
129
|
+
# check tokenizer changes
|
131
130
|
if changes[:tokenizer] && !other_schema[:rebuild_on_modify]
|
132
131
|
raise Litesearch::SchemaChangeException.new "Cannot change the tokenizer without an index rebuild!"
|
133
132
|
end
|
134
133
|
|
135
|
-
|
136
|
-
|
137
134
|
# check field changes
|
138
135
|
keys = @schema[:fields].keys.sort
|
139
136
|
other_keys = other_schema[:fields].keys.sort
|
@@ -144,50 +141,50 @@ class Litesearch::Schema
|
|
144
141
|
other_schema[:fields].delete(key)
|
145
142
|
end
|
146
143
|
end
|
147
|
-
|
144
|
+
|
148
145
|
other_keys = other_schema[:fields].keys.sort
|
149
146
|
|
150
147
|
changes[:fields] = keys != other_keys # only acceptable change is adding extra fields
|
151
|
-
changes[:extra_fields_count] = other_keys.count - keys.count
|
148
|
+
changes[:extra_fields_count] = other_keys.count - keys.count
|
152
149
|
# check for missing fields (please note that adding fields can work without a rebuild)
|
153
150
|
if keys - other_keys != []
|
154
|
-
raise Litesearch::SchemaChangeException.new "Missing fields from existing schema, they have to exist with weight zero until the next rebuild!"
|
151
|
+
raise Litesearch::SchemaChangeException.new "Missing fields from existing schema, they have to exist with weight zero until the next rebuild!"
|
155
152
|
end
|
156
|
-
|
153
|
+
|
157
154
|
# check field weights
|
158
|
-
weights = keys.collect{|key| @schema[:fields][key][:weight] }
|
159
|
-
other_weights = other_keys.collect{|key| other_schema[:fields][key][:weight] }
|
155
|
+
weights = keys.collect { |key| @schema[:fields][key][:weight] }
|
156
|
+
other_weights = other_keys.collect { |key| other_schema[:fields][key][:weight] }
|
160
157
|
changes[:weights] = weights != other_weights # will always be true if fields are added
|
161
|
-
if (removed_count = other_weights.
|
158
|
+
if (removed_count = other_weights.count { |w| w == 0 }) > 0
|
162
159
|
changes[:removed_fields_count] = removed_count
|
163
160
|
end
|
164
161
|
# check field attributes, only backed tables have attributes
|
165
|
-
attrs = keys.collect do |key|
|
162
|
+
attrs = keys.collect do |key|
|
166
163
|
f = @schema[:fields][key].dup
|
167
164
|
f.delete(:weight)
|
168
|
-
f.select{|k,v| allowed_attributes.include? k }
|
165
|
+
f.select { |k, v| allowed_attributes.include? k }
|
169
166
|
end
|
170
167
|
other_attrs = other_keys.collect do |key|
|
171
168
|
f = other_schema[:fields][key].dup
|
172
169
|
f.delete(:weight)
|
173
|
-
f.select{|k,v| allowed_attributes.include? k }
|
170
|
+
f.select { |k, v| allowed_attributes.include? k }
|
174
171
|
end
|
175
172
|
changes[:attributes] if other_attrs != attrs # this means that we will need to redefine the triggers if any are there and also the table definition if needed
|
176
|
-
|
173
|
+
|
177
174
|
# return the changes
|
178
175
|
changes
|
179
176
|
end
|
180
|
-
|
177
|
+
|
181
178
|
def clean
|
182
|
-
removable = @schema[:fields].select{|name, f| f[:weight] == 0 }.collect{|name, f| name}
|
183
|
-
removable.each{|name| @schema[:fields].delete(name)}
|
179
|
+
removable = @schema[:fields].select { |name, f| f[:weight] == 0 }.collect { |name, f| name }
|
180
|
+
removable.each { |name| @schema[:fields].delete(name) }
|
184
181
|
end
|
185
|
-
|
182
|
+
|
186
183
|
def allowed_attributes
|
187
184
|
[:weight, :col, :target]
|
188
185
|
end
|
189
|
-
|
190
186
|
end
|
191
187
|
|
192
188
|
class Litesearch::SchemaException < StandardError; end
|
189
|
+
|
193
190
|
class Litesearch::SchemaChangeException < StandardError; end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
class Litesearch::Schema::BackedAdapter < Litesearch::Schema::ContentlessAdapter
|
2
|
-
|
3
2
|
private
|
4
3
|
|
5
4
|
def table
|
@@ -14,50 +13,50 @@ class Litesearch::Schema::BackedAdapter < Litesearch::Schema::ContentlessAdapter
|
|
14
13
|
@sql[:create_primary_triggers] = :create_primary_triggers_sql
|
15
14
|
@sql[:create_secondary_triggers] = :create_secondary_triggers_sql
|
16
15
|
end
|
17
|
-
|
16
|
+
|
18
17
|
def drop_primary_triggers_sql
|
19
|
-
|
20
|
-
DROP TRIGGER IF EXISTS #{name}_insert;
|
21
|
-
DROP TRIGGER IF EXISTS #{name}_update;
|
22
|
-
DROP TRIGGER IF EXISTS #{name}_update_not;
|
23
|
-
DROP TRIGGER IF EXISTS #{name}_delete;
|
24
|
-
SQL
|
18
|
+
<<~SQL
|
19
|
+
DROP TRIGGER IF EXISTS #{name}_insert;
|
20
|
+
DROP TRIGGER IF EXISTS #{name}_update;
|
21
|
+
DROP TRIGGER IF EXISTS #{name}_update_not;
|
22
|
+
DROP TRIGGER IF EXISTS #{name}_delete;
|
23
|
+
SQL
|
25
24
|
end
|
26
|
-
|
27
|
-
def create_primary_triggers_sql(active=false)
|
25
|
+
|
26
|
+
def create_primary_triggers_sql(active = false)
|
28
27
|
when_stmt = "TRUE"
|
29
28
|
cols = active_cols_names
|
30
|
-
if filter = @schema[:filter_column]
|
29
|
+
if (filter = @schema[:filter_column])
|
31
30
|
when_stmt = "NEW.#{filter} = TRUE"
|
32
31
|
cols << filter
|
33
32
|
end
|
34
33
|
|
35
|
-
|
34
|
+
<<-SQL
|
36
35
|
CREATE TRIGGER #{name}_insert AFTER INSERT ON #{table} WHEN #{when_stmt} BEGIN
|
37
|
-
INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(
|
36
|
+
INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) VALUES (NEW.rowid, #{trigger_cols_sql});
|
38
37
|
END;
|
39
|
-
CREATE TRIGGER #{name}_update AFTER UPDATE OF #{cols.join(
|
40
|
-
INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(
|
38
|
+
CREATE TRIGGER #{name}_update AFTER UPDATE OF #{cols.join(", ")} ON #{table} WHEN #{when_stmt} BEGIN
|
39
|
+
INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) VALUES (NEW.rowid, #{trigger_cols_sql});
|
41
40
|
END;
|
42
|
-
CREATE TRIGGER #{name}_update_not AFTER UPDATE OF #{cols.join(
|
41
|
+
CREATE TRIGGER #{name}_update_not AFTER UPDATE OF #{cols.join(", ")} ON #{table} WHEN NOT #{when_stmt} BEGIN
|
43
42
|
DELETE FROM #{name} WHERE rowid = NEW.rowid;
|
44
43
|
END;
|
45
44
|
CREATE TRIGGER #{name}_delete AFTER DELETE ON #{table} BEGIN
|
46
45
|
DELETE FROM #{name} WHERE rowid = OLD.id;
|
47
46
|
END;
|
48
|
-
SQL
|
47
|
+
SQL
|
49
48
|
end
|
50
|
-
|
49
|
+
|
51
50
|
def drop_secondary_trigger_sql(target_table, target_col, col)
|
52
51
|
"DROP TRIGGER IF EXISTS #{target_table}_#{target_col}_#{col}_#{name}_update;"
|
53
52
|
end
|
54
|
-
|
53
|
+
|
55
54
|
def create_secondary_trigger_sql(target_table, target_col, col)
|
56
|
-
|
57
|
-
CREATE TRIGGER #{target_table}_#{target_col}_#{col}_#{name}_update AFTER UPDATE OF #{target_col} ON #{target_table} BEGIN
|
58
|
-
|
59
|
-
END;
|
60
|
-
SQL
|
55
|
+
<<~SQL
|
56
|
+
CREATE TRIGGER #{target_table}_#{target_col}_#{col}_#{name}_update AFTER UPDATE OF #{target_col} ON #{target_table} BEGIN
|
57
|
+
#{rebuild_sql} AND #{table}.#{col} = NEW.id;
|
58
|
+
END;
|
59
|
+
SQL
|
61
60
|
end
|
62
61
|
|
63
62
|
def drop_secondary_triggers_sql
|
@@ -67,36 +66,34 @@ SQL
|
|
67
66
|
sql << drop_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
|
68
67
|
end
|
69
68
|
end
|
70
|
-
|
69
|
+
sql.empty? ? nil : sql
|
71
70
|
end
|
72
71
|
|
73
|
-
|
74
72
|
def create_secondary_triggers_sql
|
75
|
-
sql = ""
|
73
|
+
sql = ""
|
76
74
|
@schema[:fields].each do |name, field|
|
77
75
|
if field[:trigger_sql]
|
78
76
|
sql << create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
|
79
77
|
end
|
80
78
|
end
|
81
|
-
|
79
|
+
sql.empty? ? nil : sql
|
82
80
|
end
|
83
|
-
|
84
|
-
|
81
|
+
|
85
82
|
def rebuild_sql
|
86
|
-
conditions = ""
|
83
|
+
conditions = ""
|
87
84
|
jcs = join_conditions_sql
|
88
85
|
fs = filter_sql
|
89
86
|
conditions = " ON #{jcs} #{fs}" unless jcs.empty? && fs.empty?
|
90
|
-
"INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(
|
87
|
+
"INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) SELECT #{table}.id, #{select_cols_sql} FROM #{join_tables_sql} #{conditions}"
|
91
88
|
end
|
92
|
-
|
89
|
+
|
93
90
|
def enrich_schema
|
94
|
-
@schema[:fields].each do |name, field|
|
95
|
-
if field[:target] && !
|
91
|
+
@schema[:fields].each do |name, field|
|
92
|
+
if field[:target] && !field[:target].start_with?("#{table}.")
|
96
93
|
field[:target] = field[:target].downcase
|
97
|
-
target_table, target_col = field[:target].split(
|
94
|
+
target_table, target_col = field[:target].split(".")
|
98
95
|
field[:col] = "#{name}_id".to_sym unless field[:col]
|
99
|
-
field[:target_table] = target_table.to_sym
|
96
|
+
field[:target_table] = target_table.to_sym
|
100
97
|
field[:target_col] = target_col.to_sym
|
101
98
|
field[:sql] = "(SELECT #{field[:target_col]} FROM #{field[:target_table]} WHERE id = NEW.#{field[:col]})"
|
102
99
|
field[:trigger_sql] = true # create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
|
@@ -109,9 +106,9 @@ SQL
|
|
109
106
|
end
|
110
107
|
end
|
111
108
|
end
|
112
|
-
|
109
|
+
|
113
110
|
def filter_sql
|
114
|
-
sql = ""
|
111
|
+
sql = ""
|
115
112
|
sql << " AND #{@schema[:filter_column]} = TRUE " if @schema[:filter_column]
|
116
113
|
sql
|
117
114
|
end
|
@@ -121,27 +118,26 @@ SQL
|
|
121
118
|
field[:trigger_sql] ? field[:sql] : "NEW.#{field[:sql]}"
|
122
119
|
end.join(", ")
|
123
120
|
end
|
124
|
-
|
121
|
+
|
125
122
|
def select_cols_sql
|
126
123
|
active_fields.collect do |name, field|
|
127
|
-
field[:trigger_sql]
|
128
|
-
end.join(
|
124
|
+
(!field[:trigger_sql].nil?) ? "#{field[:target_table_alias]}.#{field[:target_col]}" : field[:target]
|
125
|
+
end.join(", ")
|
129
126
|
end
|
130
|
-
|
127
|
+
|
131
128
|
def join_tables_sql
|
132
|
-
tables = [@schema[:table]]
|
129
|
+
tables = [@schema[:table]]
|
133
130
|
active_fields.each do |name, field|
|
134
|
-
tables << "#{field[:target_table]} AS #{field[:target_table_alias]}" if field[:trigger_sql]
|
131
|
+
tables << "#{field[:target_table]} AS #{field[:target_table_alias]}" if field[:trigger_sql]
|
135
132
|
end
|
136
133
|
tables.uniq.join(", ")
|
137
134
|
end
|
138
|
-
|
135
|
+
|
139
136
|
def join_conditions_sql
|
140
137
|
conditions = []
|
141
138
|
active_fields.each do |name, field|
|
142
|
-
conditions << "#{field[:target_table_alias]}.id = #{@schema[:table]}.#{field[:col]}" if field[:trigger_sql]
|
139
|
+
conditions << "#{field[:target_table_alias]}.id = #{@schema[:table]}.#{field[:col]}" if field[:trigger_sql]
|
143
140
|
end
|
144
|
-
conditions.join(" AND ")
|
145
|
-
end
|
146
|
-
|
141
|
+
conditions.join(" AND ")
|
142
|
+
end
|
147
143
|
end
|
@@ -1,16 +1,15 @@
|
|
1
1
|
class Litesearch::Schema::BasicAdapter
|
2
|
-
|
3
2
|
def initialize(schema)
|
4
3
|
@schema = schema
|
5
4
|
@sql = {}
|
6
5
|
enrich_schema
|
7
6
|
generate_sql
|
8
7
|
end
|
9
|
-
|
8
|
+
|
10
9
|
def name
|
11
10
|
@schema[:name]
|
12
11
|
end
|
13
|
-
|
12
|
+
|
14
13
|
def table
|
15
14
|
@schema[:table]
|
16
15
|
end
|
@@ -22,25 +21,25 @@ class Litesearch::Schema::BasicAdapter
|
|
22
21
|
def field_names
|
23
22
|
@schema[:fields].keys
|
24
23
|
end
|
25
|
-
|
24
|
+
|
26
25
|
def active_fields
|
27
|
-
@schema[:fields].select{|k, v| v[:weight] != 0 }
|
26
|
+
@schema[:fields].select { |k, v| v[:weight] != 0 }
|
28
27
|
end
|
29
|
-
|
28
|
+
|
30
29
|
def active_field_names
|
31
30
|
active_fields.keys
|
32
31
|
end
|
33
|
-
|
32
|
+
|
34
33
|
def active_cols_names
|
35
|
-
active_fields.collect{|k, v| v[:col]}
|
34
|
+
active_fields.collect { |k, v| v[:col] }
|
36
35
|
end
|
37
|
-
|
36
|
+
|
38
37
|
def weights
|
39
|
-
@schema[:fields].values.collect{|v| v[:weight].to_f }
|
38
|
+
@schema[:fields].values.collect { |v| v[:weight].to_f }
|
40
39
|
end
|
41
40
|
|
42
41
|
def active_weights
|
43
|
-
active_fields.values.collect{|v| v[:weight].to_f }
|
42
|
+
active_fields.values.collect { |v| v[:weight].to_f }
|
44
43
|
end
|
45
44
|
|
46
45
|
def tokenizer_sql
|
@@ -51,33 +50,34 @@ class Litesearch::Schema::BasicAdapter
|
|
51
50
|
new_fields = {}
|
52
51
|
old_field_names = old_schema.schema[:fields].keys
|
53
52
|
old_field_names.each do |name|
|
54
|
-
new_fields[name] = @schema[:fields].delete(name)
|
53
|
+
new_fields[name] = @schema[:fields].delete(name)
|
55
54
|
end
|
56
55
|
missing_field_names = field_names - old_field_names
|
57
56
|
missing_field_names.each do |name|
|
58
57
|
new_fields[name] = @schema[:fields].delete(name)
|
59
|
-
end
|
58
|
+
end
|
60
59
|
@schema[:fields] = new_fields # this should be in order now
|
61
60
|
generate_sql
|
62
61
|
enrich_schema
|
63
62
|
end
|
64
63
|
|
65
64
|
def sql_for(method, *args)
|
66
|
-
if sql = @sql[method]
|
65
|
+
if (sql = @sql[method])
|
67
66
|
if sql.is_a? String
|
68
|
-
|
67
|
+
sql
|
69
68
|
elsif sql.is_a? Proc
|
70
|
-
|
69
|
+
sql.call(*args)
|
71
70
|
elsif sql.is_a? Symbol
|
72
|
-
|
71
|
+
send(sql, *args)
|
73
72
|
elsif sql.is_a? Litesearch::SchemaChangeException
|
74
73
|
raise sql
|
75
74
|
end
|
76
75
|
end
|
77
76
|
end
|
78
|
-
|
77
|
+
|
79
78
|
def generate_sql
|
80
79
|
@sql[:create_index] = :create_index_sql
|
80
|
+
@sql[:create_vocab_tables] = :create_vocab_tables_sql
|
81
81
|
@sql[:insert] = "INSERT OR REPLACE INTO #{name}(rowid, #{active_col_names_sql}) VALUES (:id, #{active_col_names_var_sql}) RETURNING rowid"
|
82
82
|
@sql[:delete] = "DELETE FROM #{name} WHERE rowid = :id"
|
83
83
|
@sql[:count] = "SELECT count(*) FROM #{name}(:term)"
|
@@ -86,43 +86,52 @@ class Litesearch::Schema::BasicAdapter
|
|
86
86
|
@sql[:drop] = "DROP TABLE #{name}"
|
87
87
|
@sql[:expand_data] = "UPDATE #{name}_data SET block = block || zeroblob(:length) WHERE id = 1"
|
88
88
|
@sql[:expand_docsize] = "UPDATE #{name}_docsize SET sz = sz || zeroblob(:length)"
|
89
|
-
@sql[:ranks] = :ranks_sql
|
89
|
+
@sql[:ranks] = :ranks_sql
|
90
90
|
@sql[:set_config_value] = "INSERT OR REPLACE INTO #{name}_config(k, v) VALUES (:key, :value)"
|
91
91
|
@sql[:get_config_value] = "SELECT v FROM #{name}_config WHERE k = :key"
|
92
92
|
@sql[:search] = "SELECT rowid AS id, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
|
93
|
+
@sql[:similarity_terms] = "SELECT DISTINCT term FROM #{name}_instance WHERE doc = :id AND FLOOR(term) IS NULL AND LENGTH(term) > 2 AND NOT instr(term, ' ') AND NOT instr(term, '-') AND NOT instr(term, ':') AND NOT instr(term, '#') AND NOT instr(term, '_') LIMIT 15"
|
94
|
+
@sql[:similarity_query] = "SELECT group_concat('\"' || term || '\"', ' OR ') FROM #{name}_row WHERE term IN (#{@sql[:similarity_terms]})"
|
95
|
+
@sql[:similarity_search] = "SELECT rowid AS id, -rank AS search_rank FROM #{name}(:term) WHERE rowid != :id ORDER BY rank LIMIT :limit"
|
96
|
+
@sql[:similar] = "SELECT rowid AS id, -rank AS search_rank FROM #{name} WHERE #{name} = (#{@sql[:similarity_query]}) AND rowid != :id ORDER BY rank LIMIT :limit"
|
93
97
|
@sql[:update_index] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}'"
|
94
98
|
@sql[:update_content_table] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}_content'"
|
95
99
|
end
|
96
|
-
|
100
|
+
|
97
101
|
private
|
98
|
-
|
99
|
-
def
|
100
|
-
|
101
|
-
|
102
|
+
|
103
|
+
def create_vocab_tables_sql
|
104
|
+
<<~SQL
|
105
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS #{name}_row USING fts5vocab(#{name}, row);
|
106
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS #{name}_instance USING fts5vocab(#{name}, instance);
|
107
|
+
SQL
|
108
|
+
end
|
109
|
+
|
110
|
+
def ranks_sql(active = false)
|
111
|
+
weights_sql = if active
|
112
|
+
weights.join(", ")
|
102
113
|
else
|
103
|
-
|
114
|
+
active_weights.join(", ")
|
104
115
|
end
|
105
116
|
"INSERT INTO #{name}(#{name}, rank) VALUES ('rank', 'bm25(#{weights_sql})')"
|
106
117
|
end
|
107
|
-
|
118
|
+
|
108
119
|
def active_col_names_sql
|
109
|
-
active_field_names.join(
|
120
|
+
active_field_names.join(", ")
|
110
121
|
end
|
111
|
-
|
122
|
+
|
112
123
|
def active_col_names_var_sql
|
113
|
-
":#{active_field_names.join(
|
124
|
+
":#{active_field_names.join(", :")}"
|
114
125
|
end
|
115
|
-
|
126
|
+
|
116
127
|
def col_names_sql
|
117
|
-
field_names.join(
|
128
|
+
field_names.join(", ")
|
118
129
|
end
|
119
|
-
|
130
|
+
|
120
131
|
def col_names_var_sql
|
121
|
-
":#{field_names.join(
|
132
|
+
":#{field_names.join(", :")}"
|
122
133
|
end
|
123
134
|
|
124
135
|
def enrich_schema
|
125
136
|
end
|
126
|
-
|
127
137
|
end
|
128
|
-
|
@@ -1,17 +1,14 @@
|
|
1
1
|
class Litesearch::Schema::ContentlessAdapter < Litesearch::Schema::BasicAdapter
|
2
|
-
|
3
2
|
private
|
4
3
|
|
5
4
|
def generate_sql
|
6
5
|
super
|
7
|
-
|
8
|
-
|
6
|
+
# @sql[:rebuild_index] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
|
7
|
+
# @sql[:rebuild] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
|
9
8
|
end
|
10
|
-
|
9
|
+
|
11
10
|
def create_index_sql(active = false)
|
12
11
|
col_names = active ? active_col_names_sql : col_names_sql
|
13
12
|
"CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, content='', contentless_delete=1, tokenize='#{tokenizer_sql}')"
|
14
13
|
end
|
15
|
-
|
16
14
|
end
|
17
|
-
|
@@ -1,14 +1,14 @@
|
|
1
1
|
class Litesearch::Schema::StandaloneAdapter < Litesearch::Schema::BasicAdapter
|
2
|
-
|
3
2
|
def generate_sql
|
4
3
|
super
|
5
4
|
@sql[:move_content] = "ALTER TABLE #{name}_content RENAME TO #{name}_content_temp"
|
6
5
|
@sql[:adjust_temp_content] = "UPDATE sqlite_schema SET sql (SELECT sql FROM sqlite_schema WHERE name = '#{name}_content') WHERE name = #{name}_content_temp"
|
7
6
|
@sql[:restore_content] = "ALTER TABLE #{name}_content_temp RENAME TO #{name}_content"
|
8
7
|
@sql[:rebuild] = "INSERT INTO #{name}(#{name}) VALUES ('rebuild')"
|
8
|
+
@sql[:similar] = "SELECT rowid AS id, *, -rank AS search_rank FROM #{name} WHERE #{name} = (#{@sql[:similarity_query]}) AND rowid != :id ORDER BY rank LIMIT :limit"
|
9
9
|
@sql[:drop_content_table] = "DROP TABLE #{name}_content"
|
10
10
|
@sql[:drop_content_col] = :drop_content_col_sql
|
11
|
-
@sql[:create_content_table] = :create_content_table_sql
|
11
|
+
@sql[:create_content_table] = :create_content_table_sql
|
12
12
|
@sql[:search] = "SELECT rowid AS id, *, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
|
13
13
|
end
|
14
14
|
|
@@ -18,16 +18,14 @@ class Litesearch::Schema::StandaloneAdapter < Litesearch::Schema::BasicAdapter
|
|
18
18
|
col_names = active ? active_col_names_sql : col_names_sql
|
19
19
|
"CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, tokenize='#{tokenizer_sql}')"
|
20
20
|
end
|
21
|
-
|
21
|
+
|
22
22
|
def drop_content_col_sql(col_index)
|
23
23
|
"ALTER TABLE #{name}_content DROP COLUMN c#{col_index}"
|
24
24
|
end
|
25
|
-
|
25
|
+
|
26
26
|
def create_content_table_sql(count)
|
27
|
-
cols = []
|
28
|
-
count.times{|i| cols << "c#{i}" }
|
29
|
-
"CREATE TABLE #{name}_content(id INTEGER PRIMARY KEY, #{cols.join(
|
27
|
+
cols = []
|
28
|
+
count.times { |i| cols << "c#{i}" }
|
29
|
+
"CREATE TABLE #{name}_content(id INTEGER PRIMARY KEY, #{cols.join(", ")})"
|
30
30
|
end
|
31
|
-
|
32
31
|
end
|
33
|
-
|
@@ -1,9 +1,4 @@
|
|
1
|
-
require_relative
|
2
|
-
require_relative
|
3
|
-
require_relative
|
4
|
-
require_relative
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
1
|
+
require_relative "./schema_adapters/basic_adapter"
|
2
|
+
require_relative "./schema_adapters/standalone_adapter"
|
3
|
+
require_relative "./schema_adapters/contentless_adapter"
|
4
|
+
require_relative "./schema_adapters/backed_adapter"
|