litestack 0.4.1 → 0.4.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.standard.yml +3 -0
  3. data/BENCHMARKS.md +23 -7
  4. data/CHANGELOG.md +11 -0
  5. data/Gemfile +1 -7
  6. data/Gemfile.lock +92 -0
  7. data/README.md +120 -6
  8. data/ROADMAP.md +45 -0
  9. data/Rakefile +3 -1
  10. data/WHYLITESTACK.md +1 -1
  11. data/assets/litecache_metrics.png +0 -0
  12. data/assets/litedb_metrics.png +0 -0
  13. data/assets/litemetric_logo_teal.png +0 -0
  14. data/assets/litesearch_logo_teal.png +0 -0
  15. data/bench/bench.rb +17 -10
  16. data/bench/bench_cache_rails.rb +10 -13
  17. data/bench/bench_cache_raw.rb +17 -22
  18. data/bench/bench_jobs_rails.rb +18 -12
  19. data/bench/bench_jobs_raw.rb +17 -10
  20. data/bench/bench_queue.rb +4 -6
  21. data/bench/rails_job.rb +5 -7
  22. data/bench/skjob.rb +4 -4
  23. data/bench/uljob.rb +6 -6
  24. data/lib/action_cable/subscription_adapter/litecable.rb +5 -8
  25. data/lib/active_job/queue_adapters/litejob_adapter.rb +6 -8
  26. data/lib/active_record/connection_adapters/litedb_adapter.rb +65 -75
  27. data/lib/active_support/cache/litecache.rb +38 -41
  28. data/lib/generators/litestack/install/install_generator.rb +3 -3
  29. data/lib/generators/litestack/install/templates/database.yml +7 -1
  30. data/lib/litestack/liteboard/liteboard.rb +269 -149
  31. data/lib/litestack/litecable.rb +41 -37
  32. data/lib/litestack/litecable.sql.yml +22 -11
  33. data/lib/litestack/litecache.rb +79 -88
  34. data/lib/litestack/litecache.sql.yml +81 -22
  35. data/lib/litestack/litecache.yml +1 -1
  36. data/lib/litestack/litedb.rb +35 -40
  37. data/lib/litestack/litejob.rb +30 -29
  38. data/lib/litestack/litejobqueue.rb +63 -65
  39. data/lib/litestack/litemetric.rb +80 -92
  40. data/lib/litestack/litemetric.sql.yml +244 -234
  41. data/lib/litestack/litemetric_collector.sql.yml +38 -41
  42. data/lib/litestack/litequeue.rb +39 -41
  43. data/lib/litestack/litequeue.sql.yml +39 -31
  44. data/lib/litestack/litescheduler.rb +15 -15
  45. data/lib/litestack/litesearch/index.rb +93 -63
  46. data/lib/litestack/litesearch/model.rb +66 -65
  47. data/lib/litestack/litesearch/schema.rb +53 -56
  48. data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +46 -50
  49. data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +44 -35
  50. data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +3 -6
  51. data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +7 -9
  52. data/lib/litestack/litesearch/schema_adapters.rb +4 -9
  53. data/lib/litestack/litesearch.rb +6 -9
  54. data/lib/litestack/litesupport.rb +76 -86
  55. data/lib/litestack/railtie.rb +1 -1
  56. data/lib/litestack/version.rb +2 -2
  57. data/lib/litestack.rb +6 -4
  58. data/lib/railties/rails/commands/dbconsole.rb +11 -15
  59. data/lib/sequel/adapters/litedb.rb +16 -21
  60. data/lib/sequel/adapters/shared/litedb.rb +168 -168
  61. data/scripts/build_metrics.rb +91 -0
  62. data/scripts/test_cable.rb +30 -0
  63. data/scripts/test_job_retry.rb +33 -0
  64. data/scripts/test_metrics.rb +60 -0
  65. data/template.rb +2 -2
  66. metadata +101 -6
@@ -1,37 +1,36 @@
1
- require_relative './schema_adapters.rb'
1
+ require_relative "./schema_adapters"
2
2
 
3
3
  class Litesearch::Schema
4
-
5
- TOKENIZERS = {
6
- porter: 'porter unicode61 remove_diacritics 2',
7
- unicode: 'unicode61 remove_diacritics 2',
8
- ascii: 'ascii',
9
- trigram: 'trigram'
4
+ TOKENIZERS = {
5
+ porter: "porter unicode61 remove_diacritics 2",
6
+ unicode: "unicode61 remove_diacritics 2",
7
+ ascii: "ascii",
8
+ trigram: "trigram"
10
9
  }
11
-
10
+
12
11
  INDEX_TYPES = {
13
- standalone: Litesearch::Schema::StandaloneAdapter,
14
- contentless: Litesearch::Schema::ContentlessAdapter,
12
+ standalone: Litesearch::Schema::StandaloneAdapter,
13
+ contentless: Litesearch::Schema::ContentlessAdapter,
15
14
  backed: Litesearch::Schema::BackedAdapter
16
15
  }
17
16
 
18
17
  DEFAULT_SCHEMA = {
19
18
  name: nil,
20
19
  type: :standalone,
21
- fields: nil,
20
+ fields: nil,
22
21
  table: nil,
23
22
  filter_column: nil,
24
23
  tokenizer: :porter,
25
24
  auto_create: true,
26
25
  auto_modify: true,
27
- rebuild_on_create: false,
28
- rebuild_on_modify: false
26
+ rebuild_on_create: false,
27
+ rebuild_on_modify: false
29
28
  }
30
29
 
31
30
  attr_accessor :schema
32
31
 
33
32
  def initialize(schema = {})
34
- @schema = schema #DEFAULT_SCHEMA.merge(schema)
33
+ @schema = schema # DEFAULT_SCHEMA.merge(schema)
35
34
  @schema[:fields] = {} unless @schema[:fields]
36
35
  end
37
36
 
@@ -39,7 +38,7 @@ class Litesearch::Schema
39
38
  def name(new_name)
40
39
  @schema[:name] = new_name
41
40
  end
42
-
41
+
43
42
  def type(new_type)
44
43
  raise "Unknown index type" if INDEX_TYPES[new_type].nil?
45
44
  @schema[:type] = new_type
@@ -50,38 +49,38 @@ class Litesearch::Schema
50
49
  end
51
50
 
52
51
  def fields(field_names)
53
- field_names.each {|f| field f }
52
+ field_names.each { |f| field f }
54
53
  end
55
-
54
+
56
55
  def field(name, attributes = {})
57
56
  name = name.to_s.downcase.to_sym
58
- attributes = {weight: 1}.merge(attributes).select{|k, v| allowed_attributes.include?(k)} # only allow attributes we know, to ease schema comparison later
57
+ attributes = {weight: 1}.merge(attributes).select { |k, v| allowed_attributes.include?(k) } # only allow attributes we know, to ease schema comparison later
59
58
  @schema[:fields][name] = attributes
60
59
  end
61
-
60
+
62
61
  def tokenizer(new_tokenizer)
63
62
  raise "Unknown tokenizer" if TOKENIZERS[new_tokenizer].nil?
64
- @schema[:tokenizer] = new_tokenizer
63
+ @schema[:tokenizer] = new_tokenizer
65
64
  end
66
-
65
+
67
66
  def filter_column(filter_column)
68
67
  @schema[:filter_column] = filter_column
69
68
  end
70
-
69
+
71
70
  def auto_create(boolean)
72
71
  @schema[:auto_create] = boolean
73
72
  end
74
-
73
+
75
74
  def auto_modify(boolean)
76
75
  @schema[:auto_modify] = boolean
77
76
  end
78
-
77
+
79
78
  def rebuild_on_create(boolean)
80
- @schema[:rebuild_on_create] = boolean
79
+ @schema[:rebuild_on_create] = boolean
81
80
  end
82
-
81
+
83
82
  def rebuild_on_modify(boolean)
84
- @schema[:rebuild_on_modify] = boolean
83
+ @schema[:rebuild_on_modify] = boolean
85
84
  end
86
85
 
87
86
  def post_init
@@ -89,51 +88,49 @@ class Litesearch::Schema
89
88
  end
90
89
 
91
90
  # schema sql generation API
92
-
91
+
93
92
  def sql_for(method, *args)
94
93
  adapter.sql_for(method, *args)
95
94
  end
96
-
95
+
97
96
  # schema data structure API
98
97
  def get(key)
99
98
  @schema[key]
100
- end
101
-
99
+ end
100
+
102
101
  def get_field(name)
103
102
  @schema[:fields][name]
104
103
  end
105
-
104
+
106
105
  def adapter
107
106
  @adapter ||= INDEX_TYPES[@schema[:type]].new(@schema)
108
107
  end
109
-
108
+
110
109
  def reset_sql
111
110
  adapter.generate_sql
112
111
  end
113
-
112
+
114
113
  def order_fields(old_schema)
115
114
  adapter.order_fields(old_schema)
116
115
  end
117
-
116
+
118
117
  # should we do this at the schema objects level?
119
118
  def compare(other_schema)
120
119
  other_schema = other_schema.schema
121
120
  # are the schemas identical?
122
121
  # 1 - same fields?
123
122
  [:type, :tokenizer, :name, :table].each do |key|
124
- other_schema[key] = @schema[key] if other_schema[key].nil?
123
+ other_schema[key] = @schema[key] if other_schema[key].nil?
125
124
  end
126
125
  if @schema[:type] != other_schema[:type]
127
126
  raise Litesearch::SchemaChangeException.new "Cannot change the index type, please drop the index before creating it again with the new type"
128
127
  end
129
- changes = { tokenizer: @schema[:tokenizer] != other_schema[:tokenizer], table: @schema[:table] != other_schema[:table], removed_fields_count: 0, filter_column: @schema[:filter_column] != other_schema[:filter_column] }
130
- #check tokenizer changes
128
+ changes = {tokenizer: @schema[:tokenizer] != other_schema[:tokenizer], table: @schema[:table] != other_schema[:table], removed_fields_count: 0, filter_column: @schema[:filter_column] != other_schema[:filter_column]}
129
+ # check tokenizer changes
131
130
  if changes[:tokenizer] && !other_schema[:rebuild_on_modify]
132
131
  raise Litesearch::SchemaChangeException.new "Cannot change the tokenizer without an index rebuild!"
133
132
  end
134
133
 
135
-
136
-
137
134
  # check field changes
138
135
  keys = @schema[:fields].keys.sort
139
136
  other_keys = other_schema[:fields].keys.sort
@@ -144,50 +141,50 @@ class Litesearch::Schema
144
141
  other_schema[:fields].delete(key)
145
142
  end
146
143
  end
147
-
144
+
148
145
  other_keys = other_schema[:fields].keys.sort
149
146
 
150
147
  changes[:fields] = keys != other_keys # only acceptable change is adding extra fields
151
- changes[:extra_fields_count] = other_keys.count - keys.count
148
+ changes[:extra_fields_count] = other_keys.count - keys.count
152
149
  # check for missing fields (please note that adding fields can work without a rebuild)
153
150
  if keys - other_keys != []
154
- raise Litesearch::SchemaChangeException.new "Missing fields from existing schema, they have to exist with weight zero until the next rebuild!"
151
+ raise Litesearch::SchemaChangeException.new "Missing fields from existing schema, they have to exist with weight zero until the next rebuild!"
155
152
  end
156
-
153
+
157
154
  # check field weights
158
- weights = keys.collect{|key| @schema[:fields][key][:weight] }
159
- other_weights = other_keys.collect{|key| other_schema[:fields][key][:weight] }
155
+ weights = keys.collect { |key| @schema[:fields][key][:weight] }
156
+ other_weights = other_keys.collect { |key| other_schema[:fields][key][:weight] }
160
157
  changes[:weights] = weights != other_weights # will always be true if fields are added
161
- if (removed_count = other_weights.select{|w| w == 0}.count) > 0
158
+ if (removed_count = other_weights.count { |w| w == 0 }) > 0
162
159
  changes[:removed_fields_count] = removed_count
163
160
  end
164
161
  # check field attributes, only backed tables have attributes
165
- attrs = keys.collect do |key|
162
+ attrs = keys.collect do |key|
166
163
  f = @schema[:fields][key].dup
167
164
  f.delete(:weight)
168
- f.select{|k,v| allowed_attributes.include? k }
165
+ f.select { |k, v| allowed_attributes.include? k }
169
166
  end
170
167
  other_attrs = other_keys.collect do |key|
171
168
  f = other_schema[:fields][key].dup
172
169
  f.delete(:weight)
173
- f.select{|k,v| allowed_attributes.include? k }
170
+ f.select { |k, v| allowed_attributes.include? k }
174
171
  end
175
172
  changes[:attributes] if other_attrs != attrs # this means that we will need to redefine the triggers if any are there and also the table definition if needed
176
-
173
+
177
174
  # return the changes
178
175
  changes
179
176
  end
180
-
177
+
181
178
  def clean
182
- removable = @schema[:fields].select{|name, f| f[:weight] == 0 }.collect{|name, f| name}
183
- removable.each{|name| @schema[:fields].delete(name)}
179
+ removable = @schema[:fields].select { |name, f| f[:weight] == 0 }.collect { |name, f| name }
180
+ removable.each { |name| @schema[:fields].delete(name) }
184
181
  end
185
-
182
+
186
183
  def allowed_attributes
187
184
  [:weight, :col, :target]
188
185
  end
189
-
190
186
  end
191
187
 
192
188
  class Litesearch::SchemaException < StandardError; end
189
+
193
190
  class Litesearch::SchemaChangeException < StandardError; end
@@ -1,5 +1,4 @@
1
1
  class Litesearch::Schema::BackedAdapter < Litesearch::Schema::ContentlessAdapter
2
-
3
2
  private
4
3
 
5
4
  def table
@@ -14,50 +13,50 @@ class Litesearch::Schema::BackedAdapter < Litesearch::Schema::ContentlessAdapter
14
13
  @sql[:create_primary_triggers] = :create_primary_triggers_sql
15
14
  @sql[:create_secondary_triggers] = :create_secondary_triggers_sql
16
15
  end
17
-
16
+
18
17
  def drop_primary_triggers_sql
19
- sql = <<-SQL
20
- DROP TRIGGER IF EXISTS #{name}_insert;
21
- DROP TRIGGER IF EXISTS #{name}_update;
22
- DROP TRIGGER IF EXISTS #{name}_update_not;
23
- DROP TRIGGER IF EXISTS #{name}_delete;
24
- SQL
18
+ <<~SQL
19
+ DROP TRIGGER IF EXISTS #{name}_insert;
20
+ DROP TRIGGER IF EXISTS #{name}_update;
21
+ DROP TRIGGER IF EXISTS #{name}_update_not;
22
+ DROP TRIGGER IF EXISTS #{name}_delete;
23
+ SQL
25
24
  end
26
-
27
- def create_primary_triggers_sql(active=false)
25
+
26
+ def create_primary_triggers_sql(active = false)
28
27
  when_stmt = "TRUE"
29
28
  cols = active_cols_names
30
- if filter = @schema[:filter_column]
29
+ if (filter = @schema[:filter_column])
31
30
  when_stmt = "NEW.#{filter} = TRUE"
32
31
  cols << filter
33
32
  end
34
33
 
35
- sql = <<-SQL
34
+ <<-SQL
36
35
  CREATE TRIGGER #{name}_insert AFTER INSERT ON #{table} WHEN #{when_stmt} BEGIN
37
- INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) VALUES (NEW.rowid, #{trigger_cols_sql});
36
+ INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) VALUES (NEW.rowid, #{trigger_cols_sql});
38
37
  END;
39
- CREATE TRIGGER #{name}_update AFTER UPDATE OF #{cols.join(', ')} ON #{table} WHEN #{when_stmt} BEGIN
40
- INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) VALUES (NEW.rowid, #{trigger_cols_sql});
38
+ CREATE TRIGGER #{name}_update AFTER UPDATE OF #{cols.join(", ")} ON #{table} WHEN #{when_stmt} BEGIN
39
+ INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) VALUES (NEW.rowid, #{trigger_cols_sql});
41
40
  END;
42
- CREATE TRIGGER #{name}_update_not AFTER UPDATE OF #{cols.join(', ')} ON #{table} WHEN NOT #{when_stmt} BEGIN
41
+ CREATE TRIGGER #{name}_update_not AFTER UPDATE OF #{cols.join(", ")} ON #{table} WHEN NOT #{when_stmt} BEGIN
43
42
  DELETE FROM #{name} WHERE rowid = NEW.rowid;
44
43
  END;
45
44
  CREATE TRIGGER #{name}_delete AFTER DELETE ON #{table} BEGIN
46
45
  DELETE FROM #{name} WHERE rowid = OLD.id;
47
46
  END;
48
- SQL
47
+ SQL
49
48
  end
50
-
49
+
51
50
  def drop_secondary_trigger_sql(target_table, target_col, col)
52
51
  "DROP TRIGGER IF EXISTS #{target_table}_#{target_col}_#{col}_#{name}_update;"
53
52
  end
54
-
53
+
55
54
  def create_secondary_trigger_sql(target_table, target_col, col)
56
- sql = <<-SQL
57
- CREATE TRIGGER #{target_table}_#{target_col}_#{col}_#{name}_update AFTER UPDATE OF #{target_col} ON #{target_table} BEGIN
58
- #{rebuild_sql} AND #{table}.#{col} = NEW.id;
59
- END;
60
- SQL
55
+ <<~SQL
56
+ CREATE TRIGGER #{target_table}_#{target_col}_#{col}_#{name}_update AFTER UPDATE OF #{target_col} ON #{target_table} BEGIN
57
+ #{rebuild_sql} AND #{table}.#{col} = NEW.id;
58
+ END;
59
+ SQL
61
60
  end
62
61
 
63
62
  def drop_secondary_triggers_sql
@@ -67,36 +66,34 @@ SQL
67
66
  sql << drop_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
68
67
  end
69
68
  end
70
- return sql.empty? ? nil : sql
69
+ sql.empty? ? nil : sql
71
70
  end
72
71
 
73
-
74
72
  def create_secondary_triggers_sql
75
- sql = ""
73
+ sql = ""
76
74
  @schema[:fields].each do |name, field|
77
75
  if field[:trigger_sql]
78
76
  sql << create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
79
77
  end
80
78
  end
81
- return sql.empty? ? nil : sql
79
+ sql.empty? ? nil : sql
82
80
  end
83
-
84
-
81
+
85
82
  def rebuild_sql
86
- conditions = ""
83
+ conditions = ""
87
84
  jcs = join_conditions_sql
88
85
  fs = filter_sql
89
86
  conditions = " ON #{jcs} #{fs}" unless jcs.empty? && fs.empty?
90
- "INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) SELECT #{table}.id, #{select_cols_sql} FROM #{join_tables_sql} #{conditions}"
87
+ "INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) SELECT #{table}.id, #{select_cols_sql} FROM #{join_tables_sql} #{conditions}"
91
88
  end
92
-
89
+
93
90
  def enrich_schema
94
- @schema[:fields].each do |name, field|
95
- if field[:target] && ! field[:target].start_with?("#{table}.")
91
+ @schema[:fields].each do |name, field|
92
+ if field[:target] && !field[:target].start_with?("#{table}.")
96
93
  field[:target] = field[:target].downcase
97
- target_table, target_col = field[:target].split('.')
94
+ target_table, target_col = field[:target].split(".")
98
95
  field[:col] = "#{name}_id".to_sym unless field[:col]
99
- field[:target_table] = target_table.to_sym
96
+ field[:target_table] = target_table.to_sym
100
97
  field[:target_col] = target_col.to_sym
101
98
  field[:sql] = "(SELECT #{field[:target_col]} FROM #{field[:target_table]} WHERE id = NEW.#{field[:col]})"
102
99
  field[:trigger_sql] = true # create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
@@ -109,9 +106,9 @@ SQL
109
106
  end
110
107
  end
111
108
  end
112
-
109
+
113
110
  def filter_sql
114
- sql = ""
111
+ sql = ""
115
112
  sql << " AND #{@schema[:filter_column]} = TRUE " if @schema[:filter_column]
116
113
  sql
117
114
  end
@@ -121,27 +118,26 @@ SQL
121
118
  field[:trigger_sql] ? field[:sql] : "NEW.#{field[:sql]}"
122
119
  end.join(", ")
123
120
  end
124
-
121
+
125
122
  def select_cols_sql
126
123
  active_fields.collect do |name, field|
127
- field[:trigger_sql] != nil ? "#{field[:target_table_alias]}.#{field[:target_col]}" : field[:target]
128
- end.join(', ')
124
+ (!field[:trigger_sql].nil?) ? "#{field[:target_table_alias]}.#{field[:target_col]}" : field[:target]
125
+ end.join(", ")
129
126
  end
130
-
127
+
131
128
  def join_tables_sql
132
- tables = [@schema[:table]]
129
+ tables = [@schema[:table]]
133
130
  active_fields.each do |name, field|
134
- tables << "#{field[:target_table]} AS #{field[:target_table_alias]}" if field[:trigger_sql]
131
+ tables << "#{field[:target_table]} AS #{field[:target_table_alias]}" if field[:trigger_sql]
135
132
  end
136
133
  tables.uniq.join(", ")
137
134
  end
138
-
135
+
139
136
  def join_conditions_sql
140
137
  conditions = []
141
138
  active_fields.each do |name, field|
142
- conditions << "#{field[:target_table_alias]}.id = #{@schema[:table]}.#{field[:col]}" if field[:trigger_sql]
139
+ conditions << "#{field[:target_table_alias]}.id = #{@schema[:table]}.#{field[:col]}" if field[:trigger_sql]
143
140
  end
144
- conditions.join(" AND ")
145
- end
146
-
141
+ conditions.join(" AND ")
142
+ end
147
143
  end
@@ -1,16 +1,15 @@
1
1
  class Litesearch::Schema::BasicAdapter
2
-
3
2
  def initialize(schema)
4
3
  @schema = schema
5
4
  @sql = {}
6
5
  enrich_schema
7
6
  generate_sql
8
7
  end
9
-
8
+
10
9
  def name
11
10
  @schema[:name]
12
11
  end
13
-
12
+
14
13
  def table
15
14
  @schema[:table]
16
15
  end
@@ -22,25 +21,25 @@ class Litesearch::Schema::BasicAdapter
22
21
  def field_names
23
22
  @schema[:fields].keys
24
23
  end
25
-
24
+
26
25
  def active_fields
27
- @schema[:fields].select{|k, v| v[:weight] != 0 }
26
+ @schema[:fields].select { |k, v| v[:weight] != 0 }
28
27
  end
29
-
28
+
30
29
  def active_field_names
31
30
  active_fields.keys
32
31
  end
33
-
32
+
34
33
  def active_cols_names
35
- active_fields.collect{|k, v| v[:col]}
34
+ active_fields.collect { |k, v| v[:col] }
36
35
  end
37
-
36
+
38
37
  def weights
39
- @schema[:fields].values.collect{|v| v[:weight].to_f }
38
+ @schema[:fields].values.collect { |v| v[:weight].to_f }
40
39
  end
41
40
 
42
41
  def active_weights
43
- active_fields.values.collect{|v| v[:weight].to_f }
42
+ active_fields.values.collect { |v| v[:weight].to_f }
44
43
  end
45
44
 
46
45
  def tokenizer_sql
@@ -51,33 +50,34 @@ class Litesearch::Schema::BasicAdapter
51
50
  new_fields = {}
52
51
  old_field_names = old_schema.schema[:fields].keys
53
52
  old_field_names.each do |name|
54
- new_fields[name] = @schema[:fields].delete(name)
53
+ new_fields[name] = @schema[:fields].delete(name)
55
54
  end
56
55
  missing_field_names = field_names - old_field_names
57
56
  missing_field_names.each do |name|
58
57
  new_fields[name] = @schema[:fields].delete(name)
59
- end
58
+ end
60
59
  @schema[:fields] = new_fields # this should be in order now
61
60
  generate_sql
62
61
  enrich_schema
63
62
  end
64
63
 
65
64
  def sql_for(method, *args)
66
- if sql = @sql[method]
65
+ if (sql = @sql[method])
67
66
  if sql.is_a? String
68
- return sql
67
+ sql
69
68
  elsif sql.is_a? Proc
70
- return sql.call(*args)
69
+ sql.call(*args)
71
70
  elsif sql.is_a? Symbol
72
- return self.send(sql, *args)
71
+ send(sql, *args)
73
72
  elsif sql.is_a? Litesearch::SchemaChangeException
74
73
  raise sql
75
74
  end
76
75
  end
77
76
  end
78
-
77
+
79
78
  def generate_sql
80
79
  @sql[:create_index] = :create_index_sql
80
+ @sql[:create_vocab_tables] = :create_vocab_tables_sql
81
81
  @sql[:insert] = "INSERT OR REPLACE INTO #{name}(rowid, #{active_col_names_sql}) VALUES (:id, #{active_col_names_var_sql}) RETURNING rowid"
82
82
  @sql[:delete] = "DELETE FROM #{name} WHERE rowid = :id"
83
83
  @sql[:count] = "SELECT count(*) FROM #{name}(:term)"
@@ -86,43 +86,52 @@ class Litesearch::Schema::BasicAdapter
86
86
  @sql[:drop] = "DROP TABLE #{name}"
87
87
  @sql[:expand_data] = "UPDATE #{name}_data SET block = block || zeroblob(:length) WHERE id = 1"
88
88
  @sql[:expand_docsize] = "UPDATE #{name}_docsize SET sz = sz || zeroblob(:length)"
89
- @sql[:ranks] = :ranks_sql
89
+ @sql[:ranks] = :ranks_sql
90
90
  @sql[:set_config_value] = "INSERT OR REPLACE INTO #{name}_config(k, v) VALUES (:key, :value)"
91
91
  @sql[:get_config_value] = "SELECT v FROM #{name}_config WHERE k = :key"
92
92
  @sql[:search] = "SELECT rowid AS id, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
93
+ @sql[:similarity_terms] = "SELECT DISTINCT term FROM #{name}_instance WHERE doc = :id AND FLOOR(term) IS NULL AND LENGTH(term) > 2 AND NOT instr(term, ' ') AND NOT instr(term, '-') AND NOT instr(term, ':') AND NOT instr(term, '#') AND NOT instr(term, '_') LIMIT 15"
94
+ @sql[:similarity_query] = "SELECT group_concat('\"' || term || '\"', ' OR ') FROM #{name}_row WHERE term IN (#{@sql[:similarity_terms]})"
95
+ @sql[:similarity_search] = "SELECT rowid AS id, -rank AS search_rank FROM #{name}(:term) WHERE rowid != :id ORDER BY rank LIMIT :limit"
96
+ @sql[:similar] = "SELECT rowid AS id, -rank AS search_rank FROM #{name} WHERE #{name} = (#{@sql[:similarity_query]}) AND rowid != :id ORDER BY rank LIMIT :limit"
93
97
  @sql[:update_index] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}'"
94
98
  @sql[:update_content_table] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}_content'"
95
99
  end
96
-
100
+
97
101
  private
98
-
99
- def ranks_sql(active=false)
100
- if active
101
- weights_sql = weights.join(', ')
102
+
103
+ def create_vocab_tables_sql
104
+ <<~SQL
105
+ CREATE VIRTUAL TABLE IF NOT EXISTS #{name}_row USING fts5vocab(#{name}, row);
106
+ CREATE VIRTUAL TABLE IF NOT EXISTS #{name}_instance USING fts5vocab(#{name}, instance);
107
+ SQL
108
+ end
109
+
110
+ def ranks_sql(active = false)
111
+ weights_sql = if active
112
+ weights.join(", ")
102
113
  else
103
- weights_sql = active_weights.join(', ')
114
+ active_weights.join(", ")
104
115
  end
105
116
  "INSERT INTO #{name}(#{name}, rank) VALUES ('rank', 'bm25(#{weights_sql})')"
106
117
  end
107
-
118
+
108
119
  def active_col_names_sql
109
- active_field_names.join(', ')
120
+ active_field_names.join(", ")
110
121
  end
111
-
122
+
112
123
  def active_col_names_var_sql
113
- ":#{active_field_names.join(', :')}"
124
+ ":#{active_field_names.join(", :")}"
114
125
  end
115
-
126
+
116
127
  def col_names_sql
117
- field_names.join(', ')
128
+ field_names.join(", ")
118
129
  end
119
-
130
+
120
131
  def col_names_var_sql
121
- ":#{field_names.join(', :')}"
132
+ ":#{field_names.join(", :")}"
122
133
  end
123
134
 
124
135
  def enrich_schema
125
136
  end
126
-
127
137
  end
128
-
@@ -1,17 +1,14 @@
1
1
  class Litesearch::Schema::ContentlessAdapter < Litesearch::Schema::BasicAdapter
2
-
3
2
  private
4
3
 
5
4
  def generate_sql
6
5
  super
7
- #@sql[:rebuild_index] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
8
- #@sql[:rebuild] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
6
+ # @sql[:rebuild_index] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
7
+ # @sql[:rebuild] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
9
8
  end
10
-
9
+
11
10
  def create_index_sql(active = false)
12
11
  col_names = active ? active_col_names_sql : col_names_sql
13
12
  "CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, content='', contentless_delete=1, tokenize='#{tokenizer_sql}')"
14
13
  end
15
-
16
14
  end
17
-
@@ -1,14 +1,14 @@
1
1
  class Litesearch::Schema::StandaloneAdapter < Litesearch::Schema::BasicAdapter
2
-
3
2
  def generate_sql
4
3
  super
5
4
  @sql[:move_content] = "ALTER TABLE #{name}_content RENAME TO #{name}_content_temp"
6
5
  @sql[:adjust_temp_content] = "UPDATE sqlite_schema SET sql (SELECT sql FROM sqlite_schema WHERE name = '#{name}_content') WHERE name = #{name}_content_temp"
7
6
  @sql[:restore_content] = "ALTER TABLE #{name}_content_temp RENAME TO #{name}_content"
8
7
  @sql[:rebuild] = "INSERT INTO #{name}(#{name}) VALUES ('rebuild')"
8
+ @sql[:similar] = "SELECT rowid AS id, *, -rank AS search_rank FROM #{name} WHERE #{name} = (#{@sql[:similarity_query]}) AND rowid != :id ORDER BY rank LIMIT :limit"
9
9
  @sql[:drop_content_table] = "DROP TABLE #{name}_content"
10
10
  @sql[:drop_content_col] = :drop_content_col_sql
11
- @sql[:create_content_table] = :create_content_table_sql
11
+ @sql[:create_content_table] = :create_content_table_sql
12
12
  @sql[:search] = "SELECT rowid AS id, *, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
13
13
  end
14
14
 
@@ -18,16 +18,14 @@ class Litesearch::Schema::StandaloneAdapter < Litesearch::Schema::BasicAdapter
18
18
  col_names = active ? active_col_names_sql : col_names_sql
19
19
  "CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, tokenize='#{tokenizer_sql}')"
20
20
  end
21
-
21
+
22
22
  def drop_content_col_sql(col_index)
23
23
  "ALTER TABLE #{name}_content DROP COLUMN c#{col_index}"
24
24
  end
25
-
25
+
26
26
  def create_content_table_sql(count)
27
- cols = []
28
- count.times{|i| cols << "c#{i}" }
29
- "CREATE TABLE #{name}_content(id INTEGER PRIMARY KEY, #{cols.join(', ')})"
27
+ cols = []
28
+ count.times { |i| cols << "c#{i}" }
29
+ "CREATE TABLE #{name}_content(id INTEGER PRIMARY KEY, #{cols.join(", ")})"
30
30
  end
31
-
32
31
  end
33
-
@@ -1,9 +1,4 @@
1
- require_relative './schema_adapters/basic_adapter.rb'
2
- require_relative './schema_adapters/standalone_adapter.rb'
3
- require_relative './schema_adapters/contentless_adapter.rb'
4
- require_relative './schema_adapters/backed_adapter.rb'
5
-
6
-
7
-
8
-
9
-
1
+ require_relative "./schema_adapters/basic_adapter"
2
+ require_relative "./schema_adapters/standalone_adapter"
3
+ require_relative "./schema_adapters/contentless_adapter"
4
+ require_relative "./schema_adapters/backed_adapter"