litestack 0.4.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.standard.yml +3 -0
  3. data/BENCHMARKS.md +23 -7
  4. data/CHANGELOG.md +11 -0
  5. data/Gemfile +1 -7
  6. data/Gemfile.lock +92 -0
  7. data/README.md +120 -6
  8. data/ROADMAP.md +45 -0
  9. data/Rakefile +3 -1
  10. data/WHYLITESTACK.md +1 -1
  11. data/assets/litecache_metrics.png +0 -0
  12. data/assets/litedb_metrics.png +0 -0
  13. data/assets/litemetric_logo_teal.png +0 -0
  14. data/assets/litesearch_logo_teal.png +0 -0
  15. data/bench/bench.rb +17 -10
  16. data/bench/bench_cache_rails.rb +10 -13
  17. data/bench/bench_cache_raw.rb +17 -22
  18. data/bench/bench_jobs_rails.rb +18 -12
  19. data/bench/bench_jobs_raw.rb +17 -10
  20. data/bench/bench_queue.rb +4 -6
  21. data/bench/rails_job.rb +5 -7
  22. data/bench/skjob.rb +4 -4
  23. data/bench/uljob.rb +6 -6
  24. data/lib/action_cable/subscription_adapter/litecable.rb +5 -8
  25. data/lib/active_job/queue_adapters/litejob_adapter.rb +6 -8
  26. data/lib/active_record/connection_adapters/litedb_adapter.rb +65 -75
  27. data/lib/active_support/cache/litecache.rb +38 -41
  28. data/lib/generators/litestack/install/install_generator.rb +3 -3
  29. data/lib/generators/litestack/install/templates/database.yml +7 -1
  30. data/lib/litestack/liteboard/liteboard.rb +269 -149
  31. data/lib/litestack/litecable.rb +41 -37
  32. data/lib/litestack/litecable.sql.yml +22 -11
  33. data/lib/litestack/litecache.rb +79 -88
  34. data/lib/litestack/litecache.sql.yml +81 -22
  35. data/lib/litestack/litecache.yml +1 -1
  36. data/lib/litestack/litedb.rb +35 -40
  37. data/lib/litestack/litejob.rb +30 -29
  38. data/lib/litestack/litejobqueue.rb +63 -65
  39. data/lib/litestack/litemetric.rb +80 -92
  40. data/lib/litestack/litemetric.sql.yml +244 -234
  41. data/lib/litestack/litemetric_collector.sql.yml +38 -41
  42. data/lib/litestack/litequeue.rb +39 -41
  43. data/lib/litestack/litequeue.sql.yml +39 -31
  44. data/lib/litestack/litescheduler.rb +15 -15
  45. data/lib/litestack/litesearch/index.rb +93 -63
  46. data/lib/litestack/litesearch/model.rb +66 -65
  47. data/lib/litestack/litesearch/schema.rb +53 -56
  48. data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +46 -50
  49. data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +44 -35
  50. data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +3 -6
  51. data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +7 -9
  52. data/lib/litestack/litesearch/schema_adapters.rb +4 -9
  53. data/lib/litestack/litesearch.rb +6 -9
  54. data/lib/litestack/litesupport.rb +76 -86
  55. data/lib/litestack/railtie.rb +1 -1
  56. data/lib/litestack/version.rb +2 -2
  57. data/lib/litestack.rb +6 -4
  58. data/lib/railties/rails/commands/dbconsole.rb +11 -15
  59. data/lib/sequel/adapters/litedb.rb +16 -21
  60. data/lib/sequel/adapters/shared/litedb.rb +168 -168
  61. data/scripts/build_metrics.rb +91 -0
  62. data/scripts/test_cable.rb +30 -0
  63. data/scripts/test_job_retry.rb +33 -0
  64. data/scripts/test_metrics.rb +60 -0
  65. data/template.rb +2 -2
  66. metadata +101 -6
@@ -1,37 +1,36 @@
1
- require_relative './schema_adapters.rb'
1
+ require_relative "./schema_adapters"
2
2
 
3
3
  class Litesearch::Schema
4
-
5
- TOKENIZERS = {
6
- porter: 'porter unicode61 remove_diacritics 2',
7
- unicode: 'unicode61 remove_diacritics 2',
8
- ascii: 'ascii',
9
- trigram: 'trigram'
4
+ TOKENIZERS = {
5
+ porter: "porter unicode61 remove_diacritics 2",
6
+ unicode: "unicode61 remove_diacritics 2",
7
+ ascii: "ascii",
8
+ trigram: "trigram"
10
9
  }
11
-
10
+
12
11
  INDEX_TYPES = {
13
- standalone: Litesearch::Schema::StandaloneAdapter,
14
- contentless: Litesearch::Schema::ContentlessAdapter,
12
+ standalone: Litesearch::Schema::StandaloneAdapter,
13
+ contentless: Litesearch::Schema::ContentlessAdapter,
15
14
  backed: Litesearch::Schema::BackedAdapter
16
15
  }
17
16
 
18
17
  DEFAULT_SCHEMA = {
19
18
  name: nil,
20
19
  type: :standalone,
21
- fields: nil,
20
+ fields: nil,
22
21
  table: nil,
23
22
  filter_column: nil,
24
23
  tokenizer: :porter,
25
24
  auto_create: true,
26
25
  auto_modify: true,
27
- rebuild_on_create: false,
28
- rebuild_on_modify: false
26
+ rebuild_on_create: false,
27
+ rebuild_on_modify: false
29
28
  }
30
29
 
31
30
  attr_accessor :schema
32
31
 
33
32
  def initialize(schema = {})
34
- @schema = schema #DEFAULT_SCHEMA.merge(schema)
33
+ @schema = schema # DEFAULT_SCHEMA.merge(schema)
35
34
  @schema[:fields] = {} unless @schema[:fields]
36
35
  end
37
36
 
@@ -39,7 +38,7 @@ class Litesearch::Schema
39
38
  def name(new_name)
40
39
  @schema[:name] = new_name
41
40
  end
42
-
41
+
43
42
  def type(new_type)
44
43
  raise "Unknown index type" if INDEX_TYPES[new_type].nil?
45
44
  @schema[:type] = new_type
@@ -50,38 +49,38 @@ class Litesearch::Schema
50
49
  end
51
50
 
52
51
  def fields(field_names)
53
- field_names.each {|f| field f }
52
+ field_names.each { |f| field f }
54
53
  end
55
-
54
+
56
55
  def field(name, attributes = {})
57
56
  name = name.to_s.downcase.to_sym
58
- attributes = {weight: 1}.merge(attributes).select{|k, v| allowed_attributes.include?(k)} # only allow attributes we know, to ease schema comparison later
57
+ attributes = {weight: 1}.merge(attributes).select { |k, v| allowed_attributes.include?(k) } # only allow attributes we know, to ease schema comparison later
59
58
  @schema[:fields][name] = attributes
60
59
  end
61
-
60
+
62
61
  def tokenizer(new_tokenizer)
63
62
  raise "Unknown tokenizer" if TOKENIZERS[new_tokenizer].nil?
64
- @schema[:tokenizer] = new_tokenizer
63
+ @schema[:tokenizer] = new_tokenizer
65
64
  end
66
-
65
+
67
66
  def filter_column(filter_column)
68
67
  @schema[:filter_column] = filter_column
69
68
  end
70
-
69
+
71
70
  def auto_create(boolean)
72
71
  @schema[:auto_create] = boolean
73
72
  end
74
-
73
+
75
74
  def auto_modify(boolean)
76
75
  @schema[:auto_modify] = boolean
77
76
  end
78
-
77
+
79
78
  def rebuild_on_create(boolean)
80
- @schema[:rebuild_on_create] = boolean
79
+ @schema[:rebuild_on_create] = boolean
81
80
  end
82
-
81
+
83
82
  def rebuild_on_modify(boolean)
84
- @schema[:rebuild_on_modify] = boolean
83
+ @schema[:rebuild_on_modify] = boolean
85
84
  end
86
85
 
87
86
  def post_init
@@ -89,51 +88,49 @@ class Litesearch::Schema
89
88
  end
90
89
 
91
90
  # schema sql generation API
92
-
91
+
93
92
  def sql_for(method, *args)
94
93
  adapter.sql_for(method, *args)
95
94
  end
96
-
95
+
97
96
  # schema data structure API
98
97
  def get(key)
99
98
  @schema[key]
100
- end
101
-
99
+ end
100
+
102
101
  def get_field(name)
103
102
  @schema[:fields][name]
104
103
  end
105
-
104
+
106
105
  def adapter
107
106
  @adapter ||= INDEX_TYPES[@schema[:type]].new(@schema)
108
107
  end
109
-
108
+
110
109
  def reset_sql
111
110
  adapter.generate_sql
112
111
  end
113
-
112
+
114
113
  def order_fields(old_schema)
115
114
  adapter.order_fields(old_schema)
116
115
  end
117
-
116
+
118
117
  # should we do this at the schema objects level?
119
118
  def compare(other_schema)
120
119
  other_schema = other_schema.schema
121
120
  # are the schemas identical?
122
121
  # 1 - same fields?
123
122
  [:type, :tokenizer, :name, :table].each do |key|
124
- other_schema[key] = @schema[key] if other_schema[key].nil?
123
+ other_schema[key] = @schema[key] if other_schema[key].nil?
125
124
  end
126
125
  if @schema[:type] != other_schema[:type]
127
126
  raise Litesearch::SchemaChangeException.new "Cannot change the index type, please drop the index before creating it again with the new type"
128
127
  end
129
- changes = { tokenizer: @schema[:tokenizer] != other_schema[:tokenizer], table: @schema[:table] != other_schema[:table], removed_fields_count: 0, filter_column: @schema[:filter_column] != other_schema[:filter_column] }
130
- #check tokenizer changes
128
+ changes = {tokenizer: @schema[:tokenizer] != other_schema[:tokenizer], table: @schema[:table] != other_schema[:table], removed_fields_count: 0, filter_column: @schema[:filter_column] != other_schema[:filter_column]}
129
+ # check tokenizer changes
131
130
  if changes[:tokenizer] && !other_schema[:rebuild_on_modify]
132
131
  raise Litesearch::SchemaChangeException.new "Cannot change the tokenizer without an index rebuild!"
133
132
  end
134
133
 
135
-
136
-
137
134
  # check field changes
138
135
  keys = @schema[:fields].keys.sort
139
136
  other_keys = other_schema[:fields].keys.sort
@@ -144,50 +141,50 @@ class Litesearch::Schema
144
141
  other_schema[:fields].delete(key)
145
142
  end
146
143
  end
147
-
144
+
148
145
  other_keys = other_schema[:fields].keys.sort
149
146
 
150
147
  changes[:fields] = keys != other_keys # only acceptable change is adding extra fields
151
- changes[:extra_fields_count] = other_keys.count - keys.count
148
+ changes[:extra_fields_count] = other_keys.count - keys.count
152
149
  # check for missing fields (please note that adding fields can work without a rebuild)
153
150
  if keys - other_keys != []
154
- raise Litesearch::SchemaChangeException.new "Missing fields from existing schema, they have to exist with weight zero until the next rebuild!"
151
+ raise Litesearch::SchemaChangeException.new "Missing fields from existing schema, they have to exist with weight zero until the next rebuild!"
155
152
  end
156
-
153
+
157
154
  # check field weights
158
- weights = keys.collect{|key| @schema[:fields][key][:weight] }
159
- other_weights = other_keys.collect{|key| other_schema[:fields][key][:weight] }
155
+ weights = keys.collect { |key| @schema[:fields][key][:weight] }
156
+ other_weights = other_keys.collect { |key| other_schema[:fields][key][:weight] }
160
157
  changes[:weights] = weights != other_weights # will always be true if fields are added
161
- if (removed_count = other_weights.select{|w| w == 0}.count) > 0
158
+ if (removed_count = other_weights.count { |w| w == 0 }) > 0
162
159
  changes[:removed_fields_count] = removed_count
163
160
  end
164
161
  # check field attributes, only backed tables have attributes
165
- attrs = keys.collect do |key|
162
+ attrs = keys.collect do |key|
166
163
  f = @schema[:fields][key].dup
167
164
  f.delete(:weight)
168
- f.select{|k,v| allowed_attributes.include? k }
165
+ f.select { |k, v| allowed_attributes.include? k }
169
166
  end
170
167
  other_attrs = other_keys.collect do |key|
171
168
  f = other_schema[:fields][key].dup
172
169
  f.delete(:weight)
173
- f.select{|k,v| allowed_attributes.include? k }
170
+ f.select { |k, v| allowed_attributes.include? k }
174
171
  end
175
172
  changes[:attributes] if other_attrs != attrs # this means that we will need to redefine the triggers if any are there and also the table definition if needed
176
-
173
+
177
174
  # return the changes
178
175
  changes
179
176
  end
180
-
177
+
181
178
  def clean
182
- removable = @schema[:fields].select{|name, f| f[:weight] == 0 }.collect{|name, f| name}
183
- removable.each{|name| @schema[:fields].delete(name)}
179
+ removable = @schema[:fields].select { |name, f| f[:weight] == 0 }.collect { |name, f| name }
180
+ removable.each { |name| @schema[:fields].delete(name) }
184
181
  end
185
-
182
+
186
183
  def allowed_attributes
187
184
  [:weight, :col, :target]
188
185
  end
189
-
190
186
  end
191
187
 
192
188
  class Litesearch::SchemaException < StandardError; end
189
+
193
190
  class Litesearch::SchemaChangeException < StandardError; end
@@ -1,5 +1,4 @@
1
1
  class Litesearch::Schema::BackedAdapter < Litesearch::Schema::ContentlessAdapter
2
-
3
2
  private
4
3
 
5
4
  def table
@@ -14,50 +13,50 @@ class Litesearch::Schema::BackedAdapter < Litesearch::Schema::ContentlessAdapter
14
13
  @sql[:create_primary_triggers] = :create_primary_triggers_sql
15
14
  @sql[:create_secondary_triggers] = :create_secondary_triggers_sql
16
15
  end
17
-
16
+
18
17
  def drop_primary_triggers_sql
19
- sql = <<-SQL
20
- DROP TRIGGER IF EXISTS #{name}_insert;
21
- DROP TRIGGER IF EXISTS #{name}_update;
22
- DROP TRIGGER IF EXISTS #{name}_update_not;
23
- DROP TRIGGER IF EXISTS #{name}_delete;
24
- SQL
18
+ <<~SQL
19
+ DROP TRIGGER IF EXISTS #{name}_insert;
20
+ DROP TRIGGER IF EXISTS #{name}_update;
21
+ DROP TRIGGER IF EXISTS #{name}_update_not;
22
+ DROP TRIGGER IF EXISTS #{name}_delete;
23
+ SQL
25
24
  end
26
-
27
- def create_primary_triggers_sql(active=false)
25
+
26
+ def create_primary_triggers_sql(active = false)
28
27
  when_stmt = "TRUE"
29
28
  cols = active_cols_names
30
- if filter = @schema[:filter_column]
29
+ if (filter = @schema[:filter_column])
31
30
  when_stmt = "NEW.#{filter} = TRUE"
32
31
  cols << filter
33
32
  end
34
33
 
35
- sql = <<-SQL
34
+ <<-SQL
36
35
  CREATE TRIGGER #{name}_insert AFTER INSERT ON #{table} WHEN #{when_stmt} BEGIN
37
- INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) VALUES (NEW.rowid, #{trigger_cols_sql});
36
+ INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) VALUES (NEW.rowid, #{trigger_cols_sql});
38
37
  END;
39
- CREATE TRIGGER #{name}_update AFTER UPDATE OF #{cols.join(', ')} ON #{table} WHEN #{when_stmt} BEGIN
40
- INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) VALUES (NEW.rowid, #{trigger_cols_sql});
38
+ CREATE TRIGGER #{name}_update AFTER UPDATE OF #{cols.join(", ")} ON #{table} WHEN #{when_stmt} BEGIN
39
+ INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) VALUES (NEW.rowid, #{trigger_cols_sql});
41
40
  END;
42
- CREATE TRIGGER #{name}_update_not AFTER UPDATE OF #{cols.join(', ')} ON #{table} WHEN NOT #{when_stmt} BEGIN
41
+ CREATE TRIGGER #{name}_update_not AFTER UPDATE OF #{cols.join(", ")} ON #{table} WHEN NOT #{when_stmt} BEGIN
43
42
  DELETE FROM #{name} WHERE rowid = NEW.rowid;
44
43
  END;
45
44
  CREATE TRIGGER #{name}_delete AFTER DELETE ON #{table} BEGIN
46
45
  DELETE FROM #{name} WHERE rowid = OLD.id;
47
46
  END;
48
- SQL
47
+ SQL
49
48
  end
50
-
49
+
51
50
  def drop_secondary_trigger_sql(target_table, target_col, col)
52
51
  "DROP TRIGGER IF EXISTS #{target_table}_#{target_col}_#{col}_#{name}_update;"
53
52
  end
54
-
53
+
55
54
  def create_secondary_trigger_sql(target_table, target_col, col)
56
- sql = <<-SQL
57
- CREATE TRIGGER #{target_table}_#{target_col}_#{col}_#{name}_update AFTER UPDATE OF #{target_col} ON #{target_table} BEGIN
58
- #{rebuild_sql} AND #{table}.#{col} = NEW.id;
59
- END;
60
- SQL
55
+ <<~SQL
56
+ CREATE TRIGGER #{target_table}_#{target_col}_#{col}_#{name}_update AFTER UPDATE OF #{target_col} ON #{target_table} BEGIN
57
+ #{rebuild_sql} AND #{table}.#{col} = NEW.id;
58
+ END;
59
+ SQL
61
60
  end
62
61
 
63
62
  def drop_secondary_triggers_sql
@@ -67,36 +66,34 @@ SQL
67
66
  sql << drop_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
68
67
  end
69
68
  end
70
- return sql.empty? ? nil : sql
69
+ sql.empty? ? nil : sql
71
70
  end
72
71
 
73
-
74
72
  def create_secondary_triggers_sql
75
- sql = ""
73
+ sql = ""
76
74
  @schema[:fields].each do |name, field|
77
75
  if field[:trigger_sql]
78
76
  sql << create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
79
77
  end
80
78
  end
81
- return sql.empty? ? nil : sql
79
+ sql.empty? ? nil : sql
82
80
  end
83
-
84
-
81
+
85
82
  def rebuild_sql
86
- conditions = ""
83
+ conditions = ""
87
84
  jcs = join_conditions_sql
88
85
  fs = filter_sql
89
86
  conditions = " ON #{jcs} #{fs}" unless jcs.empty? && fs.empty?
90
- "INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) SELECT #{table}.id, #{select_cols_sql} FROM #{join_tables_sql} #{conditions}"
87
+ "INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(", ")}) SELECT #{table}.id, #{select_cols_sql} FROM #{join_tables_sql} #{conditions}"
91
88
  end
92
-
89
+
93
90
  def enrich_schema
94
- @schema[:fields].each do |name, field|
95
- if field[:target] && ! field[:target].start_with?("#{table}.")
91
+ @schema[:fields].each do |name, field|
92
+ if field[:target] && !field[:target].start_with?("#{table}.")
96
93
  field[:target] = field[:target].downcase
97
- target_table, target_col = field[:target].split('.')
94
+ target_table, target_col = field[:target].split(".")
98
95
  field[:col] = "#{name}_id".to_sym unless field[:col]
99
- field[:target_table] = target_table.to_sym
96
+ field[:target_table] = target_table.to_sym
100
97
  field[:target_col] = target_col.to_sym
101
98
  field[:sql] = "(SELECT #{field[:target_col]} FROM #{field[:target_table]} WHERE id = NEW.#{field[:col]})"
102
99
  field[:trigger_sql] = true # create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
@@ -109,9 +106,9 @@ SQL
109
106
  end
110
107
  end
111
108
  end
112
-
109
+
113
110
  def filter_sql
114
- sql = ""
111
+ sql = ""
115
112
  sql << " AND #{@schema[:filter_column]} = TRUE " if @schema[:filter_column]
116
113
  sql
117
114
  end
@@ -121,27 +118,26 @@ SQL
121
118
  field[:trigger_sql] ? field[:sql] : "NEW.#{field[:sql]}"
122
119
  end.join(", ")
123
120
  end
124
-
121
+
125
122
  def select_cols_sql
126
123
  active_fields.collect do |name, field|
127
- field[:trigger_sql] != nil ? "#{field[:target_table_alias]}.#{field[:target_col]}" : field[:target]
128
- end.join(', ')
124
+ (!field[:trigger_sql].nil?) ? "#{field[:target_table_alias]}.#{field[:target_col]}" : field[:target]
125
+ end.join(", ")
129
126
  end
130
-
127
+
131
128
  def join_tables_sql
132
- tables = [@schema[:table]]
129
+ tables = [@schema[:table]]
133
130
  active_fields.each do |name, field|
134
- tables << "#{field[:target_table]} AS #{field[:target_table_alias]}" if field[:trigger_sql]
131
+ tables << "#{field[:target_table]} AS #{field[:target_table_alias]}" if field[:trigger_sql]
135
132
  end
136
133
  tables.uniq.join(", ")
137
134
  end
138
-
135
+
139
136
  def join_conditions_sql
140
137
  conditions = []
141
138
  active_fields.each do |name, field|
142
- conditions << "#{field[:target_table_alias]}.id = #{@schema[:table]}.#{field[:col]}" if field[:trigger_sql]
139
+ conditions << "#{field[:target_table_alias]}.id = #{@schema[:table]}.#{field[:col]}" if field[:trigger_sql]
143
140
  end
144
- conditions.join(" AND ")
145
- end
146
-
141
+ conditions.join(" AND ")
142
+ end
147
143
  end
@@ -1,16 +1,15 @@
1
1
  class Litesearch::Schema::BasicAdapter
2
-
3
2
  def initialize(schema)
4
3
  @schema = schema
5
4
  @sql = {}
6
5
  enrich_schema
7
6
  generate_sql
8
7
  end
9
-
8
+
10
9
  def name
11
10
  @schema[:name]
12
11
  end
13
-
12
+
14
13
  def table
15
14
  @schema[:table]
16
15
  end
@@ -22,25 +21,25 @@ class Litesearch::Schema::BasicAdapter
22
21
  def field_names
23
22
  @schema[:fields].keys
24
23
  end
25
-
24
+
26
25
  def active_fields
27
- @schema[:fields].select{|k, v| v[:weight] != 0 }
26
+ @schema[:fields].select { |k, v| v[:weight] != 0 }
28
27
  end
29
-
28
+
30
29
  def active_field_names
31
30
  active_fields.keys
32
31
  end
33
-
32
+
34
33
  def active_cols_names
35
- active_fields.collect{|k, v| v[:col]}
34
+ active_fields.collect { |k, v| v[:col] }
36
35
  end
37
-
36
+
38
37
  def weights
39
- @schema[:fields].values.collect{|v| v[:weight].to_f }
38
+ @schema[:fields].values.collect { |v| v[:weight].to_f }
40
39
  end
41
40
 
42
41
  def active_weights
43
- active_fields.values.collect{|v| v[:weight].to_f }
42
+ active_fields.values.collect { |v| v[:weight].to_f }
44
43
  end
45
44
 
46
45
  def tokenizer_sql
@@ -51,33 +50,34 @@ class Litesearch::Schema::BasicAdapter
51
50
  new_fields = {}
52
51
  old_field_names = old_schema.schema[:fields].keys
53
52
  old_field_names.each do |name|
54
- new_fields[name] = @schema[:fields].delete(name)
53
+ new_fields[name] = @schema[:fields].delete(name)
55
54
  end
56
55
  missing_field_names = field_names - old_field_names
57
56
  missing_field_names.each do |name|
58
57
  new_fields[name] = @schema[:fields].delete(name)
59
- end
58
+ end
60
59
  @schema[:fields] = new_fields # this should be in order now
61
60
  generate_sql
62
61
  enrich_schema
63
62
  end
64
63
 
65
64
  def sql_for(method, *args)
66
- if sql = @sql[method]
65
+ if (sql = @sql[method])
67
66
  if sql.is_a? String
68
- return sql
67
+ sql
69
68
  elsif sql.is_a? Proc
70
- return sql.call(*args)
69
+ sql.call(*args)
71
70
  elsif sql.is_a? Symbol
72
- return self.send(sql, *args)
71
+ send(sql, *args)
73
72
  elsif sql.is_a? Litesearch::SchemaChangeException
74
73
  raise sql
75
74
  end
76
75
  end
77
76
  end
78
-
77
+
79
78
  def generate_sql
80
79
  @sql[:create_index] = :create_index_sql
80
+ @sql[:create_vocab_tables] = :create_vocab_tables_sql
81
81
  @sql[:insert] = "INSERT OR REPLACE INTO #{name}(rowid, #{active_col_names_sql}) VALUES (:id, #{active_col_names_var_sql}) RETURNING rowid"
82
82
  @sql[:delete] = "DELETE FROM #{name} WHERE rowid = :id"
83
83
  @sql[:count] = "SELECT count(*) FROM #{name}(:term)"
@@ -86,43 +86,52 @@ class Litesearch::Schema::BasicAdapter
86
86
  @sql[:drop] = "DROP TABLE #{name}"
87
87
  @sql[:expand_data] = "UPDATE #{name}_data SET block = block || zeroblob(:length) WHERE id = 1"
88
88
  @sql[:expand_docsize] = "UPDATE #{name}_docsize SET sz = sz || zeroblob(:length)"
89
- @sql[:ranks] = :ranks_sql
89
+ @sql[:ranks] = :ranks_sql
90
90
  @sql[:set_config_value] = "INSERT OR REPLACE INTO #{name}_config(k, v) VALUES (:key, :value)"
91
91
  @sql[:get_config_value] = "SELECT v FROM #{name}_config WHERE k = :key"
92
92
  @sql[:search] = "SELECT rowid AS id, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
93
+ @sql[:similarity_terms] = "SELECT DISTINCT term FROM #{name}_instance WHERE doc = :id AND FLOOR(term) IS NULL AND LENGTH(term) > 2 AND NOT instr(term, ' ') AND NOT instr(term, '-') AND NOT instr(term, ':') AND NOT instr(term, '#') AND NOT instr(term, '_') LIMIT 15"
94
+ @sql[:similarity_query] = "SELECT group_concat('\"' || term || '\"', ' OR ') FROM #{name}_row WHERE term IN (#{@sql[:similarity_terms]})"
95
+ @sql[:similarity_search] = "SELECT rowid AS id, -rank AS search_rank FROM #{name}(:term) WHERE rowid != :id ORDER BY rank LIMIT :limit"
96
+ @sql[:similar] = "SELECT rowid AS id, -rank AS search_rank FROM #{name} WHERE #{name} = (#{@sql[:similarity_query]}) AND rowid != :id ORDER BY rank LIMIT :limit"
93
97
  @sql[:update_index] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}'"
94
98
  @sql[:update_content_table] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}_content'"
95
99
  end
96
-
100
+
97
101
  private
98
-
99
- def ranks_sql(active=false)
100
- if active
101
- weights_sql = weights.join(', ')
102
+
103
+ def create_vocab_tables_sql
104
+ <<~SQL
105
+ CREATE VIRTUAL TABLE IF NOT EXISTS #{name}_row USING fts5vocab(#{name}, row);
106
+ CREATE VIRTUAL TABLE IF NOT EXISTS #{name}_instance USING fts5vocab(#{name}, instance);
107
+ SQL
108
+ end
109
+
110
+ def ranks_sql(active = false)
111
+ weights_sql = if active
112
+ weights.join(", ")
102
113
  else
103
- weights_sql = active_weights.join(', ')
114
+ active_weights.join(", ")
104
115
  end
105
116
  "INSERT INTO #{name}(#{name}, rank) VALUES ('rank', 'bm25(#{weights_sql})')"
106
117
  end
107
-
118
+
108
119
  def active_col_names_sql
109
- active_field_names.join(', ')
120
+ active_field_names.join(", ")
110
121
  end
111
-
122
+
112
123
  def active_col_names_var_sql
113
- ":#{active_field_names.join(', :')}"
124
+ ":#{active_field_names.join(", :")}"
114
125
  end
115
-
126
+
116
127
  def col_names_sql
117
- field_names.join(', ')
128
+ field_names.join(", ")
118
129
  end
119
-
130
+
120
131
  def col_names_var_sql
121
- ":#{field_names.join(', :')}"
132
+ ":#{field_names.join(", :")}"
122
133
  end
123
134
 
124
135
  def enrich_schema
125
136
  end
126
-
127
137
  end
128
-
@@ -1,17 +1,14 @@
1
1
  class Litesearch::Schema::ContentlessAdapter < Litesearch::Schema::BasicAdapter
2
-
3
2
  private
4
3
 
5
4
  def generate_sql
6
5
  super
7
- #@sql[:rebuild_index] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
8
- #@sql[:rebuild] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
6
+ # @sql[:rebuild_index] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
7
+ # @sql[:rebuild] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
9
8
  end
10
-
9
+
11
10
  def create_index_sql(active = false)
12
11
  col_names = active ? active_col_names_sql : col_names_sql
13
12
  "CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, content='', contentless_delete=1, tokenize='#{tokenizer_sql}')"
14
13
  end
15
-
16
14
  end
17
-
@@ -1,14 +1,14 @@
1
1
  class Litesearch::Schema::StandaloneAdapter < Litesearch::Schema::BasicAdapter
2
-
3
2
  def generate_sql
4
3
  super
5
4
  @sql[:move_content] = "ALTER TABLE #{name}_content RENAME TO #{name}_content_temp"
6
5
  @sql[:adjust_temp_content] = "UPDATE sqlite_schema SET sql (SELECT sql FROM sqlite_schema WHERE name = '#{name}_content') WHERE name = #{name}_content_temp"
7
6
  @sql[:restore_content] = "ALTER TABLE #{name}_content_temp RENAME TO #{name}_content"
8
7
  @sql[:rebuild] = "INSERT INTO #{name}(#{name}) VALUES ('rebuild')"
8
+ @sql[:similar] = "SELECT rowid AS id, *, -rank AS search_rank FROM #{name} WHERE #{name} = (#{@sql[:similarity_query]}) AND rowid != :id ORDER BY rank LIMIT :limit"
9
9
  @sql[:drop_content_table] = "DROP TABLE #{name}_content"
10
10
  @sql[:drop_content_col] = :drop_content_col_sql
11
- @sql[:create_content_table] = :create_content_table_sql
11
+ @sql[:create_content_table] = :create_content_table_sql
12
12
  @sql[:search] = "SELECT rowid AS id, *, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
13
13
  end
14
14
 
@@ -18,16 +18,14 @@ class Litesearch::Schema::StandaloneAdapter < Litesearch::Schema::BasicAdapter
18
18
  col_names = active ? active_col_names_sql : col_names_sql
19
19
  "CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, tokenize='#{tokenizer_sql}')"
20
20
  end
21
-
21
+
22
22
  def drop_content_col_sql(col_index)
23
23
  "ALTER TABLE #{name}_content DROP COLUMN c#{col_index}"
24
24
  end
25
-
25
+
26
26
  def create_content_table_sql(count)
27
- cols = []
28
- count.times{|i| cols << "c#{i}" }
29
- "CREATE TABLE #{name}_content(id INTEGER PRIMARY KEY, #{cols.join(', ')})"
27
+ cols = []
28
+ count.times { |i| cols << "c#{i}" }
29
+ "CREATE TABLE #{name}_content(id INTEGER PRIMARY KEY, #{cols.join(", ")})"
30
30
  end
31
-
32
31
  end
33
-
@@ -1,9 +1,4 @@
1
- require_relative './schema_adapters/basic_adapter.rb'
2
- require_relative './schema_adapters/standalone_adapter.rb'
3
- require_relative './schema_adapters/contentless_adapter.rb'
4
- require_relative './schema_adapters/backed_adapter.rb'
5
-
6
-
7
-
8
-
9
-
1
+ require_relative "./schema_adapters/basic_adapter"
2
+ require_relative "./schema_adapters/standalone_adapter"
3
+ require_relative "./schema_adapters/contentless_adapter"
4
+ require_relative "./schema_adapters/backed_adapter"