litestack 0.4.1 → 0.4.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.standard.yml +3 -0
  3. data/BENCHMARKS.md +23 -7
  4. data/CHANGELOG.md +35 -0
  5. data/Gemfile +1 -7
  6. data/README.md +124 -6
  7. data/ROADMAP.md +45 -0
  8. data/Rakefile +3 -1
  9. data/WHYLITESTACK.md +1 -1
  10. data/assets/litecache_metrics.png +0 -0
  11. data/assets/litedb_metrics.png +0 -0
  12. data/assets/litemetric_logo_teal.png +0 -0
  13. data/assets/litesearch_logo_teal.png +0 -0
  14. data/bench/bench.rb +17 -10
  15. data/bench/bench_cache_rails.rb +45 -14
  16. data/bench/bench_cache_raw.rb +44 -28
  17. data/bench/bench_jobs_rails.rb +18 -12
  18. data/bench/bench_jobs_raw.rb +17 -10
  19. data/bench/bench_queue.rb +4 -6
  20. data/bench/rails_job.rb +5 -7
  21. data/bench/skjob.rb +4 -4
  22. data/bench/uljob.rb +6 -6
  23. data/bin/liteboard +2 -1
  24. data/lib/action_cable/subscription_adapter/litecable.rb +5 -8
  25. data/lib/active_job/queue_adapters/litejob_adapter.rb +6 -8
  26. data/lib/active_record/connection_adapters/litedb_adapter.rb +72 -84
  27. data/lib/active_support/cache/litecache.rb +61 -41
  28. data/lib/generators/litestack/install/install_generator.rb +3 -3
  29. data/lib/generators/litestack/install/templates/cable.yml +0 -3
  30. data/lib/generators/litestack/install/templates/database.yml +7 -1
  31. data/lib/litestack/liteboard/liteboard.rb +269 -149
  32. data/lib/litestack/litecable.rb +41 -37
  33. data/lib/litestack/litecable.sql.yml +22 -11
  34. data/lib/litestack/litecache.rb +118 -93
  35. data/lib/litestack/litecache.sql.yml +83 -22
  36. data/lib/litestack/litecache.yml +1 -1
  37. data/lib/litestack/litedb.rb +35 -40
  38. data/lib/litestack/litejob.rb +30 -29
  39. data/lib/litestack/litejobqueue.rb +63 -65
  40. data/lib/litestack/litemetric.rb +80 -92
  41. data/lib/litestack/litemetric.sql.yml +244 -234
  42. data/lib/litestack/litemetric_collector.sql.yml +38 -41
  43. data/lib/litestack/litequeue.rb +39 -41
  44. data/lib/litestack/litequeue.sql.yml +39 -31
  45. data/lib/litestack/litescheduler.rb +24 -18
  46. data/lib/litestack/litesearch/index.rb +93 -63
  47. data/lib/litestack/litesearch/model.rb +66 -65
  48. data/lib/litestack/litesearch/schema.rb +53 -56
  49. data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +46 -50
  50. data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +44 -35
  51. data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +3 -6
  52. data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +7 -9
  53. data/lib/litestack/litesearch/schema_adapters.rb +4 -9
  54. data/lib/litestack/litesearch.rb +6 -9
  55. data/lib/litestack/litesupport.rb +78 -87
  56. data/lib/litestack/railtie.rb +1 -1
  57. data/lib/litestack/version.rb +2 -2
  58. data/lib/litestack.rb +6 -4
  59. data/lib/railties/rails/commands/dbconsole.rb +16 -20
  60. data/lib/sequel/adapters/litedb.rb +16 -21
  61. data/lib/sequel/adapters/shared/litedb.rb +168 -168
  62. data/scripts/build_metrics.rb +91 -0
  63. data/scripts/test_cable.rb +30 -0
  64. data/scripts/test_job_retry.rb +33 -0
  65. data/scripts/test_metrics.rb +60 -0
  66. data/template.rb +2 -2
  67. metadata +115 -7
@@ -1,13 +1,13 @@
1
1
  module Litesearch
2
2
  class Index; end
3
+
3
4
  class Schema; end
4
5
  end
5
6
 
6
- require_relative './litesearch/index'
7
- require_relative './litesearch/model'
7
+ require_relative "./litesearch/index"
8
+ require_relative "./litesearch/model"
8
9
 
9
10
  module Litesearch
10
-
11
11
  def litesearch_index_cache
12
12
  @litesearch_index_cache ||= {}
13
13
  end
@@ -21,17 +21,14 @@ module Litesearch
21
21
  return index if index && !block_given?
22
22
  # if either there is no index in the cache or a block is given
23
23
  # create a new index instance and then place it in the cache and return
24
- if block_given?
25
- index = Index.new(self, name) do |schema|
24
+ index = if block_given?
25
+ Index.new(self, name) do |schema|
26
26
  yield schema
27
27
  schema.name(name)
28
28
  end
29
29
  else
30
- index = Index.new(self, name)
30
+ Index.new(self, name)
31
31
  end
32
32
  litesearch_index_cache[name] = index
33
33
  end
34
-
35
34
  end
36
-
37
-
@@ -1,16 +1,15 @@
1
1
  # frozen_stringe_literal: true
2
2
 
3
- require 'sqlite3'
4
- require 'logger'
5
- require 'oj'
6
- require 'yaml'
7
- require 'pathname'
8
- require 'fileutils'
3
+ require "sqlite3"
4
+ require "logger"
5
+ require "oj"
6
+ require "yaml"
7
+ require "pathname"
8
+ require "fileutils"
9
9
 
10
10
  require_relative "./litescheduler"
11
11
 
12
12
  module Litesupport
13
-
14
13
  class Error < StandardError; end
15
14
 
16
15
  # Detect the Rack or Rails environment.
@@ -29,11 +28,11 @@ module Litesupport
29
28
  def self.environment
30
29
  @environment ||= detect_environment
31
30
  end
32
-
31
+
33
32
  # common db object options
34
33
  def self.create_db(path)
35
34
  db = SQLite3::Database.new(path)
36
- db.busy_handler{ Litescheduler.switch || sleep(0.0001) }
35
+ db.busy_handler { Litescheduler.switch || sleep(0.0001) }
37
36
  db.journal_mode = "WAL"
38
37
  db.instance_variable_set(:@stmts, {})
39
38
  class << db
@@ -43,12 +42,12 @@ module Litesupport
43
42
  end
44
43
 
45
44
  # Databases will be stored by default at this path.
46
- def self.root
47
- @root ||= ensure_root_volume detect_root
45
+ def self.root(env = Litesupport.environment)
46
+ ensure_root_volume detect_root(env)
48
47
  end
49
48
 
50
49
  # Default path where we'll store all of the databases.
51
- def self.detect_root
50
+ def self.detect_root(env)
52
51
  path = if ENV["LITESTACK_DATA_PATH"]
53
52
  ENV["LITESTACK_DATA_PATH"]
54
53
  elsif defined? Rails
@@ -57,7 +56,7 @@ module Litesupport
57
56
  "."
58
57
  end
59
58
 
60
- Pathname.new(path).join(Litesupport.environment)
59
+ Pathname.new(path).join(env)
61
60
  end
62
61
 
63
62
  def self.ensure_root_volume(path)
@@ -66,115 +65,106 @@ module Litesupport
66
65
  end
67
66
 
68
67
  class Mutex
69
-
70
68
  def initialize
71
69
  @mutex = Thread::Mutex.new
72
70
  end
73
-
71
+
74
72
  def synchronize(&block)
75
73
  if Litescheduler.backend == :threaded || Litescheduler.backend == :iodine
76
- @mutex.synchronize{ block.call }
74
+ @mutex.synchronize { block.call }
77
75
  else
78
76
  block.call
79
77
  end
80
78
  end
81
-
82
79
  end
83
-
80
+
84
81
  class Pool
85
-
86
82
  def initialize(count, &block)
87
83
  @count = count
88
84
  @block = block
89
85
  @resources = Thread::Queue.new
90
86
  @mutex = Litesupport::Mutex.new
91
87
  @count.times do
92
- resource = @mutex.synchronize{ block.call }
93
- @resources << resource
88
+ resource = @mutex.synchronize { block.call }
89
+ @resources << resource
94
90
  end
95
91
  end
96
-
92
+
97
93
  def acquire
98
94
  result = nil
99
95
  resource = @resources.pop
100
96
  begin
101
97
  result = yield resource
102
- rescue Exception => e
103
- raise e
104
98
  ensure
105
99
  @resources << resource
106
100
  end
107
101
  result
108
102
  end
109
-
110
103
  end
111
-
104
+
112
105
  module ForkListener
113
106
  def self.listeners
114
107
  @listeners ||= []
115
108
  end
116
-
109
+
117
110
  def self.listen(&block)
118
111
  listeners << block
119
112
  end
120
113
  end
121
114
 
122
115
  module Forkable
123
-
124
116
  def _fork(*args)
125
117
  ppid = Process.pid
126
118
  result = super
127
119
  if Process.pid != ppid && [:threaded, :iodine].include?(Litescheduler.backend)
128
- ForkListener.listeners.each{|l| l.call }
120
+ ForkListener.listeners.each { |l| l.call }
129
121
  end
130
122
  result
131
123
  end
132
-
133
124
  end
134
-
125
+
135
126
  module Liteconnection
136
-
137
127
  include Forkable
138
128
 
139
129
  # close, setup, run_stmt and run_sql assume a single connection was created
140
-
130
+
141
131
  def options
142
132
  @options
143
133
  end
144
-
134
+
145
135
  def close
146
136
  @running = false
147
- @conn.acquire do |q|
148
- q.stmts.each_pair {|k, v| q.stmts[k].close }
137
+ @conn.acquire do |q|
138
+ q.stmts.each_pair { |k, v| q.stmts[k].close }
149
139
  q.close
150
140
  end
151
141
  end
152
142
 
153
143
  def size
154
- run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0].to_f / (1024*1024)
144
+ run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0].to_f / (1024 * 1024)
155
145
  end
156
146
 
157
147
  def journal_mode
158
148
  run_method(:journal_mode)
159
149
  end
160
-
150
+
161
151
  def synchronous
162
152
  run_method(:synchronous)
163
153
  end
164
-
154
+
165
155
  def path
166
156
  run_method(:filename)
167
157
  end
168
-
158
+
169
159
  private # all methods are private
170
-
160
+
171
161
  def init(options = {})
172
- #c configure the object, loading options from the appropriate location
173
- configure(options)
162
+ # c configure the object, loading options from the appropriate location
163
+ configure(options)
174
164
  # setup connections and background threads
175
- setup
165
+ setup
176
166
  # handle process exiting
177
- at_exit do
167
+ at_exit do
178
168
  exit_callback
179
169
  end
180
170
  # handle forking (restart connections and background threads)
@@ -185,9 +175,17 @@ module Litesupport
185
175
 
186
176
  def configure(options = {})
187
177
  # detect enviornment (production, development, etc.)
188
- defaults = self.class::DEFAULT_OPTIONS rescue {}
178
+ defaults = begin
179
+ self.class::DEFAULT_OPTIONS
180
+ rescue
181
+ {}
182
+ end
189
183
  @options = defaults.merge(options)
190
- config = YAML.load_file(@options[:config_path]) rescue {} # an empty hash won't hurt
184
+ config = begin
185
+ YAML.load_file(@options[:config_path])
186
+ rescue
187
+ {}
188
+ end # an empty hash won't hurt
191
189
  config = config[Litesupport.environment] if config[Litesupport.environment] # if there is a config for the current enviornment defined then use it, otherwise use the top level declaration
192
190
  config.keys.each do |k| # symbolize keys
193
191
  config[k.to_sym] = config[k]
@@ -196,51 +194,51 @@ module Litesupport
196
194
  @options.merge!(config)
197
195
  @options.merge!(options) # make sure options passed to initialize trump everything else
198
196
  end
199
-
197
+
200
198
  def setup
201
- @conn = create_pooled_connection
199
+ @conn = create_pooled_connection(@options[:connection_count])
202
200
  @logger = create_logger
203
201
  @running = true
204
202
  end
205
-
203
+
206
204
  def create_logger
207
205
  @options[:logger] = nil unless @options[:logger]
208
206
  return @options[:logger] if @options[:logger].respond_to? :info
209
- return Logger.new(STDOUT) if @options[:logger] == 'STDOUT'
210
- return Logger.new(STDERR) if @options[:logger] == 'STDERR'
211
- return Logger.new(@options[:logger]) if @options[:logger].is_a? String
212
- return Logger.new(IO::NULL)
207
+ return Logger.new($stdout) if @options[:logger] == "STDOUT"
208
+ return Logger.new($stderr) if @options[:logger] == "STDERR"
209
+ return Logger.new(@options[:logger]) if @options[:logger].is_a? String
210
+ Logger.new(IO::NULL)
213
211
  end
214
-
212
+
215
213
  def exit_callback
216
214
  close
217
215
  end
218
-
216
+
219
217
  def run_stmt(stmt, *args)
220
- @conn.acquire{|q| q.stmts[stmt].execute!(*args) }
218
+ @conn.acquire { |q| q.stmts[stmt].execute!(*args) }
221
219
  end
222
220
 
223
221
  def run_sql(sql, *args)
224
- @conn.acquire{|q| q.execute(sql, *args) }
222
+ @conn.acquire { |q| q.execute(sql, args) }
225
223
  end
226
-
224
+
227
225
  def run_method(method, *args)
228
- @conn.acquire{|q| q.send(method, *args)}
226
+ @conn.acquire { |q| q.send(method, *args) }
229
227
  end
230
228
 
231
229
  def run_stmt_method(stmt, method, *args)
232
- @conn.acquire{|q| q.stmts[stmt].send(method, *args)}
230
+ @conn.acquire { |q| q.stmts[stmt].send(method, *args) }
233
231
  end
234
232
 
235
-
236
233
  def create_pooled_connection(count = 1)
237
- Litesupport::Pool.new(1){create_connection}
234
+ count = 1 unless count and count.is_a? Integer
235
+ Litesupport::Pool.new(count) { create_connection }
238
236
  end
239
237
 
240
238
  # common db object options
241
239
  def create_connection(path_to_sql_file = nil)
242
240
  conn = SQLite3::Database.new(@options[:path])
243
- conn.busy_handler{ Litescheduler.switch || sleep(rand * 0.002) }
241
+ conn.busy_handler { Litescheduler.switch || sleep(rand * 0.002) }
244
242
  conn.journal_mode = "WAL"
245
243
  conn.synchronous = @options[:sync] || 1
246
244
  conn.mmap_size = @options[:mmap_size] || 0
@@ -253,38 +251,31 @@ module Litesupport
253
251
  unless path_to_sql_file.nil?
254
252
  sql = YAML.load_file(path_to_sql_file)
255
253
  version = conn.get_first_value("PRAGMA user_version")
256
- sql["schema"].each_pair do |v, obj|
254
+ sql["schema"].each_pair do |v, obj|
257
255
  if v > version
258
- conn.transaction do
259
- obj.each do |k, s|
260
- begin
261
- conn.execute(s)
262
- rescue Exception => e
263
- STDERR.puts "Error parsing #{k}"
264
- STDERR.puts s
265
- raise e
266
- end
256
+ conn.transaction do
257
+ obj.each do |k, s|
258
+ conn.execute(s)
259
+ rescue Exception => e # standard:disable Lint/RescueException
260
+ warn "Error parsing #{k}"
261
+ warn s
262
+ raise e
267
263
  end
268
264
  conn.user_version = v
269
265
  end
270
266
  end
271
- end
272
- sql["stmts"].each do |k, v|
273
- begin
274
- conn.stmts[k.to_sym] = conn.prepare(v)
275
- rescue Exception => e
276
- STDERR.puts "Error parsing #{k}"
277
- STDERR.puts v
278
- raise e
279
- end
267
+ end
268
+ sql["stmts"].each do |k, v|
269
+ conn.stmts[k.to_sym] = conn.prepare(v)
270
+ rescue Exception => e # standard:disable Lint/RescueException
271
+ warn "Error parsing #{k}"
272
+ warn v
273
+ raise e
280
274
  end
281
275
  end
282
276
  conn
283
277
  end
284
-
285
278
  end
286
-
287
- end
279
+ end
288
280
 
289
281
  Process.singleton_class.prepend(Litesupport::Forkable)
290
-
@@ -7,4 +7,4 @@ module Litestack
7
7
  app.config.active_record.sqlite3_production_warning = false
8
8
  end
9
9
  end
10
- end
10
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Litestack
4
- VERSION = "0.4.1"
5
- end
4
+ VERSION = "0.4.3"
5
+ end
data/lib/litestack.rb CHANGED
@@ -13,14 +13,16 @@ require_relative "./litestack/litecable"
13
13
  # conditionally load integration with other libraries
14
14
  require_relative "./sequel/adapters/litedb" if defined? Sequel
15
15
  require_relative "./active_record/connection_adapters/litedb_adapter" if defined? ActiveRecord
16
- require_relative "./railties/rails/commands/dbconsole" if defined? Rails && defined? ActiveRecord
16
+ require_relative "./railties/rails/commands/dbconsole" if defined?(Rails) && defined?(ActiveRecord)
17
17
  require_relative "./active_support/cache/litecache" if defined? ActiveSupport
18
18
  require_relative "./active_job/queue_adapters/litejob_adapter" if defined? ActiveJob
19
19
  require_relative "./action_cable/subscription_adapter/litecable" if defined? ActionCable
20
20
  require_relative "./litestack/railtie" if defined? Rails::Railtie
21
21
 
22
22
  module Litestack
23
- class NotImplementedError < Exception; end
24
- class TimeoutError < Exception; end
25
- class DeadlockError < Exception; end
23
+ class NotImplementedError < RuntimeError; end
24
+
25
+ class TimeoutError < RuntimeError; end
26
+
27
+ class DeadlockError < RuntimeError; end
26
28
  end
@@ -1,6 +1,5 @@
1
1
  module Rails
2
2
  class DBConsole
3
-
4
3
  def start
5
4
  ENV["RAILS_ENV"] ||= @options[:environment] || environment
6
5
  config = db_config.configuration_hash
@@ -28,18 +27,18 @@ module Rails
28
27
 
29
28
  args << db_config.database
30
29
 
31
- find_cmd_and_exec(["mysql", "mysql5"], *args)
30
+ ActiveRecord::Base.connection.class.find_cmd_and_exec(["mysql", "mysql5"], *args)
32
31
 
33
32
  when /^postgres|^postgis/
34
- ENV["PGUSER"] = config[:username] if config[:username]
35
- ENV["PGHOST"] = config[:host] if config[:host]
36
- ENV["PGPORT"] = config[:port].to_s if config[:port]
37
- ENV["PGPASSWORD"] = config[:password].to_s if config[:password] && @options[:include_password]
38
- ENV["PGSSLMODE"] = config[:sslmode].to_s if config[:sslmode]
39
- ENV["PGSSLCERT"] = config[:sslcert].to_s if config[:sslcert]
40
- ENV["PGSSLKEY"] = config[:sslkey].to_s if config[:sslkey]
41
- ENV["PGSSLROOTCERT"] = config[:sslrootcert].to_s if config[:sslrootcert]
42
- find_cmd_and_exec("psql", db_config.database)
33
+ ENV["PGUSER"] = config[:username] if config[:username]
34
+ ENV["PGHOST"] = config[:host] if config[:host]
35
+ ENV["PGPORT"] = config[:port].to_s if config[:port]
36
+ ENV["PGPASSWORD"] = config[:password].to_s if config[:password] && @options[:include_password]
37
+ ENV["PGSSLMODE"] = config[:sslmode].to_s if config[:sslmode]
38
+ ENV["PGSSLCERT"] = config[:sslcert].to_s if config[:sslcert]
39
+ ENV["PGSSLKEY"] = config[:sslkey].to_s if config[:sslkey]
40
+ ENV["PGSSLROOTCERT"] = config[:sslrootcert].to_s if config[:sslrootcert]
41
+ ActiveRecord::Base.connection.class.find_cmd_and_exec("psql", db_config.database)
43
42
 
44
43
  when "sqlite3", "litedb"
45
44
  args = []
@@ -48,8 +47,7 @@ module Rails
48
47
  args << "-header" if @options[:header]
49
48
  args << File.expand_path(db_config.database, Rails.respond_to?(:root) ? Rails.root : nil)
50
49
 
51
- find_cmd_and_exec("sqlite3", *args)
52
-
50
+ ActiveRecord::Base.connection.class.find_cmd_and_exec("sqlite3", *args)
53
51
 
54
52
  when "oracle", "oracle_enhanced"
55
53
  logon = ""
@@ -60,14 +58,14 @@ module Rails
60
58
  logon << "@#{db_config.database}" if db_config.database
61
59
  end
62
60
 
63
- find_cmd_and_exec("sqlplus", logon)
61
+ ActiveRecord::Base.connection.class.find_cmd_and_exec("sqlplus", logon)
64
62
 
65
63
  when "sqlserver"
66
64
  args = []
67
65
 
68
- args += ["-d", "#{db_config.database}"] if db_config.database
69
- args += ["-U", "#{config[:username]}"] if config[:username]
70
- args += ["-P", "#{config[:password]}"] if config[:password]
66
+ args += ["-d", db_config.database.to_s] if db_config.database
67
+ args += ["-U", config[:username].to_s] if config[:username]
68
+ args += ["-P", config[:password].to_s] if config[:password]
71
69
 
72
70
  if config[:host]
73
71
  host_arg = +"tcp:#{config[:host]}"
@@ -75,13 +73,11 @@ module Rails
75
73
  args += ["-S", host_arg]
76
74
  end
77
75
 
78
- find_cmd_and_exec("sqlcmd", *args)
76
+ ActiveRecord::Base.connection.class.find_cmd_and_exec("sqlcmd", *args)
79
77
 
80
78
  else
81
79
  abort "Unknown command-line client for #{db_config.database}."
82
80
  end
83
81
  end
84
-
85
-
86
82
  end
87
83
  end
@@ -1,49 +1,44 @@
1
- require_relative '../../litestack/litedb'
2
- require 'sequel'
3
- require 'sequel/adapters/sqlite'
1
+ require_relative "../../litestack/litedb"
2
+ require "sequel"
3
+ require "sequel/adapters/sqlite"
4
4
 
5
5
  module Sequel
6
6
  module Litedb
7
-
8
- include SQLite
7
+ include SQLite
9
8
 
10
9
  LITEDB_TYPES = SQLITE_TYPES
11
-
10
+
12
11
  class Database < Sequel::SQLite::Database
13
-
14
12
  set_adapter_scheme :litedb
15
-
13
+
16
14
  def connect(server)
17
-
18
15
  Sequel.extension :fiber_concurrency if [:fiber, :polyphony].include? Litescheduler.backend
19
-
16
+
20
17
  opts = server_opts(server)
21
- opts[:database] = ':memory:' if blank_object?(opts[:database])
18
+ opts[:database] = ":memory:" if blank_object?(opts[:database])
22
19
  sqlite3_opts = {}
23
20
  sqlite3_opts[:readonly] = typecast_value_boolean(opts[:readonly]) if opts.has_key?(:readonly)
24
21
  db = ::Litedb.new(opts[:database].to_s, sqlite3_opts)
25
22
 
26
23
  self.transaction_mode = :immediate
27
-
28
- if sqlite_version >= 104
24
+
25
+ if sqlite_version >= 104
29
26
  db.extended_result_codes = true
30
27
  end
31
-
32
- connection_pragmas.each{|s| log_connection_yield(s, db){db.execute_batch(s)}}
33
-
28
+
29
+ connection_pragmas.each { |s| log_connection_yield(s, db) { db.execute_batch(s) } }
30
+
34
31
  class << db
35
32
  attr_reader :prepared_statements
36
33
  end
37
-
34
+
38
35
  db.instance_variable_set(:@prepared_statements, {})
39
36
  @raw_db = db
40
37
  db
41
38
  end
42
-
43
39
  end
44
-
45
- class Dataset < Sequel::SQLite::Dataset
40
+
41
+ class Dataset < Sequel::SQLite::Dataset
46
42
  end
47
-
48
43
  end
49
44
  end