litestack 0.4.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.standard.yml +3 -0
  3. data/BENCHMARKS.md +23 -7
  4. data/CHANGELOG.md +11 -0
  5. data/Gemfile +1 -7
  6. data/Gemfile.lock +92 -0
  7. data/README.md +120 -6
  8. data/ROADMAP.md +45 -0
  9. data/Rakefile +3 -1
  10. data/WHYLITESTACK.md +1 -1
  11. data/assets/litecache_metrics.png +0 -0
  12. data/assets/litedb_metrics.png +0 -0
  13. data/assets/litemetric_logo_teal.png +0 -0
  14. data/assets/litesearch_logo_teal.png +0 -0
  15. data/bench/bench.rb +17 -10
  16. data/bench/bench_cache_rails.rb +10 -13
  17. data/bench/bench_cache_raw.rb +17 -22
  18. data/bench/bench_jobs_rails.rb +18 -12
  19. data/bench/bench_jobs_raw.rb +17 -10
  20. data/bench/bench_queue.rb +4 -6
  21. data/bench/rails_job.rb +5 -7
  22. data/bench/skjob.rb +4 -4
  23. data/bench/uljob.rb +6 -6
  24. data/lib/action_cable/subscription_adapter/litecable.rb +5 -8
  25. data/lib/active_job/queue_adapters/litejob_adapter.rb +6 -8
  26. data/lib/active_record/connection_adapters/litedb_adapter.rb +65 -75
  27. data/lib/active_support/cache/litecache.rb +38 -41
  28. data/lib/generators/litestack/install/install_generator.rb +3 -3
  29. data/lib/generators/litestack/install/templates/database.yml +7 -1
  30. data/lib/litestack/liteboard/liteboard.rb +269 -149
  31. data/lib/litestack/litecable.rb +41 -37
  32. data/lib/litestack/litecable.sql.yml +22 -11
  33. data/lib/litestack/litecache.rb +79 -88
  34. data/lib/litestack/litecache.sql.yml +81 -22
  35. data/lib/litestack/litecache.yml +1 -1
  36. data/lib/litestack/litedb.rb +35 -40
  37. data/lib/litestack/litejob.rb +30 -29
  38. data/lib/litestack/litejobqueue.rb +63 -65
  39. data/lib/litestack/litemetric.rb +80 -92
  40. data/lib/litestack/litemetric.sql.yml +244 -234
  41. data/lib/litestack/litemetric_collector.sql.yml +38 -41
  42. data/lib/litestack/litequeue.rb +39 -41
  43. data/lib/litestack/litequeue.sql.yml +39 -31
  44. data/lib/litestack/litescheduler.rb +15 -15
  45. data/lib/litestack/litesearch/index.rb +93 -63
  46. data/lib/litestack/litesearch/model.rb +66 -65
  47. data/lib/litestack/litesearch/schema.rb +53 -56
  48. data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +46 -50
  49. data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +44 -35
  50. data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +3 -6
  51. data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +7 -9
  52. data/lib/litestack/litesearch/schema_adapters.rb +4 -9
  53. data/lib/litestack/litesearch.rb +6 -9
  54. data/lib/litestack/litesupport.rb +76 -86
  55. data/lib/litestack/railtie.rb +1 -1
  56. data/lib/litestack/version.rb +2 -2
  57. data/lib/litestack.rb +6 -4
  58. data/lib/railties/rails/commands/dbconsole.rb +11 -15
  59. data/lib/sequel/adapters/litedb.rb +16 -21
  60. data/lib/sequel/adapters/shared/litedb.rb +168 -168
  61. data/scripts/build_metrics.rb +91 -0
  62. data/scripts/test_cable.rb +30 -0
  63. data/scripts/test_job_retry.rb +33 -0
  64. data/scripts/test_metrics.rb +60 -0
  65. data/template.rb +2 -2
  66. metadata +101 -6
@@ -1,13 +1,13 @@
1
1
  module Litesearch
2
2
  class Index; end
3
+
3
4
  class Schema; end
4
5
  end
5
6
 
6
- require_relative './litesearch/index'
7
- require_relative './litesearch/model'
7
+ require_relative "./litesearch/index"
8
+ require_relative "./litesearch/model"
8
9
 
9
10
  module Litesearch
10
-
11
11
  def litesearch_index_cache
12
12
  @litesearch_index_cache ||= {}
13
13
  end
@@ -21,17 +21,14 @@ module Litesearch
21
21
  return index if index && !block_given?
22
22
  # if either there is no index in the cache or a block is given
23
23
  # create a new index instance and then place it in the cache and return
24
- if block_given?
25
- index = Index.new(self, name) do |schema|
24
+ index = if block_given?
25
+ Index.new(self, name) do |schema|
26
26
  yield schema
27
27
  schema.name(name)
28
28
  end
29
29
  else
30
- index = Index.new(self, name)
30
+ Index.new(self, name)
31
31
  end
32
32
  litesearch_index_cache[name] = index
33
33
  end
34
-
35
34
  end
36
-
37
-
@@ -1,16 +1,15 @@
1
1
  # frozen_stringe_literal: true
2
2
 
3
- require 'sqlite3'
4
- require 'logger'
5
- require 'oj'
6
- require 'yaml'
7
- require 'pathname'
8
- require 'fileutils'
3
+ require "sqlite3"
4
+ require "logger"
5
+ require "oj"
6
+ require "yaml"
7
+ require "pathname"
8
+ require "fileutils"
9
9
 
10
10
  require_relative "./litescheduler"
11
11
 
12
12
  module Litesupport
13
-
14
13
  class Error < StandardError; end
15
14
 
16
15
  # Detect the Rack or Rails environment.
@@ -29,11 +28,11 @@ module Litesupport
29
28
  def self.environment
30
29
  @environment ||= detect_environment
31
30
  end
32
-
31
+
33
32
  # common db object options
34
33
  def self.create_db(path)
35
34
  db = SQLite3::Database.new(path)
36
- db.busy_handler{ Litescheduler.switch || sleep(0.0001) }
35
+ db.busy_handler { Litescheduler.switch || sleep(0.0001) }
37
36
  db.journal_mode = "WAL"
38
37
  db.instance_variable_set(:@stmts, {})
39
38
  class << db
@@ -43,12 +42,12 @@ module Litesupport
43
42
  end
44
43
 
45
44
  # Databases will be stored by default at this path.
46
- def self.root
47
- @root ||= ensure_root_volume detect_root
45
+ def self.root(env = Litesupport.environment)
46
+ ensure_root_volume detect_root(env)
48
47
  end
49
48
 
50
49
  # Default path where we'll store all of the databases.
51
- def self.detect_root
50
+ def self.detect_root(env)
52
51
  path = if ENV["LITESTACK_DATA_PATH"]
53
52
  ENV["LITESTACK_DATA_PATH"]
54
53
  elsif defined? Rails
@@ -57,7 +56,7 @@ module Litesupport
57
56
  "."
58
57
  end
59
58
 
60
- Pathname.new(path).join(Litesupport.environment)
59
+ Pathname.new(path).join(env)
61
60
  end
62
61
 
63
62
  def self.ensure_root_volume(path)
@@ -66,115 +65,106 @@ module Litesupport
66
65
  end
67
66
 
68
67
  class Mutex
69
-
70
68
  def initialize
71
69
  @mutex = Thread::Mutex.new
72
70
  end
73
-
71
+
74
72
  def synchronize(&block)
75
73
  if Litescheduler.backend == :threaded || Litescheduler.backend == :iodine
76
- @mutex.synchronize{ block.call }
74
+ @mutex.synchronize { block.call }
77
75
  else
78
76
  block.call
79
77
  end
80
78
  end
81
-
82
79
  end
83
-
80
+
84
81
  class Pool
85
-
86
82
  def initialize(count, &block)
87
83
  @count = count
88
84
  @block = block
89
85
  @resources = Thread::Queue.new
90
86
  @mutex = Litesupport::Mutex.new
91
87
  @count.times do
92
- resource = @mutex.synchronize{ block.call }
93
- @resources << resource
88
+ resource = @mutex.synchronize { block.call }
89
+ @resources << resource
94
90
  end
95
91
  end
96
-
92
+
97
93
  def acquire
98
94
  result = nil
99
95
  resource = @resources.pop
100
96
  begin
101
97
  result = yield resource
102
- rescue Exception => e
103
- raise e
104
98
  ensure
105
99
  @resources << resource
106
100
  end
107
101
  result
108
102
  end
109
-
110
103
  end
111
-
104
+
112
105
  module ForkListener
113
106
  def self.listeners
114
107
  @listeners ||= []
115
108
  end
116
-
109
+
117
110
  def self.listen(&block)
118
111
  listeners << block
119
112
  end
120
113
  end
121
114
 
122
115
  module Forkable
123
-
124
116
  def _fork(*args)
125
117
  ppid = Process.pid
126
118
  result = super
127
119
  if Process.pid != ppid && [:threaded, :iodine].include?(Litescheduler.backend)
128
- ForkListener.listeners.each{|l| l.call }
120
+ ForkListener.listeners.each { |l| l.call }
129
121
  end
130
122
  result
131
123
  end
132
-
133
124
  end
134
-
125
+
135
126
  module Liteconnection
136
-
137
127
  include Forkable
138
128
 
139
129
  # close, setup, run_stmt and run_sql assume a single connection was created
140
-
130
+
141
131
  def options
142
132
  @options
143
133
  end
144
-
134
+
145
135
  def close
146
136
  @running = false
147
- @conn.acquire do |q|
148
- q.stmts.each_pair {|k, v| q.stmts[k].close }
137
+ @conn.acquire do |q|
138
+ q.stmts.each_pair { |k, v| q.stmts[k].close }
149
139
  q.close
150
140
  end
151
141
  end
152
142
 
153
143
  def size
154
- run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0].to_f / (1024*1024)
144
+ run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0].to_f / (1024 * 1024)
155
145
  end
156
146
 
157
147
  def journal_mode
158
148
  run_method(:journal_mode)
159
149
  end
160
-
150
+
161
151
  def synchronous
162
152
  run_method(:synchronous)
163
153
  end
164
-
154
+
165
155
  def path
166
156
  run_method(:filename)
167
157
  end
168
-
158
+
169
159
  private # all methods are private
170
-
160
+
171
161
  def init(options = {})
172
- #c configure the object, loading options from the appropriate location
173
- configure(options)
162
+ # c configure the object, loading options from the appropriate location
163
+ configure(options)
174
164
  # setup connections and background threads
175
- setup
165
+ setup
176
166
  # handle process exiting
177
- at_exit do
167
+ at_exit do
178
168
  exit_callback
179
169
  end
180
170
  # handle forking (restart connections and background threads)
@@ -185,9 +175,17 @@ module Litesupport
185
175
 
186
176
  def configure(options = {})
187
177
  # detect enviornment (production, development, etc.)
188
- defaults = self.class::DEFAULT_OPTIONS rescue {}
178
+ defaults = begin
179
+ self.class::DEFAULT_OPTIONS
180
+ rescue
181
+ {}
182
+ end
189
183
  @options = defaults.merge(options)
190
- config = YAML.load_file(@options[:config_path]) rescue {} # an empty hash won't hurt
184
+ config = begin
185
+ YAML.load_file(@options[:config_path])
186
+ rescue
187
+ {}
188
+ end # an empty hash won't hurt
191
189
  config = config[Litesupport.environment] if config[Litesupport.environment] # if there is a config for the current enviornment defined then use it, otherwise use the top level declaration
192
190
  config.keys.each do |k| # symbolize keys
193
191
  config[k.to_sym] = config[k]
@@ -196,51 +194,50 @@ module Litesupport
196
194
  @options.merge!(config)
197
195
  @options.merge!(options) # make sure options passed to initialize trump everything else
198
196
  end
199
-
197
+
200
198
  def setup
201
199
  @conn = create_pooled_connection
202
200
  @logger = create_logger
203
201
  @running = true
204
202
  end
205
-
203
+
206
204
  def create_logger
207
205
  @options[:logger] = nil unless @options[:logger]
208
206
  return @options[:logger] if @options[:logger].respond_to? :info
209
- return Logger.new(STDOUT) if @options[:logger] == 'STDOUT'
210
- return Logger.new(STDERR) if @options[:logger] == 'STDERR'
211
- return Logger.new(@options[:logger]) if @options[:logger].is_a? String
212
- return Logger.new(IO::NULL)
207
+ return Logger.new($stdout) if @options[:logger] == "STDOUT"
208
+ return Logger.new($stderr) if @options[:logger] == "STDERR"
209
+ return Logger.new(@options[:logger]) if @options[:logger].is_a? String
210
+ Logger.new(IO::NULL)
213
211
  end
214
-
212
+
215
213
  def exit_callback
216
214
  close
217
215
  end
218
-
216
+
219
217
  def run_stmt(stmt, *args)
220
- @conn.acquire{|q| q.stmts[stmt].execute!(*args) }
218
+ @conn.acquire { |q| q.stmts[stmt].execute!(*args) }
221
219
  end
222
220
 
223
221
  def run_sql(sql, *args)
224
- @conn.acquire{|q| q.execute(sql, *args) }
222
+ @conn.acquire { |q| q.execute(sql, args) }
225
223
  end
226
-
224
+
227
225
  def run_method(method, *args)
228
- @conn.acquire{|q| q.send(method, *args)}
226
+ @conn.acquire { |q| q.send(method, *args) }
229
227
  end
230
228
 
231
229
  def run_stmt_method(stmt, method, *args)
232
- @conn.acquire{|q| q.stmts[stmt].send(method, *args)}
230
+ @conn.acquire { |q| q.stmts[stmt].send(method, *args) }
233
231
  end
234
232
 
235
-
236
233
  def create_pooled_connection(count = 1)
237
- Litesupport::Pool.new(1){create_connection}
234
+ Litesupport::Pool.new(1) { create_connection }
238
235
  end
239
236
 
240
237
  # common db object options
241
238
  def create_connection(path_to_sql_file = nil)
242
239
  conn = SQLite3::Database.new(@options[:path])
243
- conn.busy_handler{ Litescheduler.switch || sleep(rand * 0.002) }
240
+ conn.busy_handler { Litescheduler.switch || sleep(rand * 0.002) }
244
241
  conn.journal_mode = "WAL"
245
242
  conn.synchronous = @options[:sync] || 1
246
243
  conn.mmap_size = @options[:mmap_size] || 0
@@ -253,38 +250,31 @@ module Litesupport
253
250
  unless path_to_sql_file.nil?
254
251
  sql = YAML.load_file(path_to_sql_file)
255
252
  version = conn.get_first_value("PRAGMA user_version")
256
- sql["schema"].each_pair do |v, obj|
253
+ sql["schema"].each_pair do |v, obj|
257
254
  if v > version
258
- conn.transaction do
259
- obj.each do |k, s|
260
- begin
261
- conn.execute(s)
262
- rescue Exception => e
263
- STDERR.puts "Error parsing #{k}"
264
- STDERR.puts s
265
- raise e
266
- end
255
+ conn.transaction do
256
+ obj.each do |k, s|
257
+ conn.execute(s)
258
+ rescue Exception => e # standard:disable Lint/RescueException
259
+ warn "Error parsing #{k}"
260
+ warn s
261
+ raise e
267
262
  end
268
263
  conn.user_version = v
269
264
  end
270
265
  end
271
- end
272
- sql["stmts"].each do |k, v|
273
- begin
274
- conn.stmts[k.to_sym] = conn.prepare(v)
275
- rescue Exception => e
276
- STDERR.puts "Error parsing #{k}"
277
- STDERR.puts v
278
- raise e
279
- end
266
+ end
267
+ sql["stmts"].each do |k, v|
268
+ conn.stmts[k.to_sym] = conn.prepare(v)
269
+ rescue Exception => e # standard:disable Lint/RescueException
270
+ warn "Error parsing #{k}"
271
+ warn v
272
+ raise e
280
273
  end
281
274
  end
282
275
  conn
283
276
  end
284
-
285
277
  end
286
-
287
- end
278
+ end
288
279
 
289
280
  Process.singleton_class.prepend(Litesupport::Forkable)
290
-
@@ -7,4 +7,4 @@ module Litestack
7
7
  app.config.active_record.sqlite3_production_warning = false
8
8
  end
9
9
  end
10
- end
10
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Litestack
4
- VERSION = "0.4.1"
5
- end
4
+ VERSION = "0.4.2"
5
+ end
data/lib/litestack.rb CHANGED
@@ -13,14 +13,16 @@ require_relative "./litestack/litecable"
13
13
  # conditionally load integration with other libraries
14
14
  require_relative "./sequel/adapters/litedb" if defined? Sequel
15
15
  require_relative "./active_record/connection_adapters/litedb_adapter" if defined? ActiveRecord
16
- require_relative "./railties/rails/commands/dbconsole" if defined? Rails && defined? ActiveRecord
16
+ require_relative "./railties/rails/commands/dbconsole" if defined?(Rails) && defined?(ActiveRecord)
17
17
  require_relative "./active_support/cache/litecache" if defined? ActiveSupport
18
18
  require_relative "./active_job/queue_adapters/litejob_adapter" if defined? ActiveJob
19
19
  require_relative "./action_cable/subscription_adapter/litecable" if defined? ActionCable
20
20
  require_relative "./litestack/railtie" if defined? Rails::Railtie
21
21
 
22
22
  module Litestack
23
- class NotImplementedError < Exception; end
24
- class TimeoutError < Exception; end
25
- class DeadlockError < Exception; end
23
+ class NotImplementedError < RuntimeError; end
24
+
25
+ class TimeoutError < RuntimeError; end
26
+
27
+ class DeadlockError < RuntimeError; end
26
28
  end
@@ -1,6 +1,5 @@
1
1
  module Rails
2
2
  class DBConsole
3
-
4
3
  def start
5
4
  ENV["RAILS_ENV"] ||= @options[:environment] || environment
6
5
  config = db_config.configuration_hash
@@ -31,14 +30,14 @@ module Rails
31
30
  find_cmd_and_exec(["mysql", "mysql5"], *args)
32
31
 
33
32
  when /^postgres|^postgis/
34
- ENV["PGUSER"] = config[:username] if config[:username]
35
- ENV["PGHOST"] = config[:host] if config[:host]
36
- ENV["PGPORT"] = config[:port].to_s if config[:port]
37
- ENV["PGPASSWORD"] = config[:password].to_s if config[:password] && @options[:include_password]
38
- ENV["PGSSLMODE"] = config[:sslmode].to_s if config[:sslmode]
39
- ENV["PGSSLCERT"] = config[:sslcert].to_s if config[:sslcert]
40
- ENV["PGSSLKEY"] = config[:sslkey].to_s if config[:sslkey]
41
- ENV["PGSSLROOTCERT"] = config[:sslrootcert].to_s if config[:sslrootcert]
33
+ ENV["PGUSER"] = config[:username] if config[:username]
34
+ ENV["PGHOST"] = config[:host] if config[:host]
35
+ ENV["PGPORT"] = config[:port].to_s if config[:port]
36
+ ENV["PGPASSWORD"] = config[:password].to_s if config[:password] && @options[:include_password]
37
+ ENV["PGSSLMODE"] = config[:sslmode].to_s if config[:sslmode]
38
+ ENV["PGSSLCERT"] = config[:sslcert].to_s if config[:sslcert]
39
+ ENV["PGSSLKEY"] = config[:sslkey].to_s if config[:sslkey]
40
+ ENV["PGSSLROOTCERT"] = config[:sslrootcert].to_s if config[:sslrootcert]
42
41
  find_cmd_and_exec("psql", db_config.database)
43
42
 
44
43
  when "sqlite3", "litedb"
@@ -50,7 +49,6 @@ module Rails
50
49
 
51
50
  find_cmd_and_exec("sqlite3", *args)
52
51
 
53
-
54
52
  when "oracle", "oracle_enhanced"
55
53
  logon = ""
56
54
 
@@ -65,9 +63,9 @@ module Rails
65
63
  when "sqlserver"
66
64
  args = []
67
65
 
68
- args += ["-d", "#{db_config.database}"] if db_config.database
69
- args += ["-U", "#{config[:username]}"] if config[:username]
70
- args += ["-P", "#{config[:password]}"] if config[:password]
66
+ args += ["-d", db_config.database.to_s] if db_config.database
67
+ args += ["-U", config[:username].to_s] if config[:username]
68
+ args += ["-P", config[:password].to_s] if config[:password]
71
69
 
72
70
  if config[:host]
73
71
  host_arg = +"tcp:#{config[:host]}"
@@ -81,7 +79,5 @@ module Rails
81
79
  abort "Unknown command-line client for #{db_config.database}."
82
80
  end
83
81
  end
84
-
85
-
86
82
  end
87
83
  end
@@ -1,49 +1,44 @@
1
- require_relative '../../litestack/litedb'
2
- require 'sequel'
3
- require 'sequel/adapters/sqlite'
1
+ require_relative "../../litestack/litedb"
2
+ require "sequel"
3
+ require "sequel/adapters/sqlite"
4
4
 
5
5
  module Sequel
6
6
  module Litedb
7
-
8
- include SQLite
7
+ include SQLite
9
8
 
10
9
  LITEDB_TYPES = SQLITE_TYPES
11
-
10
+
12
11
  class Database < Sequel::SQLite::Database
13
-
14
12
  set_adapter_scheme :litedb
15
-
13
+
16
14
  def connect(server)
17
-
18
15
  Sequel.extension :fiber_concurrency if [:fiber, :polyphony].include? Litescheduler.backend
19
-
16
+
20
17
  opts = server_opts(server)
21
- opts[:database] = ':memory:' if blank_object?(opts[:database])
18
+ opts[:database] = ":memory:" if blank_object?(opts[:database])
22
19
  sqlite3_opts = {}
23
20
  sqlite3_opts[:readonly] = typecast_value_boolean(opts[:readonly]) if opts.has_key?(:readonly)
24
21
  db = ::Litedb.new(opts[:database].to_s, sqlite3_opts)
25
22
 
26
23
  self.transaction_mode = :immediate
27
-
28
- if sqlite_version >= 104
24
+
25
+ if sqlite_version >= 104
29
26
  db.extended_result_codes = true
30
27
  end
31
-
32
- connection_pragmas.each{|s| log_connection_yield(s, db){db.execute_batch(s)}}
33
-
28
+
29
+ connection_pragmas.each { |s| log_connection_yield(s, db) { db.execute_batch(s) } }
30
+
34
31
  class << db
35
32
  attr_reader :prepared_statements
36
33
  end
37
-
34
+
38
35
  db.instance_variable_set(:@prepared_statements, {})
39
36
  @raw_db = db
40
37
  db
41
38
  end
42
-
43
39
  end
44
-
45
- class Dataset < Sequel::SQLite::Dataset
40
+
41
+ class Dataset < Sequel::SQLite::Dataset
46
42
  end
47
-
48
43
  end
49
44
  end