litestack 0.4.1 → 0.4.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.standard.yml +3 -0
  3. data/BENCHMARKS.md +23 -7
  4. data/CHANGELOG.md +35 -0
  5. data/Gemfile +1 -7
  6. data/README.md +124 -6
  7. data/ROADMAP.md +45 -0
  8. data/Rakefile +3 -1
  9. data/WHYLITESTACK.md +1 -1
  10. data/assets/litecache_metrics.png +0 -0
  11. data/assets/litedb_metrics.png +0 -0
  12. data/assets/litemetric_logo_teal.png +0 -0
  13. data/assets/litesearch_logo_teal.png +0 -0
  14. data/bench/bench.rb +17 -10
  15. data/bench/bench_cache_rails.rb +45 -14
  16. data/bench/bench_cache_raw.rb +44 -28
  17. data/bench/bench_jobs_rails.rb +18 -12
  18. data/bench/bench_jobs_raw.rb +17 -10
  19. data/bench/bench_queue.rb +4 -6
  20. data/bench/rails_job.rb +5 -7
  21. data/bench/skjob.rb +4 -4
  22. data/bench/uljob.rb +6 -6
  23. data/bin/liteboard +2 -1
  24. data/lib/action_cable/subscription_adapter/litecable.rb +5 -8
  25. data/lib/active_job/queue_adapters/litejob_adapter.rb +6 -8
  26. data/lib/active_record/connection_adapters/litedb_adapter.rb +72 -84
  27. data/lib/active_support/cache/litecache.rb +61 -41
  28. data/lib/generators/litestack/install/install_generator.rb +3 -3
  29. data/lib/generators/litestack/install/templates/cable.yml +0 -3
  30. data/lib/generators/litestack/install/templates/database.yml +7 -1
  31. data/lib/litestack/liteboard/liteboard.rb +269 -149
  32. data/lib/litestack/litecable.rb +41 -37
  33. data/lib/litestack/litecable.sql.yml +22 -11
  34. data/lib/litestack/litecache.rb +118 -93
  35. data/lib/litestack/litecache.sql.yml +83 -22
  36. data/lib/litestack/litecache.yml +1 -1
  37. data/lib/litestack/litedb.rb +35 -40
  38. data/lib/litestack/litejob.rb +30 -29
  39. data/lib/litestack/litejobqueue.rb +63 -65
  40. data/lib/litestack/litemetric.rb +80 -92
  41. data/lib/litestack/litemetric.sql.yml +244 -234
  42. data/lib/litestack/litemetric_collector.sql.yml +38 -41
  43. data/lib/litestack/litequeue.rb +39 -41
  44. data/lib/litestack/litequeue.sql.yml +39 -31
  45. data/lib/litestack/litescheduler.rb +24 -18
  46. data/lib/litestack/litesearch/index.rb +93 -63
  47. data/lib/litestack/litesearch/model.rb +66 -65
  48. data/lib/litestack/litesearch/schema.rb +53 -56
  49. data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +46 -50
  50. data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +44 -35
  51. data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +3 -6
  52. data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +7 -9
  53. data/lib/litestack/litesearch/schema_adapters.rb +4 -9
  54. data/lib/litestack/litesearch.rb +6 -9
  55. data/lib/litestack/litesupport.rb +78 -87
  56. data/lib/litestack/railtie.rb +1 -1
  57. data/lib/litestack/version.rb +2 -2
  58. data/lib/litestack.rb +6 -4
  59. data/lib/railties/rails/commands/dbconsole.rb +16 -20
  60. data/lib/sequel/adapters/litedb.rb +16 -21
  61. data/lib/sequel/adapters/shared/litedb.rb +168 -168
  62. data/scripts/build_metrics.rb +91 -0
  63. data/scripts/test_cable.rb +30 -0
  64. data/scripts/test_job_retry.rb +33 -0
  65. data/scripts/test_metrics.rb +60 -0
  66. data/template.rb +2 -2
  67. metadata +115 -7
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 11ef03890b2883bd21fb443d959774f9932f56e927d2ee2a40a028713526ce9c
4
- data.tar.gz: c531925eeffa84973475c14d4d230d8919d5e5ec095e5055f9bab6b0f94519e3
3
+ metadata.gz: e2ac02213caf3525c40b676550a6daa24380f3012a5d4c8f8042cf39aa477593
4
+ data.tar.gz: 00d0f60e20bcba2cc22704d4e7159fe9b819920d313e7dc7d4e4e799420e95ca
5
5
  SHA512:
6
- metadata.gz: 82c0878fb57fa89290c6550ddbf5b393e436883252285dd78083ba7e96f78947aa3fdf3e92d96f3baec6b9b2b0d2edfdc470f4e1c544427f3da8db704179de27
7
- data.tar.gz: 358bc249c3f1371714e12f4df0a0d82883e42eec14528f4faaef8a5e611e62af1039d8c00ebdc9d89be1a514a3bfd698ce3f988fa1b4f29eb1cf72fd1ca03b25
6
+ metadata.gz: 7c9c726aecbfd99ad73d82c50c285178becefb4792ece0672b55dfede10a47f0563700a691398373f372ca829491c13e0332a81b9b075f9a0435ae55ac138a1c
7
+ data.tar.gz: b4a28527632a3065d0deb8eadb5c806d260edfd89aa9b3f00ebbc20a2aa63ebfb7df907e7bd0a500f9e368ecd91b8967789b1f056632ef5bc73502dbbe1524c9
data/.standard.yml ADDED
@@ -0,0 +1,3 @@
1
+ ignore:
2
+ - 'test/**/*':
3
+ - Style/GlobalVars
data/BENCHMARKS.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Litestack Benchmarks
2
2
 
3
- This is a set of initial (simple) benchmars, designed to understand the baseline performance for different litestack components against their counterparts.
3
+ This is a set of initial (simple) benchmarks, designed to understand the baseline performance for different litestack components against their counterparts.
4
4
  These are not real life scenarios and I hope I will be able to produce some interesting ones soon.
5
5
 
6
6
  All these benchmarks were run on an 8 core, 16 thread, AMD 5700U based laptop, in a Virtual Box VM
@@ -18,7 +18,7 @@ This produces
18
18
  SELECT * FROM posts WHERE id = ?
19
19
  ```
20
20
 
21
- |Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
21
+ |Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
22
22
  |-:|-:|-:|-:|-:|
23
23
  |1|1.3K q/s|6.5K q/s|1.8K q/s|17.4K q/s|
24
24
  |2|2.6K q/s|13.9K q/s|3.5K q/s|33.2K q/s|
@@ -35,7 +35,7 @@ This produces
35
35
  SELECT * FROM posts WHERE user_id = ? LIMIT 5
36
36
  ```
37
37
 
38
- |Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
38
+ |Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
39
39
  |-:|-:|-:|-:|-:|
40
40
  |1|345 q/s|482 q/s|937 q/s|1.1K q/s|
41
41
  |2|751 q/s|848 q/s|1.3K q/s|2.3K q/s|
@@ -53,7 +53,7 @@ This produces
53
53
  Update posts SET updated_at = ? WHERE id = ?
54
54
  ```
55
55
 
56
- |Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
56
+ |Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
57
57
  |-:|-:|-:|-:|-:|
58
58
  |1|125 q/s|484 q/s|129 q/s|2.1K q/s|
59
59
  |2|265 q/s|576 q/s|333 q/s|2.5K q/s|
@@ -78,7 +78,7 @@ For testing the cache we attempted to try writing and reading different payload
78
78
 
79
79
  ### Read
80
80
 
81
- |Payload Size (bytes)|Redis|litecahce|
81
+ |Payload Size (bytes)|Redis|litecache|
82
82
  |-:|-:|-:|
83
83
  |10|5.0K q/s|69.4K q/s|
84
84
  |100|5.0K q/s|90.7K q/s|
@@ -87,11 +87,11 @@ For testing the cache we attempted to try writing and reading different payload
87
87
 
88
88
  ### Increment an int value
89
89
 
90
- |Redis|litecahce|
90
+ |Redis|litecache|
91
91
  |-:|-:|
92
92
  |5.1K q/s|16.9K q/s|
93
93
 
94
- It is not even a contest! litecache delivers way higher peroformance, specially in reading performance which is arguably the most important metric for a cache.
94
+ It is not even a contest! litecache delivers way higher performance, specially in reading performance which is arguably the most important metric for a cache.
95
95
 
96
96
  > ![litejob](https://github.com/oldmoe/litestack/blob/master/assets/litejob_logo_teal.png?raw=true)
97
97
 
@@ -122,3 +122,19 @@ A client written using the Iodine web server was used to generate the WS load in
122
122
  |100,000|3403|5385|41|36|153|235
123
123
 
124
124
  On average, Litecable is quite faster than the Redis based version and offers better latenices for over 90% of the requests, though Redis usually delivers better p99 latencies,
125
+
126
+ > ![litesearch](https://github.com/oldmoe/litestack/blob/master/assets/litesearch_logo_teal.png?raw=true)
127
+
128
+ Litesearch was benchmarked against Mielsearch, both using their respective ActiveRecord integrations. Mielsearch was running on the same machine as the benchmark script and was using the default configuration options. The dataset used for testing was the infamous Enron email corpus. Redisearch was not benchmarked due to the clients being not Rails 7.1 compatible (yet), will probably bench Redisearch when they are.
129
+
130
+ ### Building the index
131
+
132
+ ||MieliSearch|Litesearch|
133
+ |-:|-:|-:|
134
+ |Time to insert 10K docs|265.42 seconds|29.06 seconds|
135
+ |Inserted docs/second|38|344|
136
+ |Search latency (3 terms)|7.51 ms| 0.051ms|
137
+ |Searches/second|133|19608|
138
+ |Index rebuild|0.822|0.626|
139
+
140
+ We only limited the test to 10K documents becuause MieliSearch was taking a long time to index, so we decided to stop at a reasonable sample size. The search numbers for litesearch were double checked, event against a 100K document set and they remained virtually the same. It is clear that litesearch is much faster than MieliSearch in both indexing and searching, this could be partially attributed to litesearch being a simpler text search engine, but still, the difference is huge! For rebuilding the index though, Litesearch is not that much faster than Mielisearch.
data/CHANGELOG.md CHANGED
@@ -1,17 +1,44 @@
1
1
  ## [Unreleased]
2
2
 
3
+ ## [0.4.3] - 2024-02-15
4
+
5
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.4.1...master)
6
+ - Add "sequel" as a development dependency
7
+ - Diff links in CHANGELOG (thanks Weston Ganger)
8
+ - Fix deamonize type in liteboard (thanks Julian Rubisch)
9
+ - Better Litecache schema (streamlined numeric value support)
10
+ - Support for set_multi and get_multi in Litecache (read_multi and write_multi support for Rails Cache store)
11
+ - More tests written for Litecache and Rails Litecache store
12
+ - Experimenting with removing the Rails LocalCache as it doesn't show enough improvement in perfromance to compensate for the memory overhead
13
+ - Switch Litecache to a FIFO eviction model vs LRU (thanks Julian Rubisch and Stephen Margheim)
14
+
15
+ ## [0.4.2] - 2023-11-11
16
+
17
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.4.1...v0.4.2)
18
+ - Add similarity search support for Litesearch (works best for non-trigram indexes)
19
+ - Enable similarity search for ActiveRecord and Sequel models
20
+ - Fix Litesearch tests
21
+ - Suppress chatty Litejob exit detector when there are no jobs in flight
22
+ - Tidy up the test folder
23
+ - [#41](https://github.com/oldmoe/litestack/pull/41) - Fix bug in Litecable where the `connected` event was not getting propogated
24
+ - Add Litemetric and Liteboard info to README.doc
25
+ - Fix the testing rake task
26
+
3
27
  ## [0.4.1] - 2023-10-11
4
28
 
29
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.4.0...v0.4.1)
5
30
  - Add missing Litesearch::Model dependency
6
31
 
7
32
  ## [0.4.0] - 2023-10-11
8
33
 
34
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.3.0...v0.4.0)
9
35
  - Introduced Litesearch, dynamic & fast full text search capability for Litedb
10
36
  - ActiveRecord and Sequel integration for Litesearch
11
37
  - Slight improvement to the Sequel Litedb adapter for better Litesearch integration
12
38
 
13
39
  ## [0.3.0] - 2023-08-13
14
40
 
41
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.2.6...v0.3.0)
15
42
  - Reworked the Litecable thread safety model
16
43
  - Fixed multiple litejob bugs (thanks Stephen Margheim)
17
44
  - Fixed Railtie dependency (thanks Marco Roth)
@@ -21,6 +48,7 @@
21
48
 
22
49
  ## [0.2.6] - 2023-07-16
23
50
 
51
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.2.3...v0.2.6)
24
52
  - Much improved database location setting (thanks Brad Gessler)
25
53
  - A Rails generator for better Rails Litestack defaults (thanks Brad Gessler)
26
54
  - Revamped Litemetric, now much faster and more accurate (still experimental)
@@ -28,19 +56,23 @@
28
56
 
29
57
  ## [0.2.3] - 2023-05-20
30
58
 
59
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.2.2...v0.2.3)
31
60
  - Cut back on options defined in the Litejob Rails adapter
32
61
 
33
62
  ## [0.2.2] - 2023-05-18
34
63
 
64
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.2.1...v0.2.2)
35
65
  - Fix default queue location in Litejob
36
66
 
37
67
 
38
68
  ## [0.2.1] - 2023-05-08
39
69
 
70
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.2.0...v0.2.1)
40
71
  - Fix a race condition in Litecable
41
72
 
42
73
  ## [0.2.0] - 2023-05-08
43
74
 
75
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.1.8...v0.2.0)
44
76
  - Litecable, a SQLite driver for ActionCable
45
77
  - Litemetric for metrics collection support (experimental, disabled by default)
46
78
  - New schema for Litejob, old jobs are auto-migrated
@@ -50,6 +82,7 @@
50
82
 
51
83
  ## [0.1.8] - 2023-03-08
52
84
 
85
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.1.7...v0.1.8)
53
86
  - More code cleanups, more test coverage
54
87
  - Retry support for jobs in Litejob
55
88
  - Job storage and garbage collection for failed jobs
@@ -58,6 +91,7 @@
58
91
 
59
92
  ## [0.1.7] - 2023-03-05
60
93
 
94
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.1.6...v0.1.7)
61
95
  - Code cleanup, removal of references to older name
62
96
  - Fix for the litedb rake tasks (thanks: netmute)
63
97
  - More fixes for the new concurrency model
@@ -65,6 +99,7 @@
65
99
 
66
100
  ## [0.1.6] - 2023-03-03
67
101
 
102
+ - [View Diff](https://github.com/oldmoe/litestack/compare/v0.1.0...v0.1.6)
68
103
  - Revamped the locking model, more robust, minimal performance hit
69
104
  - Introduced a new resource pooling class
70
105
  - Litecache and Litejob now use the resource pool
data/Gemfile CHANGED
@@ -2,11 +2,5 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- # Specify your gem's dependencies in ultralite.gemspec
5
+ # Specify your gem's dependencies in litestack.gemspec
6
6
  gemspec
7
-
8
-
9
-
10
- gem "rack", "~> 3.0"
11
-
12
- gem "simplecov"
data/README.md CHANGED
@@ -1,12 +1,16 @@
1
1
  ![litestack](https://github.com/oldmoe/litestack/blob/master/assets/litestack_logo_teal_large.png?raw=true)
2
2
 
3
+ <a href="https://badge.fury.io/rb/litestack" target="_blank"><img height="21" style='border:0px;height:21px;' border='0' src="https://badge.fury.io/rb/litestack.svg" alt="Gem Version"></a>
4
+ <a href='https://github.com/oldmoe/litestack/actions' target='_blank'><img src="https://github.com/oldmoe/litestack/actions/workflows/ruby.yml/badge.svg?branch=master" style="max-width:100%;" height='21' style='border:0px;height:21px;' border='0' alt="CI Status"></a>
5
+ <a href='https://rubygems.org/gems/litestack' target='_blank'><img height='21' style='border:0px;height:21px;' src='https://img.shields.io/gem/dt/litestack?color=brightgreen&label=Rubygems%20Downloads' border='0' alt='RubyGems Downloads' /></a>
3
6
 
4
- litestack is a revolutionary gem for Ruby and Ruby on Rails that provides an all-in-one solution for web application development. It exploits the power and embeddedness of SQLite to include a full-fledged SQL database, a fast cache and a robust job queue all in a single package.
7
+ All your data infrastructure, in a gem!
5
8
 
6
- Compared to conventional approaches that require separate servers and databases, Litestack offers superior performance, efficiency, ease of use, and cost savings. Its embedded database and cache reduce memory and CPU usage, while its simple interface streamlines the development process. Overall, LiteStack sets a new standard for web application development and is an excellent choice for those who demand speed, efficiency, and simplicity.
9
+ Litestack is a Ruby gem that provides both Ruby and Ruby on Rails applications an all-in-one solution for web application data infrastructure. It exploits the power and embeddedness of SQLite to deliver a full-fledged SQL database, a fast cache , a robust job queue, a reliable message broker, a full text search engine and a metrics platform all in a single package.
7
10
 
8
- You can read more about why litestack can be a good choice for your next web application **[here](WHYLITESTACK.md)**, you might also be interested in litestack **[benchmarks](BENCHMARKS.md)**.
11
+ Compared to conventional approaches that require separate servers and databases, Litestack offers superior performance, efficiency, ease of use, and cost savings. Its embedded database and cache reduce memory and CPU usage, while its simple interface streamlines the development process. Overall, Litestack sets a new standard for web application development and is an excellent choice for those who demand speed, efficiency, and simplicity.
9
12
 
13
+ You can read more about why litestack can be a good choice for your next web application **[here](WHYLITESTACK.md)**, you might also be interested in litestack **[benchmarks](BENCHMARKS.md)**.
10
14
 
11
15
  litestack provides integration with popular libraries, including:
12
16
 
@@ -24,11 +28,10 @@ With litestack you only need to add a single gem to your app which would replace
24
28
  - Cache Server (e.g. Redis, Memcached)
25
29
  - Job Processor (e.g. Sidekiq, Goodjob)
26
30
  - Pubsub Server (e.g. Redis, PostgreSQL)
31
+ - Fulltext Search Server (e.g. Elasticsearch, Meilisearch)
27
32
 
28
33
  To make it even more efficient, litestack will detect the presence of Fiber based IO frameworks like Async (e.g. when you use the Falcon web server) or Polyphony. It will then switch its background workers for caches and queues to fibers (using the semantics of the existing framework). This is done transparently and will generally lead to lower CPU and memory utilization.
29
34
 
30
- Litestack is still pretty young and under heavy development, but you are welcome to give it a try today!.
31
-
32
35
  ## Installation
33
36
 
34
37
  Add the `litestack` gem line to your application's Gemfile:
@@ -41,12 +44,14 @@ To configure a Rails application to run the full litestack, run:
41
44
 
42
45
  ## Usage
43
46
 
44
- litestack currently offers three main components
47
+ litestack currently offers six main components
45
48
 
46
49
  - litedb
47
50
  - litecache
48
51
  - litejob
49
52
  - litecable
53
+ - litesearch
54
+ - litemetric
50
55
 
51
56
  > ![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_logo_teal.png?raw=true)
52
57
 
@@ -181,6 +186,119 @@ production:
181
186
  adapter: litecable
182
187
  ```
183
188
 
189
+ > ![litesearch](https://github.com/oldmoe/litestack/blob/master/assets/litesearch_logo_teal.png?raw=true)
190
+
191
+ ### Litesearch
192
+
193
+ Litesearch adds full text search capabilities to Litedb, you can use it in standalone mode as follows:
194
+
195
+ ```ruby
196
+ require 'litestack/litedb'
197
+ db = Litedb.new(":memory:")
198
+ # create the index
199
+ idx = db.search_index('index_name') do |schema|
200
+ schema.fields [:sender, :receiver, :body]
201
+ schema.field :subject, weight: 10
202
+ schema.tokenizer :trigram
203
+ end
204
+ # add documents
205
+ idx.add({sender: 'Kamal', receiver: 'Laila', subject: 'Are the girls awake?', body: 'I got them the new phones they asked for, are they awake?'})
206
+ # search the index, all fields
207
+ idx.search('kamal')
208
+ # search the index, specific field, partial workd (trigram)
209
+ idx.search('subject: awa')
210
+ ```
211
+
212
+ Litesearch integrates tightly with ActiveRecord ans Sequel, here are integration examples
213
+
214
+ #### ActiveRecord
215
+
216
+ ```ruby
217
+ class Author < ActiveRecord::Base
218
+ has_many :books
219
+ end
220
+
221
+ class Book < ActiveRecord::Base
222
+ belongs_to :author
223
+
224
+ include Litesearch::Model
225
+
226
+ litesearch do |schema|
227
+ schema.fields [:title, :description]
228
+ schema.field :author, target: 'authors.name'
229
+ schema.tokenizer :porter
230
+ end
231
+ end
232
+ # insert records
233
+ Author.create(name: 'Adam A. Writer')
234
+ Book.create(title: 'The biggest stunt', author_id: 1, description: 'a description')
235
+ # search the index, the search method integrates with AR's query interface
236
+ Book.search('author: writer').limit(1).all
237
+ ```
238
+ #### Sequel
239
+
240
+ ```ruby
241
+ class Author < Sequel::Model
242
+ one_to_many :books
243
+ end
244
+
245
+ class Book < Sequel::Model
246
+ many_to_one :author
247
+
248
+ include Litesearch::Model
249
+ litesearch do |schema|
250
+ schema.fields [:title, :description]
251
+ schema.field :author, target: 'authors.name'
252
+ schema.tokenizer :porter
253
+ end
254
+ end
255
+ # insert records
256
+ Author.create(name: 'Adam A. Writer')
257
+ Book.create(title: 'The biggest stunt', author_id: 1, description: 'a description')
258
+ # search the index, the search method integrates with Sequel's query interface
259
+ Book.search('author: writer').limit(1).all
260
+ ```
261
+
262
+ > ![litemetric](https://github.com/oldmoe/litestack/blob/master/assets/litemetric_logo_teal.png?raw=true)
263
+
264
+ ### Litemetric
265
+ Litestack comes with a module that can collect useful metrics for its different components, in each component, you need to add the following to the respective .yml file (database.yml in case of Litedb)
266
+ ```yml
267
+ metrics: true # default is false
268
+ ```
269
+ If you have the metrics enabled, it will start collecting data from the various modules and will store them in a database file called metric.db located in the Litesupport.root folder
270
+
271
+ Litemetric has an API that would enable collecting arbitrary metrics for non-litestack classes. The metrics will be in the database but currently the Liteboard is only able to show correct data for Litestack modules, displaying arbitrary metrics for other components will be included later.
272
+
273
+ ### Liteboard
274
+ Liteboard is a simple web server that provides a web interface for the collected metrics, it should be available globally, for usage instructions type
275
+ ```
276
+ liteboard -h
277
+ ```
278
+ It allows you to point to a specific metrics database file or a config file and then it will display the data in that metrics database
279
+
280
+ Example metrics views:
281
+
282
+ #### Litedb
283
+ ![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_metrics.png?raw=true)
284
+
285
+ - Database size, number of tables & indexes
286
+ - Number of read/write queries
287
+ - Read/Write query ratio over time
288
+ - Read/Write query time over time
289
+ - Slowest queries
290
+ - Most expensive queries (total run time = frequency * cost)
291
+
292
+ #### Litecache
293
+ ![litecache](https://github.com/oldmoe/litestack/blob/master/assets/litecache_metrics.png?raw=true)
294
+
295
+ - Cache size, % of size limit
296
+ - Number of entries
297
+ - Reads/Writes over time
298
+ - Read hits/misses over time
299
+ - Most written entries
300
+ - Most read entries
301
+
184
302
  ## Contributing
185
303
 
186
304
  Bug reports and pull requests are welcome on GitHub at https://github.com/oldmoe/litestack.
data/ROADMAP.md ADDED
@@ -0,0 +1,45 @@
1
+ # Litestack Roadmap
2
+
3
+ This is a list of functional/non-functional features that litestack targets to implement on the path to the 1.0 release and beyond
4
+
5
+ - [ ] Full test coverage
6
+ - [ ] Litesupport
7
+ - [ ] Litequeue
8
+ - [ ] Litejobqueue
9
+ - [ ] Litejob
10
+ - [ ] Litejob - Rails
11
+ - [ ] Litecache
12
+ - [ ] Litecache - Rails
13
+ - [ ] Litesearch
14
+ - [ ] Litesearch - ActiveRecord
15
+ - [ ] Litesearch - Sequel
16
+ - [ ] Litemetric
17
+ - [ ] Litedb
18
+ - [ ] Litedb - ActiveRecord
19
+ - [ ] Litedb - Sequel
20
+ - [ ] Github Actions integration
21
+ - [ ] Automated testing matrix
22
+ - [ ] Automated gem builds
23
+ - [ ] Litedb improvements
24
+ - [ ] ActiveRecord Sqlite3 Adapter compatibility mode
25
+ - [ ] Sequel Sqlite3 Adapter compatibility mode
26
+ - [ ] Extension bundling/loading
27
+ - [ ] Liteiob improvements
28
+ - [ ] Persist jobs even during execution
29
+ - [ ] Zombie job detection
30
+ - [ ] Better process exit handling
31
+ - [ ] Faster job dispatch for the no delay case
32
+ - [ ] Database maintenance scripts
33
+ - [ ] Online backup
34
+ - [ ] Restore
35
+ - [ ] Litestream integration
36
+ - [ ] CoW filesystems specific backup path
37
+ - [ ] Zero downtime deployment scripts
38
+ - [ ] Rails
39
+ - [ ] Genereic Rack Applications
40
+ - [ ] Litemetric improvements
41
+ - [ ] Rails performance module
42
+ - [ ] Ruby Memory/GC module
43
+ - [ ] Restore generic module
44
+ - [ ] Better HTML/CSS
45
+ - [ ] Kredis replacement implementation?
data/Rakefile CHANGED
@@ -9,4 +9,6 @@ Rake::TestTask.new(:test) do |t|
9
9
  t.test_files = FileList["test/**/test_*.rb"]
10
10
  end
11
11
 
12
- task default: :test
12
+ require "standard/rake"
13
+
14
+ task default: %i[test standard]
data/WHYLITESTACK.md CHANGED
@@ -5,7 +5,7 @@ If you're developing a Ruby web application, you may be wondering which database
5
5
  ## Standing on the shoulder of a (figurative) giant!
6
6
  At its core, Litestack is built on top of SQLite, a highly regarded open-source relational database engine. This means that Litestack is highly efficient and resource-friendly, making it an excellent choice for small to medium-sized web applications. Additionally, because SQLite is a file-based database, it is incredibly easy to set up and manage, requiring no separate server installation or configuration.
7
7
 
8
- ## Performance and Effeciency
8
+ ## Performance and Efficiency
9
9
  One of the most significant advantages of Litestack is its performance benefits. SQLite has a small memory footprint and is highly optimized, meaning that it can deliver fast and reliable database access. Additionally, Litestack's job queueing and caching functionality provide additional performance benefits, enabling you to execute tasks asynchronously and store frequently accessed data in memory for faster access.
10
10
 
11
11
  ## Dead simple Rails integration
Binary file
Binary file
Binary file
Binary file
data/bench/bench.rb CHANGED
@@ -1,23 +1,30 @@
1
- require 'sqlite3'
1
+ require "sqlite3"
2
2
 
3
- def bench(msg, iterations=1000)
3
+ def bench(msg, iterations = 1000)
4
4
  GC.start
5
- #GC.compact
6
- STDERR.puts "Starting #{iterations} iterations of #{msg} ... "
5
+ # GC.compact
6
+ warn "Starting #{iterations} iterations of #{msg} ... "
7
7
  t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
8
8
  iterations.times do |i|
9
9
  yield i
10
10
  end
11
11
  t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
12
- time = ((t2 - t1)*1000).to_i.to_f / 1000 rescue 0
13
- ips = ((iterations/(t2-t1))*100).to_i.to_f / 100 rescue "infinity?"
14
- #{m: msg, t: time, ips: iteratinos/time, i: iterations}
15
- STDERR.puts " .. finished in #{time} seconds (#{ips} ips)"
12
+ time = begin
13
+ ((t2 - t1) * 1000).to_i.to_f / 1000
14
+ rescue
15
+ 0
16
+ end
17
+ ips = begin
18
+ ((iterations / (t2 - t1)) * 100).to_i.to_f / 100
19
+ rescue
20
+ "infinity?"
21
+ end
22
+ # {m: msg, t: time, ips: iteratinos/time, i: iterations}
23
+ warn " .. finished in #{time} seconds (#{ips} ips)"
16
24
  end
17
25
 
18
26
  @db = SQLite3::Database.new(":memory:") # sqlite database for fast random string generation
19
27
 
20
28
  def random_str(size)
21
29
  @db.get_first_value("select hex(randomblob(?))", size)
22
- end
23
-
30
+ end
@@ -1,24 +1,26 @@
1
- require 'active_support'
2
- require_relative '../lib/litestack'
3
- require_relative './bench'
1
+ require "active_support"
2
+ require_relative "../lib/litestack"
3
+ require_relative "./bench"
4
4
 
5
- cache = ActiveSupport::Cache::Litecache.new({path: '../db/rails_cache.db'})
5
+ cache = ActiveSupport::Cache::Litecache.new
6
6
 
7
- #can only use the lookup method when the gem is installed
8
- #cache = ActiveSupport::Cache.lookup_store(:litecache, {path: '../db/rails_cache.db'})
7
+ # can only use the lookup method when the gem is installed
8
+ # cache = ActiveSupport::Cache.lookup_store(:litecache, {path: '../db/rails_cache.db'})
9
9
 
10
10
  redis = ActiveSupport::Cache.lookup_store(:redis_cache_store, {})
11
11
 
12
12
  values = []
13
13
  keys = []
14
14
  count = 1000
15
+ payload_sizes = [10, 100, 1000, 10000]
16
+ #payload_sizes = [100]
15
17
 
16
- [10, 100, 1000, 10000].each do |size|
18
+ payload_sizes.each do |size|
17
19
  count.times do
18
- keys << random_str(10)
19
- values << random_str(size)
20
+ keys << random_str(10)
21
+ values << random_str(size)
20
22
  end
21
-
23
+
22
24
  random_keys = keys.shuffle
23
25
  puts "Benchmarks for values of size #{size} bytes"
24
26
  puts "=========================================================="
@@ -31,6 +33,21 @@ count = 1000
31
33
  redis.write(keys[i], values[i])
32
34
  end
33
35
 
36
+ puts "== Multi Writes =="
37
+ bench("litecache multi-writes", count/5) do |i|
38
+ idx = i * 5
39
+ payload = {}
40
+ 5.times {|j| payload[keys[idx + j]] = values[idx + j] }
41
+ cache.write_multi(payload)
42
+ end
43
+
44
+ bench("Redis multi-writes", count/5) do |i|
45
+ idx = i * 5
46
+ payload = {}
47
+ 5.times {|j| payload[keys[idx + j]] = values[idx + j] }
48
+ redis.write_multi(payload)
49
+ end
50
+
34
51
  puts "== Reads =="
35
52
  bench("litecache reads", count) do |i|
36
53
  cache.read(random_keys[i])
@@ -39,14 +56,29 @@ count = 1000
39
56
  bench("Redis reads", count) do |i|
40
57
  redis.read(random_keys[i])
41
58
  end
42
- puts "=========================================================="
59
+
60
+ puts "== Multi Reads =="
61
+ bench("litecache multi-reads", count/5) do |i|
62
+ idx = i * 5
63
+ payload = []
64
+ 5.times {|j| payload << random_keys[idx+j]}
65
+ cache.read_multi(*payload)
66
+ end
67
+
68
+ bench("Redis multi-reads", count/5) do |i|
69
+ idx = i * 5
70
+ payload = []
71
+ 5.times {|j| payload << random_keys[idx+j]}
72
+ redis.read_multi(*payload)
73
+ end
43
74
 
44
75
 
76
+ puts "=========================================================="
77
+
45
78
  keys = []
46
79
  values = []
47
80
  end
48
81
 
49
-
50
82
  cache.write("somekey", 1, raw: true)
51
83
 
52
84
  redis.write("somekey", 1, raw: true)
@@ -59,9 +91,8 @@ bench("litecache increment", count) do
59
91
  end
60
92
 
61
93
  bench("Redis increment", count) do
62
- redis.increment("somekey", 1, raw: true )
94
+ redis.increment("somekey", 1, raw: true)
63
95
  end
64
96
 
65
97
  cache.clear
66
98
  redis.clear
67
-