litestack 0.4.1 → 0.4.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.standard.yml +3 -0
- data/BENCHMARKS.md +23 -7
- data/CHANGELOG.md +35 -0
- data/Gemfile +1 -7
- data/README.md +124 -6
- data/ROADMAP.md +45 -0
- data/Rakefile +3 -1
- data/WHYLITESTACK.md +1 -1
- data/assets/litecache_metrics.png +0 -0
- data/assets/litedb_metrics.png +0 -0
- data/assets/litemetric_logo_teal.png +0 -0
- data/assets/litesearch_logo_teal.png +0 -0
- data/bench/bench.rb +17 -10
- data/bench/bench_cache_rails.rb +45 -14
- data/bench/bench_cache_raw.rb +44 -28
- data/bench/bench_jobs_rails.rb +18 -12
- data/bench/bench_jobs_raw.rb +17 -10
- data/bench/bench_queue.rb +4 -6
- data/bench/rails_job.rb +5 -7
- data/bench/skjob.rb +4 -4
- data/bench/uljob.rb +6 -6
- data/bin/liteboard +2 -1
- data/lib/action_cable/subscription_adapter/litecable.rb +5 -8
- data/lib/active_job/queue_adapters/litejob_adapter.rb +6 -8
- data/lib/active_record/connection_adapters/litedb_adapter.rb +72 -84
- data/lib/active_support/cache/litecache.rb +61 -41
- data/lib/generators/litestack/install/install_generator.rb +3 -3
- data/lib/generators/litestack/install/templates/cable.yml +0 -3
- data/lib/generators/litestack/install/templates/database.yml +7 -1
- data/lib/litestack/liteboard/liteboard.rb +269 -149
- data/lib/litestack/litecable.rb +41 -37
- data/lib/litestack/litecable.sql.yml +22 -11
- data/lib/litestack/litecache.rb +118 -93
- data/lib/litestack/litecache.sql.yml +83 -22
- data/lib/litestack/litecache.yml +1 -1
- data/lib/litestack/litedb.rb +35 -40
- data/lib/litestack/litejob.rb +30 -29
- data/lib/litestack/litejobqueue.rb +63 -65
- data/lib/litestack/litemetric.rb +80 -92
- data/lib/litestack/litemetric.sql.yml +244 -234
- data/lib/litestack/litemetric_collector.sql.yml +38 -41
- data/lib/litestack/litequeue.rb +39 -41
- data/lib/litestack/litequeue.sql.yml +39 -31
- data/lib/litestack/litescheduler.rb +24 -18
- data/lib/litestack/litesearch/index.rb +93 -63
- data/lib/litestack/litesearch/model.rb +66 -65
- data/lib/litestack/litesearch/schema.rb +53 -56
- data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +46 -50
- data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +44 -35
- data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +3 -6
- data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +7 -9
- data/lib/litestack/litesearch/schema_adapters.rb +4 -9
- data/lib/litestack/litesearch.rb +6 -9
- data/lib/litestack/litesupport.rb +78 -87
- data/lib/litestack/railtie.rb +1 -1
- data/lib/litestack/version.rb +2 -2
- data/lib/litestack.rb +6 -4
- data/lib/railties/rails/commands/dbconsole.rb +16 -20
- data/lib/sequel/adapters/litedb.rb +16 -21
- data/lib/sequel/adapters/shared/litedb.rb +168 -168
- data/scripts/build_metrics.rb +91 -0
- data/scripts/test_cable.rb +30 -0
- data/scripts/test_job_retry.rb +33 -0
- data/scripts/test_metrics.rb +60 -0
- data/template.rb +2 -2
- metadata +115 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e2ac02213caf3525c40b676550a6daa24380f3012a5d4c8f8042cf39aa477593
|
4
|
+
data.tar.gz: 00d0f60e20bcba2cc22704d4e7159fe9b819920d313e7dc7d4e4e799420e95ca
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7c9c726aecbfd99ad73d82c50c285178becefb4792ece0672b55dfede10a47f0563700a691398373f372ca829491c13e0332a81b9b075f9a0435ae55ac138a1c
|
7
|
+
data.tar.gz: b4a28527632a3065d0deb8eadb5c806d260edfd89aa9b3f00ebbc20a2aa63ebfb7df907e7bd0a500f9e368ecd91b8967789b1f056632ef5bc73502dbbe1524c9
|
data/.standard.yml
ADDED
data/BENCHMARKS.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Litestack Benchmarks
|
2
2
|
|
3
|
-
This is a set of initial (simple)
|
3
|
+
This is a set of initial (simple) benchmarks, designed to understand the baseline performance for different litestack components against their counterparts.
|
4
4
|
These are not real life scenarios and I hope I will be able to produce some interesting ones soon.
|
5
5
|
|
6
6
|
All these benchmarks were run on an 8 core, 16 thread, AMD 5700U based laptop, in a Virtual Box VM
|
@@ -18,7 +18,7 @@ This produces
|
|
18
18
|
SELECT * FROM posts WHERE id = ?
|
19
19
|
```
|
20
20
|
|
21
|
-
|
|
21
|
+
|Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
|
22
22
|
|-:|-:|-:|-:|-:|
|
23
23
|
|1|1.3K q/s|6.5K q/s|1.8K q/s|17.4K q/s|
|
24
24
|
|2|2.6K q/s|13.9K q/s|3.5K q/s|33.2K q/s|
|
@@ -35,7 +35,7 @@ This produces
|
|
35
35
|
SELECT * FROM posts WHERE user_id = ? LIMIT 5
|
36
36
|
```
|
37
37
|
|
38
|
-
|
|
38
|
+
|Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
|
39
39
|
|-:|-:|-:|-:|-:|
|
40
40
|
|1|345 q/s|482 q/s|937 q/s|1.1K q/s|
|
41
41
|
|2|751 q/s|848 q/s|1.3K q/s|2.3K q/s|
|
@@ -53,7 +53,7 @@ This produces
|
|
53
53
|
Update posts SET updated_at = ? WHERE id = ?
|
54
54
|
```
|
55
55
|
|
56
|
-
|
|
56
|
+
|Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
|
57
57
|
|-:|-:|-:|-:|-:|
|
58
58
|
|1|125 q/s|484 q/s|129 q/s|2.1K q/s|
|
59
59
|
|2|265 q/s|576 q/s|333 q/s|2.5K q/s|
|
@@ -78,7 +78,7 @@ For testing the cache we attempted to try writing and reading different payload
|
|
78
78
|
|
79
79
|
### Read
|
80
80
|
|
81
|
-
|Payload Size (bytes)|Redis|
|
81
|
+
|Payload Size (bytes)|Redis|litecache|
|
82
82
|
|-:|-:|-:|
|
83
83
|
|10|5.0K q/s|69.4K q/s|
|
84
84
|
|100|5.0K q/s|90.7K q/s|
|
@@ -87,11 +87,11 @@ For testing the cache we attempted to try writing and reading different payload
|
|
87
87
|
|
88
88
|
### Increment an int value
|
89
89
|
|
90
|
-
|Redis|
|
90
|
+
|Redis|litecache|
|
91
91
|
|-:|-:|
|
92
92
|
|5.1K q/s|16.9K q/s|
|
93
93
|
|
94
|
-
It is not even a contest! litecache delivers way higher
|
94
|
+
It is not even a contest! litecache delivers way higher performance, specially in reading performance which is arguably the most important metric for a cache.
|
95
95
|
|
96
96
|
> ![litejob](https://github.com/oldmoe/litestack/blob/master/assets/litejob_logo_teal.png?raw=true)
|
97
97
|
|
@@ -122,3 +122,19 @@ A client written using the Iodine web server was used to generate the WS load in
|
|
122
122
|
|100,000|3403|5385|41|36|153|235
|
123
123
|
|
124
124
|
On average, Litecable is quite faster than the Redis based version and offers better latenices for over 90% of the requests, though Redis usually delivers better p99 latencies,
|
125
|
+
|
126
|
+
> ![litesearch](https://github.com/oldmoe/litestack/blob/master/assets/litesearch_logo_teal.png?raw=true)
|
127
|
+
|
128
|
+
Litesearch was benchmarked against Mielsearch, both using their respective ActiveRecord integrations. Mielsearch was running on the same machine as the benchmark script and was using the default configuration options. The dataset used for testing was the infamous Enron email corpus. Redisearch was not benchmarked due to the clients being not Rails 7.1 compatible (yet), will probably bench Redisearch when they are.
|
129
|
+
|
130
|
+
### Building the index
|
131
|
+
|
132
|
+
||MieliSearch|Litesearch|
|
133
|
+
|-:|-:|-:|
|
134
|
+
|Time to insert 10K docs|265.42 seconds|29.06 seconds|
|
135
|
+
|Inserted docs/second|38|344|
|
136
|
+
|Search latency (3 terms)|7.51 ms| 0.051ms|
|
137
|
+
|Searches/second|133|19608|
|
138
|
+
|Index rebuild|0.822|0.626|
|
139
|
+
|
140
|
+
We only limited the test to 10K documents becuause MieliSearch was taking a long time to index, so we decided to stop at a reasonable sample size. The search numbers for litesearch were double checked, event against a 100K document set and they remained virtually the same. It is clear that litesearch is much faster than MieliSearch in both indexing and searching, this could be partially attributed to litesearch being a simpler text search engine, but still, the difference is huge! For rebuilding the index though, Litesearch is not that much faster than Mielisearch.
|
data/CHANGELOG.md
CHANGED
@@ -1,17 +1,44 @@
|
|
1
1
|
## [Unreleased]
|
2
2
|
|
3
|
+
## [0.4.3] - 2024-02-15
|
4
|
+
|
5
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.4.1...master)
|
6
|
+
- Add "sequel" as a development dependency
|
7
|
+
- Diff links in CHANGELOG (thanks Weston Ganger)
|
8
|
+
- Fix deamonize type in liteboard (thanks Julian Rubisch)
|
9
|
+
- Better Litecache schema (streamlined numeric value support)
|
10
|
+
- Support for set_multi and get_multi in Litecache (read_multi and write_multi support for Rails Cache store)
|
11
|
+
- More tests written for Litecache and Rails Litecache store
|
12
|
+
- Experimenting with removing the Rails LocalCache as it doesn't show enough improvement in perfromance to compensate for the memory overhead
|
13
|
+
- Switch Litecache to a FIFO eviction model vs LRU (thanks Julian Rubisch and Stephen Margheim)
|
14
|
+
|
15
|
+
## [0.4.2] - 2023-11-11
|
16
|
+
|
17
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.4.1...v0.4.2)
|
18
|
+
- Add similarity search support for Litesearch (works best for non-trigram indexes)
|
19
|
+
- Enable similarity search for ActiveRecord and Sequel models
|
20
|
+
- Fix Litesearch tests
|
21
|
+
- Suppress chatty Litejob exit detector when there are no jobs in flight
|
22
|
+
- Tidy up the test folder
|
23
|
+
- [#41](https://github.com/oldmoe/litestack/pull/41) - Fix bug in Litecable where the `connected` event was not getting propogated
|
24
|
+
- Add Litemetric and Liteboard info to README.doc
|
25
|
+
- Fix the testing rake task
|
26
|
+
|
3
27
|
## [0.4.1] - 2023-10-11
|
4
28
|
|
29
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.4.0...v0.4.1)
|
5
30
|
- Add missing Litesearch::Model dependency
|
6
31
|
|
7
32
|
## [0.4.0] - 2023-10-11
|
8
33
|
|
34
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.3.0...v0.4.0)
|
9
35
|
- Introduced Litesearch, dynamic & fast full text search capability for Litedb
|
10
36
|
- ActiveRecord and Sequel integration for Litesearch
|
11
37
|
- Slight improvement to the Sequel Litedb adapter for better Litesearch integration
|
12
38
|
|
13
39
|
## [0.3.0] - 2023-08-13
|
14
40
|
|
41
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.2.6...v0.3.0)
|
15
42
|
- Reworked the Litecable thread safety model
|
16
43
|
- Fixed multiple litejob bugs (thanks Stephen Margheim)
|
17
44
|
- Fixed Railtie dependency (thanks Marco Roth)
|
@@ -21,6 +48,7 @@
|
|
21
48
|
|
22
49
|
## [0.2.6] - 2023-07-16
|
23
50
|
|
51
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.2.3...v0.2.6)
|
24
52
|
- Much improved database location setting (thanks Brad Gessler)
|
25
53
|
- A Rails generator for better Rails Litestack defaults (thanks Brad Gessler)
|
26
54
|
- Revamped Litemetric, now much faster and more accurate (still experimental)
|
@@ -28,19 +56,23 @@
|
|
28
56
|
|
29
57
|
## [0.2.3] - 2023-05-20
|
30
58
|
|
59
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.2.2...v0.2.3)
|
31
60
|
- Cut back on options defined in the Litejob Rails adapter
|
32
61
|
|
33
62
|
## [0.2.2] - 2023-05-18
|
34
63
|
|
64
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.2.1...v0.2.2)
|
35
65
|
- Fix default queue location in Litejob
|
36
66
|
|
37
67
|
|
38
68
|
## [0.2.1] - 2023-05-08
|
39
69
|
|
70
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.2.0...v0.2.1)
|
40
71
|
- Fix a race condition in Litecable
|
41
72
|
|
42
73
|
## [0.2.0] - 2023-05-08
|
43
74
|
|
75
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.1.8...v0.2.0)
|
44
76
|
- Litecable, a SQLite driver for ActionCable
|
45
77
|
- Litemetric for metrics collection support (experimental, disabled by default)
|
46
78
|
- New schema for Litejob, old jobs are auto-migrated
|
@@ -50,6 +82,7 @@
|
|
50
82
|
|
51
83
|
## [0.1.8] - 2023-03-08
|
52
84
|
|
85
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.1.7...v0.1.8)
|
53
86
|
- More code cleanups, more test coverage
|
54
87
|
- Retry support for jobs in Litejob
|
55
88
|
- Job storage and garbage collection for failed jobs
|
@@ -58,6 +91,7 @@
|
|
58
91
|
|
59
92
|
## [0.1.7] - 2023-03-05
|
60
93
|
|
94
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.1.6...v0.1.7)
|
61
95
|
- Code cleanup, removal of references to older name
|
62
96
|
- Fix for the litedb rake tasks (thanks: netmute)
|
63
97
|
- More fixes for the new concurrency model
|
@@ -65,6 +99,7 @@
|
|
65
99
|
|
66
100
|
## [0.1.6] - 2023-03-03
|
67
101
|
|
102
|
+
- [View Diff](https://github.com/oldmoe/litestack/compare/v0.1.0...v0.1.6)
|
68
103
|
- Revamped the locking model, more robust, minimal performance hit
|
69
104
|
- Introduced a new resource pooling class
|
70
105
|
- Litecache and Litejob now use the resource pool
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,12 +1,16 @@
|
|
1
1
|
![litestack](https://github.com/oldmoe/litestack/blob/master/assets/litestack_logo_teal_large.png?raw=true)
|
2
2
|
|
3
|
+
<a href="https://badge.fury.io/rb/litestack" target="_blank"><img height="21" style='border:0px;height:21px;' border='0' src="https://badge.fury.io/rb/litestack.svg" alt="Gem Version"></a>
|
4
|
+
<a href='https://github.com/oldmoe/litestack/actions' target='_blank'><img src="https://github.com/oldmoe/litestack/actions/workflows/ruby.yml/badge.svg?branch=master" style="max-width:100%;" height='21' style='border:0px;height:21px;' border='0' alt="CI Status"></a>
|
5
|
+
<a href='https://rubygems.org/gems/litestack' target='_blank'><img height='21' style='border:0px;height:21px;' src='https://img.shields.io/gem/dt/litestack?color=brightgreen&label=Rubygems%20Downloads' border='0' alt='RubyGems Downloads' /></a>
|
3
6
|
|
4
|
-
|
7
|
+
All your data infrastructure, in a gem!
|
5
8
|
|
6
|
-
|
9
|
+
Litestack is a Ruby gem that provides both Ruby and Ruby on Rails applications an all-in-one solution for web application data infrastructure. It exploits the power and embeddedness of SQLite to deliver a full-fledged SQL database, a fast cache , a robust job queue, a reliable message broker, a full text search engine and a metrics platform all in a single package.
|
7
10
|
|
8
|
-
|
11
|
+
Compared to conventional approaches that require separate servers and databases, Litestack offers superior performance, efficiency, ease of use, and cost savings. Its embedded database and cache reduce memory and CPU usage, while its simple interface streamlines the development process. Overall, Litestack sets a new standard for web application development and is an excellent choice for those who demand speed, efficiency, and simplicity.
|
9
12
|
|
13
|
+
You can read more about why litestack can be a good choice for your next web application **[here](WHYLITESTACK.md)**, you might also be interested in litestack **[benchmarks](BENCHMARKS.md)**.
|
10
14
|
|
11
15
|
litestack provides integration with popular libraries, including:
|
12
16
|
|
@@ -24,11 +28,10 @@ With litestack you only need to add a single gem to your app which would replace
|
|
24
28
|
- Cache Server (e.g. Redis, Memcached)
|
25
29
|
- Job Processor (e.g. Sidekiq, Goodjob)
|
26
30
|
- Pubsub Server (e.g. Redis, PostgreSQL)
|
31
|
+
- Fulltext Search Server (e.g. Elasticsearch, Meilisearch)
|
27
32
|
|
28
33
|
To make it even more efficient, litestack will detect the presence of Fiber based IO frameworks like Async (e.g. when you use the Falcon web server) or Polyphony. It will then switch its background workers for caches and queues to fibers (using the semantics of the existing framework). This is done transparently and will generally lead to lower CPU and memory utilization.
|
29
34
|
|
30
|
-
Litestack is still pretty young and under heavy development, but you are welcome to give it a try today!.
|
31
|
-
|
32
35
|
## Installation
|
33
36
|
|
34
37
|
Add the `litestack` gem line to your application's Gemfile:
|
@@ -41,12 +44,14 @@ To configure a Rails application to run the full litestack, run:
|
|
41
44
|
|
42
45
|
## Usage
|
43
46
|
|
44
|
-
litestack currently offers
|
47
|
+
litestack currently offers six main components
|
45
48
|
|
46
49
|
- litedb
|
47
50
|
- litecache
|
48
51
|
- litejob
|
49
52
|
- litecable
|
53
|
+
- litesearch
|
54
|
+
- litemetric
|
50
55
|
|
51
56
|
> ![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_logo_teal.png?raw=true)
|
52
57
|
|
@@ -181,6 +186,119 @@ production:
|
|
181
186
|
adapter: litecable
|
182
187
|
```
|
183
188
|
|
189
|
+
> ![litesearch](https://github.com/oldmoe/litestack/blob/master/assets/litesearch_logo_teal.png?raw=true)
|
190
|
+
|
191
|
+
### Litesearch
|
192
|
+
|
193
|
+
Litesearch adds full text search capabilities to Litedb, you can use it in standalone mode as follows:
|
194
|
+
|
195
|
+
```ruby
|
196
|
+
require 'litestack/litedb'
|
197
|
+
db = Litedb.new(":memory:")
|
198
|
+
# create the index
|
199
|
+
idx = db.search_index('index_name') do |schema|
|
200
|
+
schema.fields [:sender, :receiver, :body]
|
201
|
+
schema.field :subject, weight: 10
|
202
|
+
schema.tokenizer :trigram
|
203
|
+
end
|
204
|
+
# add documents
|
205
|
+
idx.add({sender: 'Kamal', receiver: 'Laila', subject: 'Are the girls awake?', body: 'I got them the new phones they asked for, are they awake?'})
|
206
|
+
# search the index, all fields
|
207
|
+
idx.search('kamal')
|
208
|
+
# search the index, specific field, partial workd (trigram)
|
209
|
+
idx.search('subject: awa')
|
210
|
+
```
|
211
|
+
|
212
|
+
Litesearch integrates tightly with ActiveRecord ans Sequel, here are integration examples
|
213
|
+
|
214
|
+
#### ActiveRecord
|
215
|
+
|
216
|
+
```ruby
|
217
|
+
class Author < ActiveRecord::Base
|
218
|
+
has_many :books
|
219
|
+
end
|
220
|
+
|
221
|
+
class Book < ActiveRecord::Base
|
222
|
+
belongs_to :author
|
223
|
+
|
224
|
+
include Litesearch::Model
|
225
|
+
|
226
|
+
litesearch do |schema|
|
227
|
+
schema.fields [:title, :description]
|
228
|
+
schema.field :author, target: 'authors.name'
|
229
|
+
schema.tokenizer :porter
|
230
|
+
end
|
231
|
+
end
|
232
|
+
# insert records
|
233
|
+
Author.create(name: 'Adam A. Writer')
|
234
|
+
Book.create(title: 'The biggest stunt', author_id: 1, description: 'a description')
|
235
|
+
# search the index, the search method integrates with AR's query interface
|
236
|
+
Book.search('author: writer').limit(1).all
|
237
|
+
```
|
238
|
+
#### Sequel
|
239
|
+
|
240
|
+
```ruby
|
241
|
+
class Author < Sequel::Model
|
242
|
+
one_to_many :books
|
243
|
+
end
|
244
|
+
|
245
|
+
class Book < Sequel::Model
|
246
|
+
many_to_one :author
|
247
|
+
|
248
|
+
include Litesearch::Model
|
249
|
+
litesearch do |schema|
|
250
|
+
schema.fields [:title, :description]
|
251
|
+
schema.field :author, target: 'authors.name'
|
252
|
+
schema.tokenizer :porter
|
253
|
+
end
|
254
|
+
end
|
255
|
+
# insert records
|
256
|
+
Author.create(name: 'Adam A. Writer')
|
257
|
+
Book.create(title: 'The biggest stunt', author_id: 1, description: 'a description')
|
258
|
+
# search the index, the search method integrates with Sequel's query interface
|
259
|
+
Book.search('author: writer').limit(1).all
|
260
|
+
```
|
261
|
+
|
262
|
+
> ![litemetric](https://github.com/oldmoe/litestack/blob/master/assets/litemetric_logo_teal.png?raw=true)
|
263
|
+
|
264
|
+
### Litemetric
|
265
|
+
Litestack comes with a module that can collect useful metrics for its different components, in each component, you need to add the following to the respective .yml file (database.yml in case of Litedb)
|
266
|
+
```yml
|
267
|
+
metrics: true # default is false
|
268
|
+
```
|
269
|
+
If you have the metrics enabled, it will start collecting data from the various modules and will store them in a database file called metric.db located in the Litesupport.root folder
|
270
|
+
|
271
|
+
Litemetric has an API that would enable collecting arbitrary metrics for non-litestack classes. The metrics will be in the database but currently the Liteboard is only able to show correct data for Litestack modules, displaying arbitrary metrics for other components will be included later.
|
272
|
+
|
273
|
+
### Liteboard
|
274
|
+
Liteboard is a simple web server that provides a web interface for the collected metrics, it should be available globally, for usage instructions type
|
275
|
+
```
|
276
|
+
liteboard -h
|
277
|
+
```
|
278
|
+
It allows you to point to a specific metrics database file or a config file and then it will display the data in that metrics database
|
279
|
+
|
280
|
+
Example metrics views:
|
281
|
+
|
282
|
+
#### Litedb
|
283
|
+
![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_metrics.png?raw=true)
|
284
|
+
|
285
|
+
- Database size, number of tables & indexes
|
286
|
+
- Number of read/write queries
|
287
|
+
- Read/Write query ratio over time
|
288
|
+
- Read/Write query time over time
|
289
|
+
- Slowest queries
|
290
|
+
- Most expensive queries (total run time = frequency * cost)
|
291
|
+
|
292
|
+
#### Litecache
|
293
|
+
![litecache](https://github.com/oldmoe/litestack/blob/master/assets/litecache_metrics.png?raw=true)
|
294
|
+
|
295
|
+
- Cache size, % of size limit
|
296
|
+
- Number of entries
|
297
|
+
- Reads/Writes over time
|
298
|
+
- Read hits/misses over time
|
299
|
+
- Most written entries
|
300
|
+
- Most read entries
|
301
|
+
|
184
302
|
## Contributing
|
185
303
|
|
186
304
|
Bug reports and pull requests are welcome on GitHub at https://github.com/oldmoe/litestack.
|
data/ROADMAP.md
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# Litestack Roadmap
|
2
|
+
|
3
|
+
This is a list of functional/non-functional features that litestack targets to implement on the path to the 1.0 release and beyond
|
4
|
+
|
5
|
+
- [ ] Full test coverage
|
6
|
+
- [ ] Litesupport
|
7
|
+
- [ ] Litequeue
|
8
|
+
- [ ] Litejobqueue
|
9
|
+
- [ ] Litejob
|
10
|
+
- [ ] Litejob - Rails
|
11
|
+
- [ ] Litecache
|
12
|
+
- [ ] Litecache - Rails
|
13
|
+
- [ ] Litesearch
|
14
|
+
- [ ] Litesearch - ActiveRecord
|
15
|
+
- [ ] Litesearch - Sequel
|
16
|
+
- [ ] Litemetric
|
17
|
+
- [ ] Litedb
|
18
|
+
- [ ] Litedb - ActiveRecord
|
19
|
+
- [ ] Litedb - Sequel
|
20
|
+
- [ ] Github Actions integration
|
21
|
+
- [ ] Automated testing matrix
|
22
|
+
- [ ] Automated gem builds
|
23
|
+
- [ ] Litedb improvements
|
24
|
+
- [ ] ActiveRecord Sqlite3 Adapter compatibility mode
|
25
|
+
- [ ] Sequel Sqlite3 Adapter compatibility mode
|
26
|
+
- [ ] Extension bundling/loading
|
27
|
+
- [ ] Liteiob improvements
|
28
|
+
- [ ] Persist jobs even during execution
|
29
|
+
- [ ] Zombie job detection
|
30
|
+
- [ ] Better process exit handling
|
31
|
+
- [ ] Faster job dispatch for the no delay case
|
32
|
+
- [ ] Database maintenance scripts
|
33
|
+
- [ ] Online backup
|
34
|
+
- [ ] Restore
|
35
|
+
- [ ] Litestream integration
|
36
|
+
- [ ] CoW filesystems specific backup path
|
37
|
+
- [ ] Zero downtime deployment scripts
|
38
|
+
- [ ] Rails
|
39
|
+
- [ ] Genereic Rack Applications
|
40
|
+
- [ ] Litemetric improvements
|
41
|
+
- [ ] Rails performance module
|
42
|
+
- [ ] Ruby Memory/GC module
|
43
|
+
- [ ] Restore generic module
|
44
|
+
- [ ] Better HTML/CSS
|
45
|
+
- [ ] Kredis replacement implementation?
|
data/Rakefile
CHANGED
data/WHYLITESTACK.md
CHANGED
@@ -5,7 +5,7 @@ If you're developing a Ruby web application, you may be wondering which database
|
|
5
5
|
## Standing on the shoulder of a (figurative) giant!
|
6
6
|
At its core, Litestack is built on top of SQLite, a highly regarded open-source relational database engine. This means that Litestack is highly efficient and resource-friendly, making it an excellent choice for small to medium-sized web applications. Additionally, because SQLite is a file-based database, it is incredibly easy to set up and manage, requiring no separate server installation or configuration.
|
7
7
|
|
8
|
-
## Performance and
|
8
|
+
## Performance and Efficiency
|
9
9
|
One of the most significant advantages of Litestack is its performance benefits. SQLite has a small memory footprint and is highly optimized, meaning that it can deliver fast and reliable database access. Additionally, Litestack's job queueing and caching functionality provide additional performance benefits, enabling you to execute tasks asynchronously and store frequently accessed data in memory for faster access.
|
10
10
|
|
11
11
|
## Dead simple Rails integration
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/bench/bench.rb
CHANGED
@@ -1,23 +1,30 @@
|
|
1
|
-
require
|
1
|
+
require "sqlite3"
|
2
2
|
|
3
|
-
def bench(msg, iterations=1000)
|
3
|
+
def bench(msg, iterations = 1000)
|
4
4
|
GC.start
|
5
|
-
#GC.compact
|
6
|
-
|
5
|
+
# GC.compact
|
6
|
+
warn "Starting #{iterations} iterations of #{msg} ... "
|
7
7
|
t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
8
8
|
iterations.times do |i|
|
9
9
|
yield i
|
10
10
|
end
|
11
11
|
t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
12
|
-
time =
|
13
|
-
|
14
|
-
|
15
|
-
|
12
|
+
time = begin
|
13
|
+
((t2 - t1) * 1000).to_i.to_f / 1000
|
14
|
+
rescue
|
15
|
+
0
|
16
|
+
end
|
17
|
+
ips = begin
|
18
|
+
((iterations / (t2 - t1)) * 100).to_i.to_f / 100
|
19
|
+
rescue
|
20
|
+
"infinity?"
|
21
|
+
end
|
22
|
+
# {m: msg, t: time, ips: iteratinos/time, i: iterations}
|
23
|
+
warn " .. finished in #{time} seconds (#{ips} ips)"
|
16
24
|
end
|
17
25
|
|
18
26
|
@db = SQLite3::Database.new(":memory:") # sqlite database for fast random string generation
|
19
27
|
|
20
28
|
def random_str(size)
|
21
29
|
@db.get_first_value("select hex(randomblob(?))", size)
|
22
|
-
end
|
23
|
-
|
30
|
+
end
|
data/bench/bench_cache_rails.rb
CHANGED
@@ -1,24 +1,26 @@
|
|
1
|
-
require
|
2
|
-
require_relative
|
3
|
-
require_relative
|
1
|
+
require "active_support"
|
2
|
+
require_relative "../lib/litestack"
|
3
|
+
require_relative "./bench"
|
4
4
|
|
5
|
-
cache = ActiveSupport::Cache::Litecache.new
|
5
|
+
cache = ActiveSupport::Cache::Litecache.new
|
6
6
|
|
7
|
-
#can only use the lookup method when the gem is installed
|
8
|
-
#cache = ActiveSupport::Cache.lookup_store(:litecache, {path: '../db/rails_cache.db'})
|
7
|
+
# can only use the lookup method when the gem is installed
|
8
|
+
# cache = ActiveSupport::Cache.lookup_store(:litecache, {path: '../db/rails_cache.db'})
|
9
9
|
|
10
10
|
redis = ActiveSupport::Cache.lookup_store(:redis_cache_store, {})
|
11
11
|
|
12
12
|
values = []
|
13
13
|
keys = []
|
14
14
|
count = 1000
|
15
|
+
payload_sizes = [10, 100, 1000, 10000]
|
16
|
+
#payload_sizes = [100]
|
15
17
|
|
16
|
-
|
18
|
+
payload_sizes.each do |size|
|
17
19
|
count.times do
|
18
|
-
keys << random_str(10)
|
19
|
-
values << random_str(size)
|
20
|
+
keys << random_str(10)
|
21
|
+
values << random_str(size)
|
20
22
|
end
|
21
|
-
|
23
|
+
|
22
24
|
random_keys = keys.shuffle
|
23
25
|
puts "Benchmarks for values of size #{size} bytes"
|
24
26
|
puts "=========================================================="
|
@@ -31,6 +33,21 @@ count = 1000
|
|
31
33
|
redis.write(keys[i], values[i])
|
32
34
|
end
|
33
35
|
|
36
|
+
puts "== Multi Writes =="
|
37
|
+
bench("litecache multi-writes", count/5) do |i|
|
38
|
+
idx = i * 5
|
39
|
+
payload = {}
|
40
|
+
5.times {|j| payload[keys[idx + j]] = values[idx + j] }
|
41
|
+
cache.write_multi(payload)
|
42
|
+
end
|
43
|
+
|
44
|
+
bench("Redis multi-writes", count/5) do |i|
|
45
|
+
idx = i * 5
|
46
|
+
payload = {}
|
47
|
+
5.times {|j| payload[keys[idx + j]] = values[idx + j] }
|
48
|
+
redis.write_multi(payload)
|
49
|
+
end
|
50
|
+
|
34
51
|
puts "== Reads =="
|
35
52
|
bench("litecache reads", count) do |i|
|
36
53
|
cache.read(random_keys[i])
|
@@ -39,14 +56,29 @@ count = 1000
|
|
39
56
|
bench("Redis reads", count) do |i|
|
40
57
|
redis.read(random_keys[i])
|
41
58
|
end
|
42
|
-
|
59
|
+
|
60
|
+
puts "== Multi Reads =="
|
61
|
+
bench("litecache multi-reads", count/5) do |i|
|
62
|
+
idx = i * 5
|
63
|
+
payload = []
|
64
|
+
5.times {|j| payload << random_keys[idx+j]}
|
65
|
+
cache.read_multi(*payload)
|
66
|
+
end
|
67
|
+
|
68
|
+
bench("Redis multi-reads", count/5) do |i|
|
69
|
+
idx = i * 5
|
70
|
+
payload = []
|
71
|
+
5.times {|j| payload << random_keys[idx+j]}
|
72
|
+
redis.read_multi(*payload)
|
73
|
+
end
|
43
74
|
|
44
75
|
|
76
|
+
puts "=========================================================="
|
77
|
+
|
45
78
|
keys = []
|
46
79
|
values = []
|
47
80
|
end
|
48
81
|
|
49
|
-
|
50
82
|
cache.write("somekey", 1, raw: true)
|
51
83
|
|
52
84
|
redis.write("somekey", 1, raw: true)
|
@@ -59,9 +91,8 @@ bench("litecache increment", count) do
|
|
59
91
|
end
|
60
92
|
|
61
93
|
bench("Redis increment", count) do
|
62
|
-
redis.increment("somekey", 1, raw: true
|
94
|
+
redis.increment("somekey", 1, raw: true)
|
63
95
|
end
|
64
96
|
|
65
97
|
cache.clear
|
66
98
|
redis.clear
|
67
|
-
|