litestack 0.3.0 → 0.4.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.standard.yml +3 -0
- data/BENCHMARKS.md +34 -7
- data/CHANGELOG.md +21 -0
- data/Gemfile +1 -5
- data/Gemfile.lock +92 -0
- data/README.md +120 -6
- data/ROADMAP.md +45 -0
- data/Rakefile +3 -1
- data/WHYLITESTACK.md +1 -1
- data/assets/litecache_metrics.png +0 -0
- data/assets/litedb_metrics.png +0 -0
- data/assets/litemetric_logo_teal.png +0 -0
- data/assets/litesearch_logo_teal.png +0 -0
- data/bench/bench.rb +17 -10
- data/bench/bench_cache_rails.rb +10 -13
- data/bench/bench_cache_raw.rb +17 -22
- data/bench/bench_jobs_rails.rb +19 -13
- data/bench/bench_jobs_raw.rb +17 -10
- data/bench/bench_queue.rb +4 -6
- data/bench/rails_job.rb +5 -7
- data/bench/skjob.rb +4 -4
- data/bench/uljob.rb +6 -6
- data/lib/action_cable/subscription_adapter/litecable.rb +5 -8
- data/lib/active_job/queue_adapters/litejob_adapter.rb +6 -8
- data/lib/active_record/connection_adapters/litedb_adapter.rb +65 -75
- data/lib/active_support/cache/litecache.rb +38 -41
- data/lib/generators/litestack/install/install_generator.rb +3 -3
- data/lib/generators/litestack/install/templates/database.yml +7 -1
- data/lib/litestack/liteboard/liteboard.rb +269 -149
- data/lib/litestack/litecable.rb +44 -40
- data/lib/litestack/litecable.sql.yml +22 -11
- data/lib/litestack/litecache.rb +80 -89
- data/lib/litestack/litecache.sql.yml +81 -22
- data/lib/litestack/litecache.yml +1 -1
- data/lib/litestack/litedb.rb +39 -38
- data/lib/litestack/litejob.rb +31 -31
- data/lib/litestack/litejobqueue.rb +107 -106
- data/lib/litestack/litemetric.rb +83 -95
- data/lib/litestack/litemetric.sql.yml +244 -234
- data/lib/litestack/litemetric_collector.sql.yml +38 -41
- data/lib/litestack/litequeue.rb +39 -41
- data/lib/litestack/litequeue.sql.yml +39 -31
- data/lib/litestack/litescheduler.rb +84 -0
- data/lib/litestack/litesearch/index.rb +260 -0
- data/lib/litestack/litesearch/model.rb +179 -0
- data/lib/litestack/litesearch/schema.rb +190 -0
- data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +143 -0
- data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +137 -0
- data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +14 -0
- data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +31 -0
- data/lib/litestack/litesearch/schema_adapters.rb +4 -0
- data/lib/litestack/litesearch.rb +34 -0
- data/lib/litestack/litesupport.rb +85 -186
- data/lib/litestack/railtie.rb +1 -1
- data/lib/litestack/version.rb +2 -2
- data/lib/litestack.rb +7 -4
- data/lib/railties/rails/commands/dbconsole.rb +11 -15
- data/lib/sequel/adapters/litedb.rb +18 -22
- data/lib/sequel/adapters/shared/litedb.rb +168 -168
- data/scripts/build_metrics.rb +91 -0
- data/scripts/test_cable.rb +30 -0
- data/scripts/test_job_retry.rb +33 -0
- data/scripts/test_metrics.rb +60 -0
- data/template.rb +2 -2
- metadata +112 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 98a82dcbe7e4fe0b215e254fac5791127142fcbfd4a156b2509434f97b58aea9
|
4
|
+
data.tar.gz: a914b64d6e031559b1bdca83a8198426a49005f56ca1d0b4da2a7c6cb2384ab7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 72935d15fcced99bcc0da9cfb6c1ffcd43478a5456c073a3d8ab468a3e49f7c681bbdec861cb7c1e2c16659e0d11429e16d855be38fd30735448078977205dc9
|
7
|
+
data.tar.gz: ac181213e66533926ce5d075e8256dcf2c5dfa727ff7becaa244bbf053f354c564f29a294f4b2b8c1436320d41e9dc4e4697ba9e3169324bc798c60d8bfc762a
|
data/.standard.yml
ADDED
data/BENCHMARKS.md
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
# Litestack Benchmarks
|
2
2
|
|
3
|
-
This is a set of initial (simple)
|
3
|
+
This is a set of initial (simple) benchmarks, designed to understand the baseline performance for different litestack components against their counterparts.
|
4
4
|
These are not real life scenarios and I hope I will be able to produce some interesting ones soon.
|
5
5
|
|
6
|
+
All these benchmarks were run on an 8 core, 16 thread, AMD 5700U based laptop, in a Virtual Box VM
|
7
|
+
|
6
8
|
> ![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_logo_teal.png?raw=true)
|
7
9
|
|
8
10
|
### Point Read
|
@@ -16,7 +18,7 @@ This produces
|
|
16
18
|
SELECT * FROM posts WHERE id = ?
|
17
19
|
```
|
18
20
|
|
19
|
-
|
|
21
|
+
|Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
|
20
22
|
|-:|-:|-:|-:|-:|
|
21
23
|
|1|1.3K q/s|6.5K q/s|1.8K q/s|17.4K q/s|
|
22
24
|
|2|2.6K q/s|13.9K q/s|3.5K q/s|33.2K q/s|
|
@@ -33,7 +35,7 @@ This produces
|
|
33
35
|
SELECT * FROM posts WHERE user_id = ? LIMIT 5
|
34
36
|
```
|
35
37
|
|
36
|
-
|
|
38
|
+
|Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
|
37
39
|
|-:|-:|-:|-:|-:|
|
38
40
|
|1|345 q/s|482 q/s|937 q/s|1.1K q/s|
|
39
41
|
|2|751 q/s|848 q/s|1.3K q/s|2.3K q/s|
|
@@ -51,7 +53,7 @@ This produces
|
|
51
53
|
Update posts SET updated_at = ? WHERE id = ?
|
52
54
|
```
|
53
55
|
|
54
|
-
|
|
56
|
+
|Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
|
55
57
|
|-:|-:|-:|-:|-:|
|
56
58
|
|1|125 q/s|484 q/s|129 q/s|2.1K q/s|
|
57
59
|
|2|265 q/s|576 q/s|333 q/s|2.5K q/s|
|
@@ -76,7 +78,7 @@ For testing the cache we attempted to try writing and reading different payload
|
|
76
78
|
|
77
79
|
### Read
|
78
80
|
|
79
|
-
|Payload Size (bytes)|Redis|
|
81
|
+
|Payload Size (bytes)|Redis|litecache|
|
80
82
|
|-:|-:|-:|
|
81
83
|
|10|5.0K q/s|69.4K q/s|
|
82
84
|
|100|5.0K q/s|90.7K q/s|
|
@@ -85,11 +87,11 @@ For testing the cache we attempted to try writing and reading different payload
|
|
85
87
|
|
86
88
|
### Increment an int value
|
87
89
|
|
88
|
-
|Redis|
|
90
|
+
|Redis|litecache|
|
89
91
|
|-:|-:|
|
90
92
|
|5.1K q/s|16.9K q/s|
|
91
93
|
|
92
|
-
It is not even a contest! litecache delivers way higher
|
94
|
+
It is not even a contest! litecache delivers way higher performance, specially in reading performance which is arguably the most important metric for a cache.
|
93
95
|
|
94
96
|
> ![litejob](https://github.com/oldmoe/litestack/blob/master/assets/litejob_logo_teal.png?raw=true)
|
95
97
|
|
@@ -109,5 +111,30 @@ Two scenarios were benchmarked, an empty job and one with a 100ms sleep to simul
|
|
109
111
|
|
110
112
|
Running Litejob with fibers is producing much faster results than any threaded solution. Still though, threaded Litejob remains ahead of Sidekiq in all scenarios.
|
111
113
|
|
114
|
+
> ![litecable](https://github.com/oldmoe/litestack/blob/master/assets/litecable_logo_teal.png?raw=true)
|
115
|
+
|
116
|
+
A client written using the Iodine web server was used to generate the WS load in an event driven fashion. The Rails application, the Iodine based load generator and the Redis server were all run on the same machine to exclude network overheads (Redis still pays for the TCP stack overhead though)
|
117
|
+
|
118
|
+
|Requests|Redis Req/Sec|Litestack Req/sec|Redis p90 Latency (ms)|Litestack p90 Latency (ms)|Redis p99 Latency (ms)|Litestack p99 Latancy (ms)|
|
119
|
+
|-:|-:|-:|-:|-:|-:|-:|
|
120
|
+
|1,000|2611|3058|34|27|153|78|
|
121
|
+
|10,000|3110|5328|81|40|138|122
|
122
|
+
|100,000|3403|5385|41|36|153|235
|
123
|
+
|
124
|
+
On average, Litecable is quite faster than the Redis based version and offers better latenices for over 90% of the requests, though Redis usually delivers better p99 latencies,
|
112
125
|
|
126
|
+
> ![litesearch](https://github.com/oldmoe/litestack/blob/master/assets/litesearch_logo_teal.png?raw=true)
|
127
|
+
|
128
|
+
Litesearch was benchmarked against Mielsearch, both using their respective ActiveRecord integrations. Mielsearch was running on the same machine as the benchmark script and was using the default configuration options. The dataset used for testing was the infamous Enron email corpus. Redisearch was not benchmarked due to the clients being not Rails 7.1 compatible (yet), will probably bench Redisearch when they are.
|
129
|
+
|
130
|
+
### Building the index
|
131
|
+
|
132
|
+
||MieliSearch|Litesearch|
|
133
|
+
|-:|-:|-:|
|
134
|
+
|Time to insert 10K docs|265.42 seconds|29.06 seconds|
|
135
|
+
|Inserted docs/second|38|344|
|
136
|
+
|Search latency (3 terms)|7.51 ms| 0.051ms|
|
137
|
+
|Searches/second|133|19608|
|
138
|
+
|Index rebuild|0.822|0.626|
|
113
139
|
|
140
|
+
We only limited the test to 10K documents becuause MieliSearch was taking a long time to index, so we decided to stop at a reasonable sample size. The search numbers for litesearch were double checked, event against a 100K document set and they remained virtually the same. It is clear that litesearch is much faster than MieliSearch in both indexing and searching, this could be partially attributed to litesearch being a simpler text search engine, but still, the difference is huge! For rebuilding the index though, Litesearch is not that much faster than Mielisearch.
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,26 @@
|
|
1
1
|
## [Unreleased]
|
2
2
|
|
3
|
+
## [0.4.2] - 2023-11-11
|
4
|
+
|
5
|
+
- Add similarity search support for Litesearch (works best for non-trigram indexes)
|
6
|
+
- Enable similarity search for ActiveRecord and Sequel models
|
7
|
+
- Fix Litesearch tests
|
8
|
+
- Suppress chatty Litejob exit detector when there are no jobs in flight
|
9
|
+
- Tidy up the test folder
|
10
|
+
- [#41](https://github.com/oldmoe/litestack/pull/41) - Fix bug in Litecable where the `connected` event was not getting propogated
|
11
|
+
- Add Litemetric and Liteboard info to README.doc
|
12
|
+
- Fix the testing rake task
|
13
|
+
|
14
|
+
## [0.4.1] - 2023-10-11
|
15
|
+
|
16
|
+
- Add missing Litesearch::Model dependency
|
17
|
+
|
18
|
+
## [0.4.0] - 2023-10-11
|
19
|
+
|
20
|
+
- Introduced Litesearch, dynamic & fast full text search capability for Litedb
|
21
|
+
- ActiveRecord and Sequel integration for Litesearch
|
22
|
+
- Slight improvement to the Sequel Litedb adapter for better Litesearch integration
|
23
|
+
|
3
24
|
## [0.3.0] - 2023-08-13
|
4
25
|
|
5
26
|
- Reworked the Litecable thread safety model
|
data/Gemfile
CHANGED
data/Gemfile.lock
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
litestack (0.4.1)
|
5
|
+
erubi
|
6
|
+
hanami-router
|
7
|
+
oj
|
8
|
+
rack
|
9
|
+
sqlite3
|
10
|
+
tilt
|
11
|
+
|
12
|
+
GEM
|
13
|
+
remote: https://rubygems.org/
|
14
|
+
specs:
|
15
|
+
ast (2.4.2)
|
16
|
+
docile (1.4.0)
|
17
|
+
erubi (1.12.0)
|
18
|
+
hanami-router (0.6.2)
|
19
|
+
hanami-utils (~> 0.7)
|
20
|
+
http_router (~> 0.11)
|
21
|
+
hanami-utils (0.9.2)
|
22
|
+
http_router (0.11.2)
|
23
|
+
rack (>= 1.0.0)
|
24
|
+
url_mount (~> 0.2.1)
|
25
|
+
json (2.6.3)
|
26
|
+
language_server-protocol (3.17.0.3)
|
27
|
+
lint_roller (1.1.0)
|
28
|
+
minitest (5.19.0)
|
29
|
+
oj (3.15.1)
|
30
|
+
parallel (1.23.0)
|
31
|
+
parser (3.2.2.3)
|
32
|
+
ast (~> 2.4.1)
|
33
|
+
racc
|
34
|
+
racc (1.7.1)
|
35
|
+
rack (3.0.8)
|
36
|
+
rainbow (3.1.1)
|
37
|
+
rake (13.0.6)
|
38
|
+
regexp_parser (2.8.1)
|
39
|
+
rexml (3.2.6)
|
40
|
+
rubocop (1.52.1)
|
41
|
+
json (~> 2.3)
|
42
|
+
parallel (~> 1.10)
|
43
|
+
parser (>= 3.2.2.3)
|
44
|
+
rainbow (>= 2.2.2, < 4.0)
|
45
|
+
regexp_parser (>= 1.8, < 3.0)
|
46
|
+
rexml (>= 3.2.5, < 4.0)
|
47
|
+
rubocop-ast (>= 1.28.0, < 2.0)
|
48
|
+
ruby-progressbar (~> 1.7)
|
49
|
+
unicode-display_width (>= 2.4.0, < 3.0)
|
50
|
+
rubocop-ast (1.29.0)
|
51
|
+
parser (>= 3.2.1.0)
|
52
|
+
rubocop-performance (1.18.0)
|
53
|
+
rubocop (>= 1.7.0, < 2.0)
|
54
|
+
rubocop-ast (>= 0.4.0)
|
55
|
+
ruby-progressbar (1.13.0)
|
56
|
+
simplecov (0.22.0)
|
57
|
+
docile (~> 1.1)
|
58
|
+
simplecov-html (~> 0.11)
|
59
|
+
simplecov_json_formatter (~> 0.1)
|
60
|
+
simplecov-html (0.12.3)
|
61
|
+
simplecov_json_formatter (0.1.4)
|
62
|
+
sqlite3 (1.6.3-arm64-darwin)
|
63
|
+
standard (1.30.1)
|
64
|
+
language_server-protocol (~> 3.17.0.2)
|
65
|
+
lint_roller (~> 1.0)
|
66
|
+
rubocop (~> 1.52.0)
|
67
|
+
standard-custom (~> 1.0.0)
|
68
|
+
standard-performance (~> 1.1.0)
|
69
|
+
standard-custom (1.0.2)
|
70
|
+
lint_roller (~> 1.0)
|
71
|
+
rubocop (~> 1.50)
|
72
|
+
standard-performance (1.1.2)
|
73
|
+
lint_roller (~> 1.1)
|
74
|
+
rubocop-performance (~> 1.18.0)
|
75
|
+
tilt (2.2.0)
|
76
|
+
unicode-display_width (2.4.2)
|
77
|
+
url_mount (0.2.1)
|
78
|
+
rack
|
79
|
+
|
80
|
+
PLATFORMS
|
81
|
+
arm64-darwin-21
|
82
|
+
|
83
|
+
DEPENDENCIES
|
84
|
+
litestack!
|
85
|
+
minitest (~> 5.0)
|
86
|
+
rack (~> 3.0)
|
87
|
+
rake (~> 13.0)
|
88
|
+
simplecov
|
89
|
+
standard (~> 1.3)
|
90
|
+
|
91
|
+
BUNDLED WITH
|
92
|
+
2.4.8
|
data/README.md
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
![litestack](https://github.com/oldmoe/litestack/blob/master/assets/litestack_logo_teal_large.png?raw=true)
|
2
2
|
|
3
|
+
All your data infrastructure, in a gem!
|
3
4
|
|
4
|
-
|
5
|
+
Litestack is a Ruby gem that provides both Ruby and Ruby on Rails applications an all-in-one solution for web application data infrastructure. It exploits the power and embeddedness of SQLite to deliver a full-fledged SQL database, a fast cache , a robust job queue, a reliable message broker, a full text search engine and a metrics platform all in a single package.
|
5
6
|
|
6
|
-
Compared to conventional approaches that require separate servers and databases, Litestack offers superior performance, efficiency, ease of use, and cost savings. Its embedded database and cache reduce memory and CPU usage, while its simple interface streamlines the development process. Overall,
|
7
|
+
Compared to conventional approaches that require separate servers and databases, Litestack offers superior performance, efficiency, ease of use, and cost savings. Its embedded database and cache reduce memory and CPU usage, while its simple interface streamlines the development process. Overall, Litestack sets a new standard for web application development and is an excellent choice for those who demand speed, efficiency, and simplicity.
|
7
8
|
|
8
9
|
You can read more about why litestack can be a good choice for your next web application **[here](WHYLITESTACK.md)**, you might also be interested in litestack **[benchmarks](BENCHMARKS.md)**.
|
9
10
|
|
10
|
-
|
11
11
|
litestack provides integration with popular libraries, including:
|
12
12
|
|
13
13
|
- Rack
|
@@ -24,11 +24,10 @@ With litestack you only need to add a single gem to your app which would replace
|
|
24
24
|
- Cache Server (e.g. Redis, Memcached)
|
25
25
|
- Job Processor (e.g. Sidekiq, Goodjob)
|
26
26
|
- Pubsub Server (e.g. Redis, PostgreSQL)
|
27
|
+
- Fulltext Search Server (e.g. Elasticsearch, Mielisearch)
|
27
28
|
|
28
29
|
To make it even more efficient, litestack will detect the presence of Fiber based IO frameworks like Async (e.g. when you use the Falcon web server) or Polyphony. It will then switch its background workers for caches and queues to fibers (using the semantics of the existing framework). This is done transparently and will generally lead to lower CPU and memory utilization.
|
29
30
|
|
30
|
-
Litestack is still pretty young and under heavy development, but you are welcome to give it a try today!.
|
31
|
-
|
32
31
|
## Installation
|
33
32
|
|
34
33
|
Add the `litestack` gem line to your application's Gemfile:
|
@@ -41,12 +40,14 @@ To configure a Rails application to run the full litestack, run:
|
|
41
40
|
|
42
41
|
## Usage
|
43
42
|
|
44
|
-
litestack currently offers
|
43
|
+
litestack currently offers six main components
|
45
44
|
|
46
45
|
- litedb
|
47
46
|
- litecache
|
48
47
|
- litejob
|
49
48
|
- litecable
|
49
|
+
- litesearch
|
50
|
+
- litemetric
|
50
51
|
|
51
52
|
> ![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_logo_teal.png?raw=true)
|
52
53
|
|
@@ -181,6 +182,119 @@ production:
|
|
181
182
|
adapter: litecable
|
182
183
|
```
|
183
184
|
|
185
|
+
> ![litesearch](https://github.com/oldmoe/litestack/blob/master/assets/litesearch_logo_teal.png?raw=true)
|
186
|
+
|
187
|
+
### Litesearch
|
188
|
+
|
189
|
+
Litesearch adds full text search capabilities to Litedb, you can use it in standalone mode as follows:
|
190
|
+
|
191
|
+
```ruby
|
192
|
+
require 'litestack/litedb'
|
193
|
+
db = Litedb.new(":memory:")
|
194
|
+
# create the index
|
195
|
+
idx = db.search_index('index_name') do |schema|
|
196
|
+
schema.fields [:sender, :receiver, :body]
|
197
|
+
schema.field :subject, weight: 10
|
198
|
+
schema.tokenizer :trigram
|
199
|
+
end
|
200
|
+
# add documents
|
201
|
+
idx.add({sender: 'Kamal', receiver: 'Laila', subject: 'Are the girls awake?', body: 'I got them the new phones they asked for, are they awake?'})
|
202
|
+
# search the index, all fields
|
203
|
+
idx.search('kamal')
|
204
|
+
# search the index, specific field, partial workd (trigram)
|
205
|
+
idx.search('subject: awa')
|
206
|
+
```
|
207
|
+
|
208
|
+
Litesearch integrates tightly with ActiveRecord ans Sequel, here are integration examples
|
209
|
+
|
210
|
+
#### ActiveRecord
|
211
|
+
|
212
|
+
```ruby
|
213
|
+
class Author < ActiveRecord::Base
|
214
|
+
has_many :books
|
215
|
+
end
|
216
|
+
|
217
|
+
class Book < ActiveRecord::Base
|
218
|
+
belongs_to :author
|
219
|
+
|
220
|
+
include Litesearch::Model
|
221
|
+
|
222
|
+
litesearch do |schema|
|
223
|
+
schema.fields [:title, :description]
|
224
|
+
schema.field :author, target: 'authors.name'
|
225
|
+
schema.tokenizer :porter
|
226
|
+
end
|
227
|
+
end
|
228
|
+
# insert records
|
229
|
+
Author.create(name: 'Adam A. Writer')
|
230
|
+
Book.create(title: 'The biggest stunt', author_id: 1, description: 'a description')
|
231
|
+
# search the index, the search method integrates with AR's query interface
|
232
|
+
Book.search('author: writer').limit(1).all
|
233
|
+
```
|
234
|
+
#### Sequel
|
235
|
+
|
236
|
+
```ruby
|
237
|
+
class Author < Sequel::Model
|
238
|
+
one_to_many :books
|
239
|
+
end
|
240
|
+
|
241
|
+
class Book < Sequel::Model
|
242
|
+
many_to_one :author
|
243
|
+
|
244
|
+
include Litesearch::Model
|
245
|
+
litesearch do |schema|
|
246
|
+
schema.fields [:title, :description]
|
247
|
+
schema.field :author, target: 'authors.name'
|
248
|
+
schema.tokenizer :porter
|
249
|
+
end
|
250
|
+
end
|
251
|
+
# insert records
|
252
|
+
Author.create(name: 'Adam A. Writer')
|
253
|
+
Book.create(title: 'The biggest stunt', author_id: 1, description: 'a description')
|
254
|
+
# search the index, the search method integrates with Sequel's query interface
|
255
|
+
Book.search('author: writer').limit(1).all
|
256
|
+
```
|
257
|
+
|
258
|
+
> ![litemetric](https://github.com/oldmoe/litestack/blob/master/assets/litemetric_logo_teal.png?raw=true)
|
259
|
+
|
260
|
+
### Litemetric
|
261
|
+
Litestack comes with a mdoule that can collect useful metrics for its different components, in each component, you need to add the following to the respective .yml file (database.yml in case of Litedb)
|
262
|
+
```yml
|
263
|
+
metrics: true # default is false
|
264
|
+
```
|
265
|
+
If you have the metrics enabled, it will start collecting data from the various modules and will store them in a database file called metric.db located in the Litesupport.root folder
|
266
|
+
|
267
|
+
Litemetric has an API that would enable collecting arbitrary metrics for non-litestack classes. The metrics will be in the database but currently the Liteboard is only able to show correct data for Litestack modules, displaying arbitrary metrics for other components will be included later.
|
268
|
+
|
269
|
+
### Liteboard
|
270
|
+
Liteboard is a simple web server that provides a web interface for the collected metrics, it should be available globally, for usage instructions type
|
271
|
+
```
|
272
|
+
liteboard -h
|
273
|
+
```
|
274
|
+
It allows you to point to a specific metrics database file or a config file and then it will display the data in that metrics database
|
275
|
+
|
276
|
+
Example metrics views:
|
277
|
+
|
278
|
+
#### Litedb
|
279
|
+
![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_metrics.png?raw=true)
|
280
|
+
|
281
|
+
- Database size, number of tables & indexes
|
282
|
+
- Number of read/write queries
|
283
|
+
- Read/Write query ratio over time
|
284
|
+
- Read/Write query time over time
|
285
|
+
- Slowest queries
|
286
|
+
- Most expensive queries (total run time = frequency * cost)
|
287
|
+
|
288
|
+
#### Litecache
|
289
|
+
![litecache](https://github.com/oldmoe/litestack/blob/master/assets/litecache_metrics.png?raw=true)
|
290
|
+
|
291
|
+
- Cache size, % of size limit
|
292
|
+
- Number of entries
|
293
|
+
- Reads/Writes over time
|
294
|
+
- Read hits/misses over time
|
295
|
+
- Most written entries
|
296
|
+
- Most read entries
|
297
|
+
|
184
298
|
## Contributing
|
185
299
|
|
186
300
|
Bug reports and pull requests are welcome on GitHub at https://github.com/oldmoe/litestack.
|
data/ROADMAP.md
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# Litestack Roadmap
|
2
|
+
|
3
|
+
This is a list of functional/non-functional features that litestack targets to implement on the path to the 1.0 release and beyond
|
4
|
+
|
5
|
+
- [ ] Full test coverage
|
6
|
+
- [ ] Litesupport
|
7
|
+
- [ ] Litequeue
|
8
|
+
- [ ] Litejobqueue
|
9
|
+
- [ ] Litejob
|
10
|
+
- [ ] Litejob - Rails
|
11
|
+
- [ ] Litecache
|
12
|
+
- [ ] Litecache - Rails
|
13
|
+
- [ ] Litesearch
|
14
|
+
- [ ] Litesearch - ActiveRecord
|
15
|
+
- [ ] Litesearch - Sequel
|
16
|
+
- [ ] Litemetric
|
17
|
+
- [ ] Litedb
|
18
|
+
- [ ] Litedb - ActiveRecord
|
19
|
+
- [ ] Litedb - Sequel
|
20
|
+
- [ ] Github Actions integration
|
21
|
+
- [ ] Automated testing matrix
|
22
|
+
- [ ] Automated gem builds
|
23
|
+
- [ ] Litedb improvements
|
24
|
+
- [ ] ActiveRecord Sqlite3 Adapter compatibility mode
|
25
|
+
- [ ] Sequel Sqlite3 Adapter compatibility mode
|
26
|
+
- [ ] Extension bundling/loading
|
27
|
+
- [ ] Liteiob improvements
|
28
|
+
- [ ] Persist jobs even during execution
|
29
|
+
- [ ] Zombie job detection
|
30
|
+
- [ ] Better process exit handling
|
31
|
+
- [ ] Faster job dispatch for the no delay case
|
32
|
+
- [ ] Database maintenance scripts
|
33
|
+
- [ ] Online backup
|
34
|
+
- [ ] Restore
|
35
|
+
- [ ] Litestream integration
|
36
|
+
- [ ] CoW filesystems specific backup path
|
37
|
+
- [ ] Zero downtime deployment scripts
|
38
|
+
- [ ] Rails
|
39
|
+
- [ ] Genereic Rack Applications
|
40
|
+
- [ ] Litemetric improvements
|
41
|
+
- [ ] Rails performance module
|
42
|
+
- [ ] Ruby Memory/GC module
|
43
|
+
- [ ] Restore generic module
|
44
|
+
- [ ] Better HTML/CSS
|
45
|
+
- [ ] Kredis replacement implementation?
|
data/Rakefile
CHANGED
data/WHYLITESTACK.md
CHANGED
@@ -5,7 +5,7 @@ If you're developing a Ruby web application, you may be wondering which database
|
|
5
5
|
## Standing on the shoulder of a (figurative) giant!
|
6
6
|
At its core, Litestack is built on top of SQLite, a highly regarded open-source relational database engine. This means that Litestack is highly efficient and resource-friendly, making it an excellent choice for small to medium-sized web applications. Additionally, because SQLite is a file-based database, it is incredibly easy to set up and manage, requiring no separate server installation or configuration.
|
7
7
|
|
8
|
-
## Performance and
|
8
|
+
## Performance and Efficiency
|
9
9
|
One of the most significant advantages of Litestack is its performance benefits. SQLite has a small memory footprint and is highly optimized, meaning that it can deliver fast and reliable database access. Additionally, Litestack's job queueing and caching functionality provide additional performance benefits, enabling you to execute tasks asynchronously and store frequently accessed data in memory for faster access.
|
10
10
|
|
11
11
|
## Dead simple Rails integration
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/bench/bench.rb
CHANGED
@@ -1,23 +1,30 @@
|
|
1
|
-
require
|
1
|
+
require "sqlite3"
|
2
2
|
|
3
|
-
def bench(msg, iterations=1000)
|
3
|
+
def bench(msg, iterations = 1000)
|
4
4
|
GC.start
|
5
|
-
#GC.compact
|
6
|
-
|
5
|
+
# GC.compact
|
6
|
+
warn "Starting #{iterations} iterations of #{msg} ... "
|
7
7
|
t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
8
8
|
iterations.times do |i|
|
9
9
|
yield i
|
10
10
|
end
|
11
11
|
t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
12
|
-
time =
|
13
|
-
|
14
|
-
|
15
|
-
|
12
|
+
time = begin
|
13
|
+
((t2 - t1) * 1000).to_i.to_f / 1000
|
14
|
+
rescue
|
15
|
+
0
|
16
|
+
end
|
17
|
+
ips = begin
|
18
|
+
((iterations / (t2 - t1)) * 100).to_i.to_f / 100
|
19
|
+
rescue
|
20
|
+
"infinity?"
|
21
|
+
end
|
22
|
+
# {m: msg, t: time, ips: iteratinos/time, i: iterations}
|
23
|
+
warn " .. finished in #{time} seconds (#{ips} ips)"
|
16
24
|
end
|
17
25
|
|
18
26
|
@db = SQLite3::Database.new(":memory:") # sqlite database for fast random string generation
|
19
27
|
|
20
28
|
def random_str(size)
|
21
29
|
@db.get_first_value("select hex(randomblob(?))", size)
|
22
|
-
end
|
23
|
-
|
30
|
+
end
|
data/bench/bench_cache_rails.rb
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
-
require
|
2
|
-
require_relative
|
3
|
-
require_relative
|
1
|
+
require "active_support"
|
2
|
+
require_relative "../lib/litestack"
|
3
|
+
require_relative "./bench"
|
4
4
|
|
5
|
-
cache = ActiveSupport::Cache::Litecache.new({path:
|
5
|
+
cache = ActiveSupport::Cache::Litecache.new({path: "../db/rails_cache.db"})
|
6
6
|
|
7
|
-
#can only use the lookup method when the gem is installed
|
8
|
-
#cache = ActiveSupport::Cache.lookup_store(:litecache, {path: '../db/rails_cache.db'})
|
7
|
+
# can only use the lookup method when the gem is installed
|
8
|
+
# cache = ActiveSupport::Cache.lookup_store(:litecache, {path: '../db/rails_cache.db'})
|
9
9
|
|
10
10
|
redis = ActiveSupport::Cache.lookup_store(:redis_cache_store, {})
|
11
11
|
|
@@ -15,10 +15,10 @@ count = 1000
|
|
15
15
|
|
16
16
|
[10, 100, 1000, 10000].each do |size|
|
17
17
|
count.times do
|
18
|
-
keys << random_str(10)
|
19
|
-
values << random_str(size)
|
18
|
+
keys << random_str(10)
|
19
|
+
values << random_str(size)
|
20
20
|
end
|
21
|
-
|
21
|
+
|
22
22
|
random_keys = keys.shuffle
|
23
23
|
puts "Benchmarks for values of size #{size} bytes"
|
24
24
|
puts "=========================================================="
|
@@ -41,12 +41,10 @@ count = 1000
|
|
41
41
|
end
|
42
42
|
puts "=========================================================="
|
43
43
|
|
44
|
-
|
45
44
|
keys = []
|
46
45
|
values = []
|
47
46
|
end
|
48
47
|
|
49
|
-
|
50
48
|
cache.write("somekey", 1, raw: true)
|
51
49
|
|
52
50
|
redis.write("somekey", 1, raw: true)
|
@@ -59,9 +57,8 @@ bench("litecache increment", count) do
|
|
59
57
|
end
|
60
58
|
|
61
59
|
bench("Redis increment", count) do
|
62
|
-
redis.increment("somekey", 1, raw: true
|
60
|
+
redis.increment("somekey", 1, raw: true)
|
63
61
|
end
|
64
62
|
|
65
63
|
cache.clear
|
66
64
|
redis.clear
|
67
|
-
|
data/bench/bench_cache_raw.rb
CHANGED
@@ -1,17 +1,17 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require_relative
|
1
|
+
require "redis"
|
2
|
+
require "sqlite3"
|
3
|
+
require_relative "./bench"
|
4
4
|
|
5
|
-
#require 'polyphony'
|
6
|
-
require
|
5
|
+
# require 'polyphony'
|
6
|
+
require "async/scheduler"
|
7
7
|
|
8
8
|
Fiber.set_scheduler Async::Scheduler.new
|
9
9
|
Fiber.scheduler.run
|
10
10
|
|
11
|
-
require_relative
|
12
|
-
#require 'litestack'
|
11
|
+
require_relative "../lib/litestack/litecache"
|
12
|
+
# require 'litestack'
|
13
13
|
|
14
|
-
cache = Litecache.new({path:
|
14
|
+
cache = Litecache.new({path: "../db/cache.db"}) # default settings
|
15
15
|
redis = Redis.new # default settings
|
16
16
|
|
17
17
|
values = []
|
@@ -21,13 +21,13 @@ count.times { keys << random_str(10) }
|
|
21
21
|
|
22
22
|
[10, 100, 1000, 10000].each do |size|
|
23
23
|
count.times do
|
24
|
-
values << random_str(size)
|
24
|
+
values << random_str(size)
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
random_keys = keys.shuffle
|
28
|
-
|
28
|
+
|
29
29
|
GC.compact
|
30
|
-
|
30
|
+
|
31
31
|
puts "Benchmarks for values of size #{size} bytes"
|
32
32
|
puts "=========================================================="
|
33
33
|
puts "== Writes =="
|
@@ -35,12 +35,11 @@ count.times { keys << random_str(10) }
|
|
35
35
|
cache.set(keys[i], values[i])
|
36
36
|
end
|
37
37
|
|
38
|
-
#bench("file writes", count) do |i|
|
38
|
+
# bench("file writes", count) do |i|
|
39
39
|
# f = File.open("../files/#{keys[i]}.data", 'w+')
|
40
40
|
# f.write(values[i])
|
41
41
|
# f.close
|
42
|
-
#end
|
43
|
-
|
42
|
+
# end
|
44
43
|
|
45
44
|
bench("Redis writes", count) do |i|
|
46
45
|
redis.set(keys[i], values[i])
|
@@ -51,9 +50,9 @@ count.times { keys << random_str(10) }
|
|
51
50
|
cache.get(random_keys[i])
|
52
51
|
end
|
53
52
|
|
54
|
-
#bench("file reads", count) do |i|
|
53
|
+
# bench("file reads", count) do |i|
|
55
54
|
# data = File.read("../files/#{keys[i]}.data")
|
56
|
-
#end
|
55
|
+
# end
|
57
56
|
|
58
57
|
bench("Redis reads", count) do |i|
|
59
58
|
redis.get(random_keys[i])
|
@@ -61,11 +60,8 @@ count.times { keys << random_str(10) }
|
|
61
60
|
puts "=========================================================="
|
62
61
|
|
63
62
|
values = []
|
64
|
-
|
65
|
-
|
66
63
|
end
|
67
64
|
|
68
|
-
|
69
65
|
cache.set("somekey", 1)
|
70
66
|
redis.set("somekey", 1)
|
71
67
|
|
@@ -80,5 +76,4 @@ end
|
|
80
76
|
cache.clear
|
81
77
|
redis.flushdb
|
82
78
|
|
83
|
-
#sleep
|
84
|
-
|
79
|
+
# sleep
|