litestack 0.3.0 → 0.4.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.standard.yml +3 -0
  3. data/BENCHMARKS.md +34 -7
  4. data/CHANGELOG.md +21 -0
  5. data/Gemfile +1 -5
  6. data/Gemfile.lock +92 -0
  7. data/README.md +120 -6
  8. data/ROADMAP.md +45 -0
  9. data/Rakefile +3 -1
  10. data/WHYLITESTACK.md +1 -1
  11. data/assets/litecache_metrics.png +0 -0
  12. data/assets/litedb_metrics.png +0 -0
  13. data/assets/litemetric_logo_teal.png +0 -0
  14. data/assets/litesearch_logo_teal.png +0 -0
  15. data/bench/bench.rb +17 -10
  16. data/bench/bench_cache_rails.rb +10 -13
  17. data/bench/bench_cache_raw.rb +17 -22
  18. data/bench/bench_jobs_rails.rb +19 -13
  19. data/bench/bench_jobs_raw.rb +17 -10
  20. data/bench/bench_queue.rb +4 -6
  21. data/bench/rails_job.rb +5 -7
  22. data/bench/skjob.rb +4 -4
  23. data/bench/uljob.rb +6 -6
  24. data/lib/action_cable/subscription_adapter/litecable.rb +5 -8
  25. data/lib/active_job/queue_adapters/litejob_adapter.rb +6 -8
  26. data/lib/active_record/connection_adapters/litedb_adapter.rb +65 -75
  27. data/lib/active_support/cache/litecache.rb +38 -41
  28. data/lib/generators/litestack/install/install_generator.rb +3 -3
  29. data/lib/generators/litestack/install/templates/database.yml +7 -1
  30. data/lib/litestack/liteboard/liteboard.rb +269 -149
  31. data/lib/litestack/litecable.rb +44 -40
  32. data/lib/litestack/litecable.sql.yml +22 -11
  33. data/lib/litestack/litecache.rb +80 -89
  34. data/lib/litestack/litecache.sql.yml +81 -22
  35. data/lib/litestack/litecache.yml +1 -1
  36. data/lib/litestack/litedb.rb +39 -38
  37. data/lib/litestack/litejob.rb +31 -31
  38. data/lib/litestack/litejobqueue.rb +107 -106
  39. data/lib/litestack/litemetric.rb +83 -95
  40. data/lib/litestack/litemetric.sql.yml +244 -234
  41. data/lib/litestack/litemetric_collector.sql.yml +38 -41
  42. data/lib/litestack/litequeue.rb +39 -41
  43. data/lib/litestack/litequeue.sql.yml +39 -31
  44. data/lib/litestack/litescheduler.rb +84 -0
  45. data/lib/litestack/litesearch/index.rb +260 -0
  46. data/lib/litestack/litesearch/model.rb +179 -0
  47. data/lib/litestack/litesearch/schema.rb +190 -0
  48. data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +143 -0
  49. data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +137 -0
  50. data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +14 -0
  51. data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +31 -0
  52. data/lib/litestack/litesearch/schema_adapters.rb +4 -0
  53. data/lib/litestack/litesearch.rb +34 -0
  54. data/lib/litestack/litesupport.rb +85 -186
  55. data/lib/litestack/railtie.rb +1 -1
  56. data/lib/litestack/version.rb +2 -2
  57. data/lib/litestack.rb +7 -4
  58. data/lib/railties/rails/commands/dbconsole.rb +11 -15
  59. data/lib/sequel/adapters/litedb.rb +18 -22
  60. data/lib/sequel/adapters/shared/litedb.rb +168 -168
  61. data/scripts/build_metrics.rb +91 -0
  62. data/scripts/test_cable.rb +30 -0
  63. data/scripts/test_job_retry.rb +33 -0
  64. data/scripts/test_metrics.rb +60 -0
  65. data/template.rb +2 -2
  66. metadata +112 -7
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a1125a2cb5a2d9ea277fb9a0774632eb297ed4d322882dd33776d8a8f1c6471e
4
- data.tar.gz: 9cb917c999850b2bd668e8826a4be4f091ee4e8e0f963b54ed87f5b7a7b060d6
3
+ metadata.gz: 98a82dcbe7e4fe0b215e254fac5791127142fcbfd4a156b2509434f97b58aea9
4
+ data.tar.gz: a914b64d6e031559b1bdca83a8198426a49005f56ca1d0b4da2a7c6cb2384ab7
5
5
  SHA512:
6
- metadata.gz: e51460d99e66732e3dcd72e9444f1d739df7ae37804eb9b308cdae0faa7c9c34d4e7554e8f9d0bf91905f7ae3e9b4d7470509a84653da2ee736582b268fe188a
7
- data.tar.gz: 873b188bd6db924f1e56034c22f5cc465b01d653ac925a2d30de648271b0b64a705894667b90005bd3d457ad318d3ee4736b16a60fa128c34d894172bc9b8e16
6
+ metadata.gz: 72935d15fcced99bcc0da9cfb6c1ffcd43478a5456c073a3d8ab468a3e49f7c681bbdec861cb7c1e2c16659e0d11429e16d855be38fd30735448078977205dc9
7
+ data.tar.gz: ac181213e66533926ce5d075e8256dcf2c5dfa727ff7becaa244bbf053f354c564f29a294f4b2b8c1436320d41e9dc4e4697ba9e3169324bc798c60d8bfc762a
data/.standard.yml ADDED
@@ -0,0 +1,3 @@
1
+ ignore:
2
+ - 'test/**/*':
3
+ - Style/GlobalVars
data/BENCHMARKS.md CHANGED
@@ -1,8 +1,10 @@
1
1
  # Litestack Benchmarks
2
2
 
3
- This is a set of initial (simple) benchmars, designed to understand the baseline performance for different litestack components against their counterparts.
3
+ This is a set of initial (simple) benchmarks, designed to understand the baseline performance for different litestack components against their counterparts.
4
4
  These are not real life scenarios and I hope I will be able to produce some interesting ones soon.
5
5
 
6
+ All these benchmarks were run on an 8 core, 16 thread, AMD 5700U based laptop, in a Virtual Box VM
7
+
6
8
  > ![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_logo_teal.png?raw=true)
7
9
 
8
10
  ### Point Read
@@ -16,7 +18,7 @@ This produces
16
18
  SELECT * FROM posts WHERE id = ?
17
19
  ```
18
20
 
19
- |Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
21
+ |Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
20
22
  |-:|-:|-:|-:|-:|
21
23
  |1|1.3K q/s|6.5K q/s|1.8K q/s|17.4K q/s|
22
24
  |2|2.6K q/s|13.9K q/s|3.5K q/s|33.2K q/s|
@@ -33,7 +35,7 @@ This produces
33
35
  SELECT * FROM posts WHERE user_id = ? LIMIT 5
34
36
  ```
35
37
 
36
- |Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
38
+ |Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
37
39
  |-:|-:|-:|-:|-:|
38
40
  |1|345 q/s|482 q/s|937 q/s|1.1K q/s|
39
41
  |2|751 q/s|848 q/s|1.3K q/s|2.3K q/s|
@@ -51,7 +53,7 @@ This produces
51
53
  Update posts SET updated_at = ? WHERE id = ?
52
54
  ```
53
55
 
54
- |Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
56
+ |Processes|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
55
57
  |-:|-:|-:|-:|-:|
56
58
  |1|125 q/s|484 q/s|129 q/s|2.1K q/s|
57
59
  |2|265 q/s|576 q/s|333 q/s|2.5K q/s|
@@ -76,7 +78,7 @@ For testing the cache we attempted to try writing and reading different payload
76
78
 
77
79
  ### Read
78
80
 
79
- |Payload Size (bytes)|Redis|litecahce|
81
+ |Payload Size (bytes)|Redis|litecache|
80
82
  |-:|-:|-:|
81
83
  |10|5.0K q/s|69.4K q/s|
82
84
  |100|5.0K q/s|90.7K q/s|
@@ -85,11 +87,11 @@ For testing the cache we attempted to try writing and reading different payload
85
87
 
86
88
  ### Increment an int value
87
89
 
88
- |Redis|litecahce|
90
+ |Redis|litecache|
89
91
  |-:|-:|
90
92
  |5.1K q/s|16.9K q/s|
91
93
 
92
- It is not even a contest! litecache delivers way higher peroformance, specially in reading performance which is arguably the most important metric for a cache.
94
+ It is not even a contest! litecache delivers way higher performance, specially in reading performance which is arguably the most important metric for a cache.
93
95
 
94
96
  > ![litejob](https://github.com/oldmoe/litestack/blob/master/assets/litejob_logo_teal.png?raw=true)
95
97
 
@@ -109,5 +111,30 @@ Two scenarios were benchmarked, an empty job and one with a 100ms sleep to simul
109
111
 
110
112
  Running Litejob with fibers is producing much faster results than any threaded solution. Still though, threaded Litejob remains ahead of Sidekiq in all scenarios.
111
113
 
114
+ > ![litecable](https://github.com/oldmoe/litestack/blob/master/assets/litecable_logo_teal.png?raw=true)
115
+
116
+ A client written using the Iodine web server was used to generate the WS load in an event driven fashion. The Rails application, the Iodine based load generator and the Redis server were all run on the same machine to exclude network overheads (Redis still pays for the TCP stack overhead though)
117
+
118
+ |Requests|Redis Req/Sec|Litestack Req/sec|Redis p90 Latency (ms)|Litestack p90 Latency (ms)|Redis p99 Latency (ms)|Litestack p99 Latancy (ms)|
119
+ |-:|-:|-:|-:|-:|-:|-:|
120
+ |1,000|2611|3058|34|27|153|78|
121
+ |10,000|3110|5328|81|40|138|122
122
+ |100,000|3403|5385|41|36|153|235
123
+
124
+ On average, Litecable is quite faster than the Redis based version and offers better latenices for over 90% of the requests, though Redis usually delivers better p99 latencies,
112
125
 
126
+ > ![litesearch](https://github.com/oldmoe/litestack/blob/master/assets/litesearch_logo_teal.png?raw=true)
127
+
128
+ Litesearch was benchmarked against Mielsearch, both using their respective ActiveRecord integrations. Mielsearch was running on the same machine as the benchmark script and was using the default configuration options. The dataset used for testing was the infamous Enron email corpus. Redisearch was not benchmarked due to the clients being not Rails 7.1 compatible (yet), will probably bench Redisearch when they are.
129
+
130
+ ### Building the index
131
+
132
+ ||MieliSearch|Litesearch|
133
+ |-:|-:|-:|
134
+ |Time to insert 10K docs|265.42 seconds|29.06 seconds|
135
+ |Inserted docs/second|38|344|
136
+ |Search latency (3 terms)|7.51 ms| 0.051ms|
137
+ |Searches/second|133|19608|
138
+ |Index rebuild|0.822|0.626|
113
139
 
140
+ We only limited the test to 10K documents becuause MieliSearch was taking a long time to index, so we decided to stop at a reasonable sample size. The search numbers for litesearch were double checked, event against a 100K document set and they remained virtually the same. It is clear that litesearch is much faster than MieliSearch in both indexing and searching, this could be partially attributed to litesearch being a simpler text search engine, but still, the difference is huge! For rebuilding the index though, Litesearch is not that much faster than Mielisearch.
data/CHANGELOG.md CHANGED
@@ -1,5 +1,26 @@
1
1
  ## [Unreleased]
2
2
 
3
+ ## [0.4.2] - 2023-11-11
4
+
5
+ - Add similarity search support for Litesearch (works best for non-trigram indexes)
6
+ - Enable similarity search for ActiveRecord and Sequel models
7
+ - Fix Litesearch tests
8
+ - Suppress chatty Litejob exit detector when there are no jobs in flight
9
+ - Tidy up the test folder
10
+ - [#41](https://github.com/oldmoe/litestack/pull/41) - Fix bug in Litecable where the `connected` event was not getting propogated
11
+ - Add Litemetric and Liteboard info to README.doc
12
+ - Fix the testing rake task
13
+
14
+ ## [0.4.1] - 2023-10-11
15
+
16
+ - Add missing Litesearch::Model dependency
17
+
18
+ ## [0.4.0] - 2023-10-11
19
+
20
+ - Introduced Litesearch, dynamic & fast full text search capability for Litedb
21
+ - ActiveRecord and Sequel integration for Litesearch
22
+ - Slight improvement to the Sequel Litedb adapter for better Litesearch integration
23
+
3
24
  ## [0.3.0] - 2023-08-13
4
25
 
5
26
  - Reworked the Litecable thread safety model
data/Gemfile CHANGED
@@ -2,9 +2,5 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- # Specify your gem's dependencies in ultralite.gemspec
5
+ # Specify your gem's dependencies in litestack.gemspec
6
6
  gemspec
7
-
8
-
9
-
10
- gem "rack", "~> 3.0"
data/Gemfile.lock ADDED
@@ -0,0 +1,92 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ litestack (0.4.1)
5
+ erubi
6
+ hanami-router
7
+ oj
8
+ rack
9
+ sqlite3
10
+ tilt
11
+
12
+ GEM
13
+ remote: https://rubygems.org/
14
+ specs:
15
+ ast (2.4.2)
16
+ docile (1.4.0)
17
+ erubi (1.12.0)
18
+ hanami-router (0.6.2)
19
+ hanami-utils (~> 0.7)
20
+ http_router (~> 0.11)
21
+ hanami-utils (0.9.2)
22
+ http_router (0.11.2)
23
+ rack (>= 1.0.0)
24
+ url_mount (~> 0.2.1)
25
+ json (2.6.3)
26
+ language_server-protocol (3.17.0.3)
27
+ lint_roller (1.1.0)
28
+ minitest (5.19.0)
29
+ oj (3.15.1)
30
+ parallel (1.23.0)
31
+ parser (3.2.2.3)
32
+ ast (~> 2.4.1)
33
+ racc
34
+ racc (1.7.1)
35
+ rack (3.0.8)
36
+ rainbow (3.1.1)
37
+ rake (13.0.6)
38
+ regexp_parser (2.8.1)
39
+ rexml (3.2.6)
40
+ rubocop (1.52.1)
41
+ json (~> 2.3)
42
+ parallel (~> 1.10)
43
+ parser (>= 3.2.2.3)
44
+ rainbow (>= 2.2.2, < 4.0)
45
+ regexp_parser (>= 1.8, < 3.0)
46
+ rexml (>= 3.2.5, < 4.0)
47
+ rubocop-ast (>= 1.28.0, < 2.0)
48
+ ruby-progressbar (~> 1.7)
49
+ unicode-display_width (>= 2.4.0, < 3.0)
50
+ rubocop-ast (1.29.0)
51
+ parser (>= 3.2.1.0)
52
+ rubocop-performance (1.18.0)
53
+ rubocop (>= 1.7.0, < 2.0)
54
+ rubocop-ast (>= 0.4.0)
55
+ ruby-progressbar (1.13.0)
56
+ simplecov (0.22.0)
57
+ docile (~> 1.1)
58
+ simplecov-html (~> 0.11)
59
+ simplecov_json_formatter (~> 0.1)
60
+ simplecov-html (0.12.3)
61
+ simplecov_json_formatter (0.1.4)
62
+ sqlite3 (1.6.3-arm64-darwin)
63
+ standard (1.30.1)
64
+ language_server-protocol (~> 3.17.0.2)
65
+ lint_roller (~> 1.0)
66
+ rubocop (~> 1.52.0)
67
+ standard-custom (~> 1.0.0)
68
+ standard-performance (~> 1.1.0)
69
+ standard-custom (1.0.2)
70
+ lint_roller (~> 1.0)
71
+ rubocop (~> 1.50)
72
+ standard-performance (1.1.2)
73
+ lint_roller (~> 1.1)
74
+ rubocop-performance (~> 1.18.0)
75
+ tilt (2.2.0)
76
+ unicode-display_width (2.4.2)
77
+ url_mount (0.2.1)
78
+ rack
79
+
80
+ PLATFORMS
81
+ arm64-darwin-21
82
+
83
+ DEPENDENCIES
84
+ litestack!
85
+ minitest (~> 5.0)
86
+ rack (~> 3.0)
87
+ rake (~> 13.0)
88
+ simplecov
89
+ standard (~> 1.3)
90
+
91
+ BUNDLED WITH
92
+ 2.4.8
data/README.md CHANGED
@@ -1,13 +1,13 @@
1
1
  ![litestack](https://github.com/oldmoe/litestack/blob/master/assets/litestack_logo_teal_large.png?raw=true)
2
2
 
3
+ All your data infrastructure, in a gem!
3
4
 
4
- litestack is a revolutionary gem for Ruby and Ruby on Rails that provides an all-in-one solution for web application development. It exploits the power and embeddedness of SQLite to include a full-fledged SQL database, a fast cache and a robust job queue all in a single package.
5
+ Litestack is a Ruby gem that provides both Ruby and Ruby on Rails applications an all-in-one solution for web application data infrastructure. It exploits the power and embeddedness of SQLite to deliver a full-fledged SQL database, a fast cache , a robust job queue, a reliable message broker, a full text search engine and a metrics platform all in a single package.
5
6
 
6
- Compared to conventional approaches that require separate servers and databases, Litestack offers superior performance, efficiency, ease of use, and cost savings. Its embedded database and cache reduce memory and CPU usage, while its simple interface streamlines the development process. Overall, LiteStack sets a new standard for web application development and is an excellent choice for those who demand speed, efficiency, and simplicity.
7
+ Compared to conventional approaches that require separate servers and databases, Litestack offers superior performance, efficiency, ease of use, and cost savings. Its embedded database and cache reduce memory and CPU usage, while its simple interface streamlines the development process. Overall, Litestack sets a new standard for web application development and is an excellent choice for those who demand speed, efficiency, and simplicity.
7
8
 
8
9
  You can read more about why litestack can be a good choice for your next web application **[here](WHYLITESTACK.md)**, you might also be interested in litestack **[benchmarks](BENCHMARKS.md)**.
9
10
 
10
-
11
11
  litestack provides integration with popular libraries, including:
12
12
 
13
13
  - Rack
@@ -24,11 +24,10 @@ With litestack you only need to add a single gem to your app which would replace
24
24
  - Cache Server (e.g. Redis, Memcached)
25
25
  - Job Processor (e.g. Sidekiq, Goodjob)
26
26
  - Pubsub Server (e.g. Redis, PostgreSQL)
27
+ - Fulltext Search Server (e.g. Elasticsearch, Mielisearch)
27
28
 
28
29
  To make it even more efficient, litestack will detect the presence of Fiber based IO frameworks like Async (e.g. when you use the Falcon web server) or Polyphony. It will then switch its background workers for caches and queues to fibers (using the semantics of the existing framework). This is done transparently and will generally lead to lower CPU and memory utilization.
29
30
 
30
- Litestack is still pretty young and under heavy development, but you are welcome to give it a try today!.
31
-
32
31
  ## Installation
33
32
 
34
33
  Add the `litestack` gem line to your application's Gemfile:
@@ -41,12 +40,14 @@ To configure a Rails application to run the full litestack, run:
41
40
 
42
41
  ## Usage
43
42
 
44
- litestack currently offers three main components
43
+ litestack currently offers six main components
45
44
 
46
45
  - litedb
47
46
  - litecache
48
47
  - litejob
49
48
  - litecable
49
+ - litesearch
50
+ - litemetric
50
51
 
51
52
  > ![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_logo_teal.png?raw=true)
52
53
 
@@ -181,6 +182,119 @@ production:
181
182
  adapter: litecable
182
183
  ```
183
184
 
185
+ > ![litesearch](https://github.com/oldmoe/litestack/blob/master/assets/litesearch_logo_teal.png?raw=true)
186
+
187
+ ### Litesearch
188
+
189
+ Litesearch adds full text search capabilities to Litedb, you can use it in standalone mode as follows:
190
+
191
+ ```ruby
192
+ require 'litestack/litedb'
193
+ db = Litedb.new(":memory:")
194
+ # create the index
195
+ idx = db.search_index('index_name') do |schema|
196
+ schema.fields [:sender, :receiver, :body]
197
+ schema.field :subject, weight: 10
198
+ schema.tokenizer :trigram
199
+ end
200
+ # add documents
201
+ idx.add({sender: 'Kamal', receiver: 'Laila', subject: 'Are the girls awake?', body: 'I got them the new phones they asked for, are they awake?'})
202
+ # search the index, all fields
203
+ idx.search('kamal')
204
+ # search the index, specific field, partial workd (trigram)
205
+ idx.search('subject: awa')
206
+ ```
207
+
208
+ Litesearch integrates tightly with ActiveRecord ans Sequel, here are integration examples
209
+
210
+ #### ActiveRecord
211
+
212
+ ```ruby
213
+ class Author < ActiveRecord::Base
214
+ has_many :books
215
+ end
216
+
217
+ class Book < ActiveRecord::Base
218
+ belongs_to :author
219
+
220
+ include Litesearch::Model
221
+
222
+ litesearch do |schema|
223
+ schema.fields [:title, :description]
224
+ schema.field :author, target: 'authors.name'
225
+ schema.tokenizer :porter
226
+ end
227
+ end
228
+ # insert records
229
+ Author.create(name: 'Adam A. Writer')
230
+ Book.create(title: 'The biggest stunt', author_id: 1, description: 'a description')
231
+ # search the index, the search method integrates with AR's query interface
232
+ Book.search('author: writer').limit(1).all
233
+ ```
234
+ #### Sequel
235
+
236
+ ```ruby
237
+ class Author < Sequel::Model
238
+ one_to_many :books
239
+ end
240
+
241
+ class Book < Sequel::Model
242
+ many_to_one :author
243
+
244
+ include Litesearch::Model
245
+ litesearch do |schema|
246
+ schema.fields [:title, :description]
247
+ schema.field :author, target: 'authors.name'
248
+ schema.tokenizer :porter
249
+ end
250
+ end
251
+ # insert records
252
+ Author.create(name: 'Adam A. Writer')
253
+ Book.create(title: 'The biggest stunt', author_id: 1, description: 'a description')
254
+ # search the index, the search method integrates with Sequel's query interface
255
+ Book.search('author: writer').limit(1).all
256
+ ```
257
+
258
+ > ![litemetric](https://github.com/oldmoe/litestack/blob/master/assets/litemetric_logo_teal.png?raw=true)
259
+
260
+ ### Litemetric
261
+ Litestack comes with a mdoule that can collect useful metrics for its different components, in each component, you need to add the following to the respective .yml file (database.yml in case of Litedb)
262
+ ```yml
263
+ metrics: true # default is false
264
+ ```
265
+ If you have the metrics enabled, it will start collecting data from the various modules and will store them in a database file called metric.db located in the Litesupport.root folder
266
+
267
+ Litemetric has an API that would enable collecting arbitrary metrics for non-litestack classes. The metrics will be in the database but currently the Liteboard is only able to show correct data for Litestack modules, displaying arbitrary metrics for other components will be included later.
268
+
269
+ ### Liteboard
270
+ Liteboard is a simple web server that provides a web interface for the collected metrics, it should be available globally, for usage instructions type
271
+ ```
272
+ liteboard -h
273
+ ```
274
+ It allows you to point to a specific metrics database file or a config file and then it will display the data in that metrics database
275
+
276
+ Example metrics views:
277
+
278
+ #### Litedb
279
+ ![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_metrics.png?raw=true)
280
+
281
+ - Database size, number of tables & indexes
282
+ - Number of read/write queries
283
+ - Read/Write query ratio over time
284
+ - Read/Write query time over time
285
+ - Slowest queries
286
+ - Most expensive queries (total run time = frequency * cost)
287
+
288
+ #### Litecache
289
+ ![litecache](https://github.com/oldmoe/litestack/blob/master/assets/litecache_metrics.png?raw=true)
290
+
291
+ - Cache size, % of size limit
292
+ - Number of entries
293
+ - Reads/Writes over time
294
+ - Read hits/misses over time
295
+ - Most written entries
296
+ - Most read entries
297
+
184
298
  ## Contributing
185
299
 
186
300
  Bug reports and pull requests are welcome on GitHub at https://github.com/oldmoe/litestack.
data/ROADMAP.md ADDED
@@ -0,0 +1,45 @@
1
+ # Litestack Roadmap
2
+
3
+ This is a list of functional/non-functional features that litestack targets to implement on the path to the 1.0 release and beyond
4
+
5
+ - [ ] Full test coverage
6
+ - [ ] Litesupport
7
+ - [ ] Litequeue
8
+ - [ ] Litejobqueue
9
+ - [ ] Litejob
10
+ - [ ] Litejob - Rails
11
+ - [ ] Litecache
12
+ - [ ] Litecache - Rails
13
+ - [ ] Litesearch
14
+ - [ ] Litesearch - ActiveRecord
15
+ - [ ] Litesearch - Sequel
16
+ - [ ] Litemetric
17
+ - [ ] Litedb
18
+ - [ ] Litedb - ActiveRecord
19
+ - [ ] Litedb - Sequel
20
+ - [ ] Github Actions integration
21
+ - [ ] Automated testing matrix
22
+ - [ ] Automated gem builds
23
+ - [ ] Litedb improvements
24
+ - [ ] ActiveRecord Sqlite3 Adapter compatibility mode
25
+ - [ ] Sequel Sqlite3 Adapter compatibility mode
26
+ - [ ] Extension bundling/loading
27
+ - [ ] Liteiob improvements
28
+ - [ ] Persist jobs even during execution
29
+ - [ ] Zombie job detection
30
+ - [ ] Better process exit handling
31
+ - [ ] Faster job dispatch for the no delay case
32
+ - [ ] Database maintenance scripts
33
+ - [ ] Online backup
34
+ - [ ] Restore
35
+ - [ ] Litestream integration
36
+ - [ ] CoW filesystems specific backup path
37
+ - [ ] Zero downtime deployment scripts
38
+ - [ ] Rails
39
+ - [ ] Genereic Rack Applications
40
+ - [ ] Litemetric improvements
41
+ - [ ] Rails performance module
42
+ - [ ] Ruby Memory/GC module
43
+ - [ ] Restore generic module
44
+ - [ ] Better HTML/CSS
45
+ - [ ] Kredis replacement implementation?
data/Rakefile CHANGED
@@ -9,4 +9,6 @@ Rake::TestTask.new(:test) do |t|
9
9
  t.test_files = FileList["test/**/test_*.rb"]
10
10
  end
11
11
 
12
- task default: :test
12
+ require "standard/rake"
13
+
14
+ task default: %i[test standard]
data/WHYLITESTACK.md CHANGED
@@ -5,7 +5,7 @@ If you're developing a Ruby web application, you may be wondering which database
5
5
  ## Standing on the shoulder of a (figurative) giant!
6
6
  At its core, Litestack is built on top of SQLite, a highly regarded open-source relational database engine. This means that Litestack is highly efficient and resource-friendly, making it an excellent choice for small to medium-sized web applications. Additionally, because SQLite is a file-based database, it is incredibly easy to set up and manage, requiring no separate server installation or configuration.
7
7
 
8
- ## Performance and Effeciency
8
+ ## Performance and Efficiency
9
9
  One of the most significant advantages of Litestack is its performance benefits. SQLite has a small memory footprint and is highly optimized, meaning that it can deliver fast and reliable database access. Additionally, Litestack's job queueing and caching functionality provide additional performance benefits, enabling you to execute tasks asynchronously and store frequently accessed data in memory for faster access.
10
10
 
11
11
  ## Dead simple Rails integration
Binary file
Binary file
Binary file
Binary file
data/bench/bench.rb CHANGED
@@ -1,23 +1,30 @@
1
- require 'sqlite3'
1
+ require "sqlite3"
2
2
 
3
- def bench(msg, iterations=1000)
3
+ def bench(msg, iterations = 1000)
4
4
  GC.start
5
- #GC.compact
6
- STDERR.puts "Starting #{iterations} iterations of #{msg} ... "
5
+ # GC.compact
6
+ warn "Starting #{iterations} iterations of #{msg} ... "
7
7
  t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
8
8
  iterations.times do |i|
9
9
  yield i
10
10
  end
11
11
  t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
12
- time = ((t2 - t1)*1000).to_i.to_f / 1000 rescue 0
13
- ips = ((iterations/(t2-t1))*100).to_i.to_f / 100 rescue "infinity?"
14
- #{m: msg, t: time, ips: iteratinos/time, i: iterations}
15
- STDERR.puts " .. finished in #{time} seconds (#{ips} ips)"
12
+ time = begin
13
+ ((t2 - t1) * 1000).to_i.to_f / 1000
14
+ rescue
15
+ 0
16
+ end
17
+ ips = begin
18
+ ((iterations / (t2 - t1)) * 100).to_i.to_f / 100
19
+ rescue
20
+ "infinity?"
21
+ end
22
+ # {m: msg, t: time, ips: iteratinos/time, i: iterations}
23
+ warn " .. finished in #{time} seconds (#{ips} ips)"
16
24
  end
17
25
 
18
26
  @db = SQLite3::Database.new(":memory:") # sqlite database for fast random string generation
19
27
 
20
28
  def random_str(size)
21
29
  @db.get_first_value("select hex(randomblob(?))", size)
22
- end
23
-
30
+ end
@@ -1,11 +1,11 @@
1
- require 'active_support'
2
- require_relative '../lib/litestack'
3
- require_relative './bench'
1
+ require "active_support"
2
+ require_relative "../lib/litestack"
3
+ require_relative "./bench"
4
4
 
5
- cache = ActiveSupport::Cache::Litecache.new({path: '../db/rails_cache.db'})
5
+ cache = ActiveSupport::Cache::Litecache.new({path: "../db/rails_cache.db"})
6
6
 
7
- #can only use the lookup method when the gem is installed
8
- #cache = ActiveSupport::Cache.lookup_store(:litecache, {path: '../db/rails_cache.db'})
7
+ # can only use the lookup method when the gem is installed
8
+ # cache = ActiveSupport::Cache.lookup_store(:litecache, {path: '../db/rails_cache.db'})
9
9
 
10
10
  redis = ActiveSupport::Cache.lookup_store(:redis_cache_store, {})
11
11
 
@@ -15,10 +15,10 @@ count = 1000
15
15
 
16
16
  [10, 100, 1000, 10000].each do |size|
17
17
  count.times do
18
- keys << random_str(10)
19
- values << random_str(size)
18
+ keys << random_str(10)
19
+ values << random_str(size)
20
20
  end
21
-
21
+
22
22
  random_keys = keys.shuffle
23
23
  puts "Benchmarks for values of size #{size} bytes"
24
24
  puts "=========================================================="
@@ -41,12 +41,10 @@ count = 1000
41
41
  end
42
42
  puts "=========================================================="
43
43
 
44
-
45
44
  keys = []
46
45
  values = []
47
46
  end
48
47
 
49
-
50
48
  cache.write("somekey", 1, raw: true)
51
49
 
52
50
  redis.write("somekey", 1, raw: true)
@@ -59,9 +57,8 @@ bench("litecache increment", count) do
59
57
  end
60
58
 
61
59
  bench("Redis increment", count) do
62
- redis.increment("somekey", 1, raw: true )
60
+ redis.increment("somekey", 1, raw: true)
63
61
  end
64
62
 
65
63
  cache.clear
66
64
  redis.clear
67
-
@@ -1,17 +1,17 @@
1
- require 'redis'
2
- require 'sqlite3'
3
- require_relative './bench'
1
+ require "redis"
2
+ require "sqlite3"
3
+ require_relative "./bench"
4
4
 
5
- #require 'polyphony'
6
- require 'async/scheduler'
5
+ # require 'polyphony'
6
+ require "async/scheduler"
7
7
 
8
8
  Fiber.set_scheduler Async::Scheduler.new
9
9
  Fiber.scheduler.run
10
10
 
11
- require_relative '../lib/litestack/litecache'
12
- #require 'litestack'
11
+ require_relative "../lib/litestack/litecache"
12
+ # require 'litestack'
13
13
 
14
- cache = Litecache.new({path: '../db/cache.db'}) # default settings
14
+ cache = Litecache.new({path: "../db/cache.db"}) # default settings
15
15
  redis = Redis.new # default settings
16
16
 
17
17
  values = []
@@ -21,13 +21,13 @@ count.times { keys << random_str(10) }
21
21
 
22
22
  [10, 100, 1000, 10000].each do |size|
23
23
  count.times do
24
- values << random_str(size)
24
+ values << random_str(size)
25
25
  end
26
-
26
+
27
27
  random_keys = keys.shuffle
28
-
28
+
29
29
  GC.compact
30
-
30
+
31
31
  puts "Benchmarks for values of size #{size} bytes"
32
32
  puts "=========================================================="
33
33
  puts "== Writes =="
@@ -35,12 +35,11 @@ count.times { keys << random_str(10) }
35
35
  cache.set(keys[i], values[i])
36
36
  end
37
37
 
38
- #bench("file writes", count) do |i|
38
+ # bench("file writes", count) do |i|
39
39
  # f = File.open("../files/#{keys[i]}.data", 'w+')
40
40
  # f.write(values[i])
41
41
  # f.close
42
- #end
43
-
42
+ # end
44
43
 
45
44
  bench("Redis writes", count) do |i|
46
45
  redis.set(keys[i], values[i])
@@ -51,9 +50,9 @@ count.times { keys << random_str(10) }
51
50
  cache.get(random_keys[i])
52
51
  end
53
52
 
54
- #bench("file reads", count) do |i|
53
+ # bench("file reads", count) do |i|
55
54
  # data = File.read("../files/#{keys[i]}.data")
56
- #end
55
+ # end
57
56
 
58
57
  bench("Redis reads", count) do |i|
59
58
  redis.get(random_keys[i])
@@ -61,11 +60,8 @@ count.times { keys << random_str(10) }
61
60
  puts "=========================================================="
62
61
 
63
62
  values = []
64
-
65
-
66
63
  end
67
64
 
68
-
69
65
  cache.set("somekey", 1)
70
66
  redis.set("somekey", 1)
71
67
 
@@ -80,5 +76,4 @@ end
80
76
  cache.clear
81
77
  redis.flushdb
82
78
 
83
- #sleep
84
-
79
+ # sleep