sqlite_shards 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +17 -0
- data/lib/sqlite_shards/version.rb +1 -1
- data/lib/sqlite_shards.rb +41 -4
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d0db36d1ba5f1916a8547c428af2128aca5f5687d4c8b34f51f5513f85f99399
|
|
4
|
+
data.tar.gz: 9e242bf12e883b6e5cd0ab2a410732368e0a67f648ca2642facae85db2065ada
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 11a3b95af5ee5efb0ff87a9bee853e1d4365a0e43de625e78d8b158ac10bc69a7849bc01a0224b9f50fd523eb14d58f89efac7c899360c896381ca5f82bfb37d
|
|
7
|
+
data.tar.gz: fb6215483c91b92ea69bc18bbbc641860cf466c96394530926ed95c0bef1d1d10c6439095bfb5f8934c63c0ae43754b83d8136f5a6e6f96520b2689316061ad7
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.0
|
|
4
|
+
|
|
5
|
+
- At most `max_open_databases` files (100 by default) stay connected: connecting one more closes every file no thread
|
|
6
|
+
is using. Before, every file kept its connection pool, file descriptors and page cache for the life of the process.
|
|
7
|
+
- A file whose migration fails no longer leaves its connection pool behind.
|
|
8
|
+
|
|
3
9
|
## 0.1.0
|
|
4
10
|
|
|
5
11
|
- First version, extracted from the onlylogs gem, where it keeps saved searches next to each log directory:
|
data/README.md
CHANGED
|
@@ -77,6 +77,8 @@ The records it returns are wrappers that forward every call, so `note.class` is
|
|
|
77
77
|
creating it. Useful for reading without leaving empty databases behind.
|
|
78
78
|
- `NotesRecord.transaction { ... }` inside `with_database` spans all models of that file: they share one connection.
|
|
79
79
|
- `with_database` calls nest, for the same class or different ones.
|
|
80
|
+
- `NotesRecord.max_open_databases = 20` limits the files connected at once (100 by default). Connecting one more
|
|
81
|
+
closes every file no thread is using.
|
|
80
82
|
- `clear_database_connections` closes every connection the class opened.
|
|
81
83
|
|
|
82
84
|
## Requirements
|
|
@@ -91,6 +93,21 @@ bin/check # tests
|
|
|
91
93
|
bin/fastcheck # standardrb
|
|
92
94
|
```
|
|
93
95
|
|
|
96
|
+
## Release
|
|
97
|
+
|
|
98
|
+
1. Bump the version in `lib/sqlite_shards/version.rb`, run `bundle install`, update `CHANGELOG.md` and commit to `main`.
|
|
99
|
+
1. Tag the commit with the bare version and push the tag:
|
|
100
|
+
|
|
101
|
+
```sh
|
|
102
|
+
git tag -a 1.2.3 -m 1.2.3
|
|
103
|
+
git push origin 1.2.3
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
1. Create a GitHub release for the tag with the changelog section as notes.
|
|
107
|
+
|
|
108
|
+
Semaphore runs the tests on the tag and, when they pass, promotes the `release` pipeline, which pushes the gem to
|
|
109
|
+
[rubygems.org](https://rubygems.org). It fails when the tag does not match the gem version.
|
|
110
|
+
|
|
94
111
|
## License
|
|
95
112
|
|
|
96
113
|
MIT, see [LICENSE.txt](LICENSE.txt).
|
data/lib/sqlite_shards.rb
CHANGED
|
@@ -31,8 +31,11 @@ require_relative "sqlite_shards/bound"
|
|
|
31
31
|
# note.update!(title: "Hi")
|
|
32
32
|
#
|
|
33
33
|
# Each file is a shard of the class, connected on first use. A file deleted or replaced since (a purged or renamed
|
|
34
|
-
# directory, say) has another inode and so a shard of its own
|
|
35
|
-
#
|
|
34
|
+
# directory, say) has another inode and so a shard of its own.
|
|
35
|
+
#
|
|
36
|
+
# Every open file costs a connection pool, a few file descriptors and SQLite's page cache, so the files open at once
|
|
37
|
+
# are limited (max_open_databases, 100 by default). Connecting one more closes every file no thread is using:
|
|
38
|
+
# a connection in use, like a test transaction's, is never pulled away.
|
|
36
39
|
#
|
|
37
40
|
# Migrations are blocks run in order, tracked by SQLite's user_version: append new ones, never edit or reorder
|
|
38
41
|
# shipped ones.
|
|
@@ -44,6 +47,7 @@ module SqliteShards
|
|
|
44
47
|
class UnavailableError < Error; end
|
|
45
48
|
|
|
46
49
|
CONNECTION_TIMEOUT = 5_000
|
|
50
|
+
MAX_OPEN_DATABASES = 100
|
|
47
51
|
|
|
48
52
|
included do
|
|
49
53
|
self.connection_class = true
|
|
@@ -51,6 +55,7 @@ module SqliteShards
|
|
|
51
55
|
|
|
52
56
|
@database_migrations = []
|
|
53
57
|
@database_shards = Set.new
|
|
58
|
+
@database_users = Hash.new(0)
|
|
54
59
|
@database_lock = Mutex.new
|
|
55
60
|
end
|
|
56
61
|
|
|
@@ -62,6 +67,10 @@ module SqliteShards
|
|
|
62
67
|
@database_migrations = migrations
|
|
63
68
|
end
|
|
64
69
|
|
|
70
|
+
attr_writer :max_open_databases
|
|
71
|
+
|
|
72
|
+
def max_open_databases = @max_open_databases || MAX_OPEN_DATABASES
|
|
73
|
+
|
|
65
74
|
# Without create, a file that does not exist yields nothing and returns nil.
|
|
66
75
|
def with_database(path, create: true, &)
|
|
67
76
|
path = ::File.expand_path(path.to_s)
|
|
@@ -71,7 +80,12 @@ module SqliteShards
|
|
|
71
80
|
return
|
|
72
81
|
end
|
|
73
82
|
|
|
74
|
-
|
|
83
|
+
shard = database_shard(path)
|
|
84
|
+
begin
|
|
85
|
+
connected_to(role: ActiveRecord.writing_role, shard: shard, &)
|
|
86
|
+
ensure
|
|
87
|
+
@database_lock.synchronize { @database_users.delete(shard) if (@database_users[shard] -= 1).zero? }
|
|
88
|
+
end
|
|
75
89
|
end
|
|
76
90
|
|
|
77
91
|
def bound_to(path, &scope)
|
|
@@ -133,18 +147,41 @@ module SqliteShards
|
|
|
133
147
|
|
|
134
148
|
@database_lock.synchronize do
|
|
135
149
|
unless @database_shards.include?(shard)
|
|
150
|
+
close_idle_databases if @database_shards.count { |shard| shard != SCHEMA_SHARD } >= max_open_databases
|
|
136
151
|
pool = connection_handler.establish_connection(
|
|
137
152
|
{adapter: "sqlite3", database: path, timeout: CONNECTION_TIMEOUT},
|
|
138
153
|
owner_name: self, role: ActiveRecord.writing_role, shard: shard
|
|
139
154
|
)
|
|
140
|
-
|
|
155
|
+
begin
|
|
156
|
+
pool.with_connection { |connection| migrate_database(connection) }
|
|
157
|
+
rescue
|
|
158
|
+
connection_handler.remove_connection_pool(connection_specification_name,
|
|
159
|
+
role: ActiveRecord.writing_role, shard: shard)
|
|
160
|
+
raise
|
|
161
|
+
end
|
|
141
162
|
@database_shards << shard
|
|
142
163
|
end
|
|
164
|
+
# Counted while the lock is held, so the shard cannot be closed between here and connected_to.
|
|
165
|
+
@database_users[shard] += 1
|
|
143
166
|
end
|
|
144
167
|
|
|
145
168
|
shard
|
|
146
169
|
end
|
|
147
170
|
|
|
171
|
+
def close_idle_databases
|
|
172
|
+
@database_shards.delete_if do |shard|
|
|
173
|
+
next false if shard == SCHEMA_SHARD || @database_users.key?(shard)
|
|
174
|
+
|
|
175
|
+
pool = connection_handler.retrieve_connection_pool(connection_specification_name,
|
|
176
|
+
role: ActiveRecord.writing_role, shard: shard)
|
|
177
|
+
next false if pool && pool.stat[:busy].positive?
|
|
178
|
+
|
|
179
|
+
connection_handler.remove_connection_pool(connection_specification_name, role: ActiveRecord.writing_role,
|
|
180
|
+
shard: shard)
|
|
181
|
+
true
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
|
|
148
185
|
# Transactions are IMMEDIATE, so the version is read under the write lock and
|
|
149
186
|
# concurrent workers cannot run the same migration twice.
|
|
150
187
|
def migrate_database(connection)
|