sqlite_shards 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 70840b03f18bb37414203a8218c534e498a0b9a249663d1b010b56c88f6948d8
4
- data.tar.gz: d3c83ebabc64f00d8ec71bd24b007618628f3e18b33bafd91a3ea5845cbe432a
3
+ metadata.gz: d0db36d1ba5f1916a8547c428af2128aca5f5687d4c8b34f51f5513f85f99399
4
+ data.tar.gz: 9e242bf12e883b6e5cd0ab2a410732368e0a67f648ca2642facae85db2065ada
5
5
  SHA512:
6
- metadata.gz: 667dfce81ea40beeac34d1a31f96c1cec3b965c441bd05843649db65a46a23e916ded2711e444d37e4bafde6f0a4a4b3a30c4828ca232acdc713f5459ef51fba
7
- data.tar.gz: 7510233606657377dc206baf3aafab9e764c9151b5e76c37537b42f5a4ffd932daf4133ad9b1e667310212e59fe5f3209a5d6f92c796c63088750a5bebd4cc01
6
+ metadata.gz: 11a3b95af5ee5efb0ff87a9bee853e1d4365a0e43de625e78d8b158ac10bc69a7849bc01a0224b9f50fd523eb14d58f89efac7c899360c896381ca5f82bfb37d
7
+ data.tar.gz: fb6215483c91b92ea69bc18bbbc641860cf466c96394530926ed95c0bef1d1d10c6439095bfb5f8934c63c0ae43754b83d8136f5a6e6f96520b2689316061ad7
data/CHANGELOG.md CHANGED
@@ -1,5 +1,11 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.2.0
4
+
5
+ - At most `max_open_databases` files (100 by default) stay connected: connecting one more closes every file no thread
6
+ is using. Before, every file kept its connection pool, file descriptors and page cache for the life of the process.
7
+ - A file whose migration fails no longer leaves its connection pool behind.
8
+
3
9
  ## 0.1.0
4
10
 
5
11
  - First version, extracted from the onlylogs gem, where it keeps saved searches next to each log directory:
data/README.md CHANGED
@@ -77,6 +77,8 @@ The records it returns are wrappers that forward every call, so `note.class` is
77
77
  creating it. Useful for reading without leaving empty databases behind.
78
78
  - `NotesRecord.transaction { ... }` inside `with_database` spans all models of that file: they share one connection.
79
79
  - `with_database` calls nest, for the same class or different ones.
80
+ - `NotesRecord.max_open_databases = 20` limits the files connected at once (100 by default). Connecting one more
81
+ closes every file no thread is using.
80
82
  - `clear_database_connections` closes every connection the class opened.
81
83
 
82
84
  ## Requirements
@@ -91,6 +93,21 @@ bin/check # tests
91
93
  bin/fastcheck # standardrb
92
94
  ```
93
95
 
96
+ ## Release
97
+
98
+ 1. Bump the version in `lib/sqlite_shards/version.rb`, run `bundle install`, update `CHANGELOG.md` and commit to `main`.
99
+ 1. Tag the commit with the bare version and push the tag:
100
+
101
+ ```sh
102
+ git tag -a 1.2.3 -m 1.2.3
103
+ git push origin 1.2.3
104
+ ```
105
+
106
+ 1. Create a GitHub release for the tag with the changelog section as notes.
107
+
108
+ Semaphore runs the tests on the tag and, when they pass, promotes the `release` pipeline, which pushes the gem to
109
+ [rubygems.org](https://rubygems.org). It fails when the tag does not match the gem version.
110
+
94
111
  ## License
95
112
 
96
113
  MIT, see [LICENSE.txt](LICENSE.txt).
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module SqliteShards
4
- VERSION = "0.1.0"
4
+ VERSION = "0.2.0"
5
5
  end
data/lib/sqlite_shards.rb CHANGED
@@ -31,8 +31,11 @@ require_relative "sqlite_shards/bound"
31
31
  # note.update!(title: "Hi")
32
32
  #
33
33
  # Each file is a shard of the class, connected on first use. A file deleted or replaced since (a purged or renamed
34
- # directory, say) has another inode and so a shard of its own: the old shard's idle connections are reaped, and a
35
- # connection in use, like a test transaction's, is never pulled away.
34
+ # directory, say) has another inode and so a shard of its own.
35
+ #
36
+ # Every open file costs a connection pool, a few file descriptors and SQLite's page cache, so the files open at once
37
+ # are limited (max_open_databases, 100 by default). Connecting one more closes every file no thread is using:
38
+ # a connection in use, like a test transaction's, is never pulled away.
36
39
  #
37
40
  # Migrations are blocks run in order, tracked by SQLite's user_version: append new ones, never edit or reorder
38
41
  # shipped ones.
@@ -44,6 +47,7 @@ module SqliteShards
44
47
  class UnavailableError < Error; end
45
48
 
46
49
  CONNECTION_TIMEOUT = 5_000
50
+ MAX_OPEN_DATABASES = 100
47
51
 
48
52
  included do
49
53
  self.connection_class = true
@@ -51,6 +55,7 @@ module SqliteShards
51
55
 
52
56
  @database_migrations = []
53
57
  @database_shards = Set.new
58
+ @database_users = Hash.new(0)
54
59
  @database_lock = Mutex.new
55
60
  end
56
61
 
@@ -62,6 +67,10 @@ module SqliteShards
62
67
  @database_migrations = migrations
63
68
  end
64
69
 
70
+ attr_writer :max_open_databases
71
+
72
+ def max_open_databases = @max_open_databases || MAX_OPEN_DATABASES
73
+
65
74
  # Without create, a file that does not exist yields nothing and returns nil.
66
75
  def with_database(path, create: true, &)
67
76
  path = ::File.expand_path(path.to_s)
@@ -71,7 +80,12 @@ module SqliteShards
71
80
  return
72
81
  end
73
82
 
74
- connected_to(role: ActiveRecord.writing_role, shard: database_shard(path), &)
83
+ shard = database_shard(path)
84
+ begin
85
+ connected_to(role: ActiveRecord.writing_role, shard: shard, &)
86
+ ensure
87
+ @database_lock.synchronize { @database_users.delete(shard) if (@database_users[shard] -= 1).zero? }
88
+ end
75
89
  end
76
90
 
77
91
  def bound_to(path, &scope)
@@ -133,18 +147,41 @@ module SqliteShards
133
147
 
134
148
  @database_lock.synchronize do
135
149
  unless @database_shards.include?(shard)
150
+ close_idle_databases if @database_shards.count { |shard| shard != SCHEMA_SHARD } >= max_open_databases
136
151
  pool = connection_handler.establish_connection(
137
152
  {adapter: "sqlite3", database: path, timeout: CONNECTION_TIMEOUT},
138
153
  owner_name: self, role: ActiveRecord.writing_role, shard: shard
139
154
  )
140
- pool.with_connection { |connection| migrate_database(connection) }
155
+ begin
156
+ pool.with_connection { |connection| migrate_database(connection) }
157
+ rescue
158
+ connection_handler.remove_connection_pool(connection_specification_name,
159
+ role: ActiveRecord.writing_role, shard: shard)
160
+ raise
161
+ end
141
162
  @database_shards << shard
142
163
  end
164
+ # Counted while the lock is held, so the shard cannot be closed between here and connected_to.
165
+ @database_users[shard] += 1
143
166
  end
144
167
 
145
168
  shard
146
169
  end
147
170
 
171
+ def close_idle_databases
172
+ @database_shards.delete_if do |shard|
173
+ next false if shard == SCHEMA_SHARD || @database_users.key?(shard)
174
+
175
+ pool = connection_handler.retrieve_connection_pool(connection_specification_name,
176
+ role: ActiveRecord.writing_role, shard: shard)
177
+ next false if pool && pool.stat[:busy].positive?
178
+
179
+ connection_handler.remove_connection_pool(connection_specification_name, role: ActiveRecord.writing_role,
180
+ shard: shard)
181
+ true
182
+ end
183
+ end
184
+
148
185
  # Transactions are IMMEDIATE, so the version is read under the write lock and
149
186
  # concurrent workers cannot run the same migration twice.
150
187
  def migrate_database(connection)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sqlite_shards
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Alessandro Rodi