gemvault 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +26 -0
- data/CLAUDE.md +1 -1
- data/Dockerfile.test +4 -0
- data/README.md +23 -3
- data/Rakefile +37 -17
- data/docs/tarvault-findings.md +186 -0
- data/docs/tarvault.md +42 -0
- data/lib/bundler/plugin/vault_source.rb +14 -1
- data/lib/gemvault/cli/commands/upgrade.rb +48 -0
- data/lib/gemvault/dbvault.rb +98 -0
- data/lib/gemvault/deprecation.rb +46 -0
- data/lib/gemvault/gem_extraction.rb +45 -0
- data/lib/gemvault/manifest.rb +76 -0
- data/lib/gemvault/tar_archive.rb +66 -0
- data/lib/gemvault/tarvault.rb +148 -0
- data/lib/gemvault/vault.rb +53 -159
- data/lib/gemvault/vault_session.rb +16 -0
- data/lib/gemvault/vault_upgrade.rb +87 -0
- data/lib/gemvault/version.rb +1 -1
- data/lib/gemvault.rb +1 -0
- data/lib/rubygems_plugin.rb +9 -0
- metadata +17 -20
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 100a593cb7221bf6cc5842ad00c67335809a351905251a6088c3698b41fa9b95
|
|
4
|
+
data.tar.gz: 6645dea254fbdd76dbd2fc1de7f8471f27185d205d5a23e038665d060f253cfa
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a973851f5b6b7b9eaa48fcf21a25660bc3770d36c992ff758e4b9ace061a64e5b2cdd6a0270ee335ca3d2a6cf1b80f842b69e6768a98fd85741ec91141786a60
|
|
7
|
+
data.tar.gz: 16b6a27438852230e49569493d526efdcf6b07f499137cdcb73a18b29ef06bd89ddf2def281649109bbf1cfbacce1e7e3d4bfdddae44984855fa378e1b79c51f
|
data/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,32 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- Tarball vault format ("Tarvault"): new vaults are portable tarballs with a
|
|
12
|
+
`manifest.json` index and per-gem SHA256 integrity, with no sqlite3 dependency
|
|
13
|
+
on the read/write path (works on JRuby). The original SQLite format
|
|
14
|
+
("Dbvault") is still read transparently.
|
|
15
|
+
- Vaults carry an explicit on-disk **format version**, decoupled from the gem
|
|
16
|
+
version and validated on open; gemvault refuses a vault written by a newer
|
|
17
|
+
gemvault instead of misreading it.
|
|
18
|
+
- `gemvault upgrade` migrates a vault to the current format (e.g. SQLite → tar),
|
|
19
|
+
preserving every gem and timestamp, writing a `.bak` backup by default, with
|
|
20
|
+
`--dry-run` and `--no-backup` flags. It is a no-op on an already-current vault.
|
|
21
|
+
|
|
22
|
+
### Changed
|
|
23
|
+
- `sqlite3` is no longer a runtime dependency. Gemvault runs dependency-free on
|
|
24
|
+
the tarball path (including JRuby); `sqlite3` is loaded lazily only to read a
|
|
25
|
+
legacy SQLite vault, with a clear error if it is not installed.
|
|
26
|
+
|
|
27
|
+
### Deprecated
|
|
28
|
+
- The SQLite vault format is deprecated and now **read-only**: existing SQLite
|
|
29
|
+
vaults can be read and migrated but no longer written (`add`/`remove` raise and
|
|
30
|
+
point at `gemvault upgrade`). Opening one prints a one-time deprecation notice
|
|
31
|
+
(silenceable with `GEMVAULT_SILENCE_DEPRECATIONS=1`). SQLite support will be
|
|
32
|
+
removed in a future release (0.3-0.5); migrate with `gemvault upgrade`.
|
|
33
|
+
|
|
8
34
|
## [0.1.3] - 2026-06-22
|
|
9
35
|
|
|
10
36
|
### Added
|
data/CLAUDE.md
CHANGED
|
@@ -93,7 +93,7 @@ Integration tests use a manually-written Bundler plugin index to avoid rubygems.
|
|
|
93
93
|
|
|
94
94
|
## Dependencies
|
|
95
95
|
|
|
96
|
-
- `sqlite3` (~> 2.0) — runtime
|
|
97
96
|
- `bundler` (>= 2.0) — runtime
|
|
98
97
|
- `command_kit` (~> 0.6) — runtime (CLI)
|
|
98
|
+
- `sqlite3` (~> 2.0) — NOT a runtime dependency; loaded lazily only to read a legacy SQLite (Dbvault) vault. Declared in the Gemfile for development/test.
|
|
99
99
|
- `minitest`, `rake` — development
|
data/Dockerfile.test
CHANGED
|
@@ -9,3 +9,7 @@ RUN cd /tmp/build \
|
|
|
9
9
|
&& gem build bundler-source-vault.gemspec \
|
|
10
10
|
&& gem install --no-document bundler-source-vault-*.gem \
|
|
11
11
|
&& rm -rf /tmp/build
|
|
12
|
+
|
|
13
|
+
# sqlite3 is not a gemvault runtime dependency; install it here so the upgrade
|
|
14
|
+
# integration test can build a legacy SQLite (Dbvault) fixture to migrate.
|
|
15
|
+
RUN gem install --no-document sqlite3
|
data/README.md
CHANGED
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
A gem server in a file. No HTTP. No infrastructure.
|
|
4
4
|
|
|
5
|
-
A `.gemv` file is a
|
|
5
|
+
A `.gemv` file is a self-contained archive that contains Ruby gems. Commit it to your repo, drop it on S3, email it, put it on a USB drive — Bundler and RubyGems read from it directly. Private gems without running a server.
|
|
6
|
+
|
|
7
|
+
New vaults are portable tarballs (no runtime dependencies, works on JRuby); the original SQLite format is still read transparently. See [Vault format versioning](#vault-format-versioning).
|
|
6
8
|
|
|
7
9
|
## Installation
|
|
8
10
|
|
|
@@ -44,14 +46,32 @@ gemvault add myvault.gemv foo.gem bar.gem # add .gem files
|
|
|
44
46
|
gemvault list myvault.gemv # list contents
|
|
45
47
|
gemvault remove myvault.gemv foo 1.0.0 # remove a gem
|
|
46
48
|
gemvault extract myvault.gemv foo -o vendor/ # extract .gem file to disk
|
|
49
|
+
gemvault upgrade myvault.gemv # migrate to the current format
|
|
47
50
|
```
|
|
48
51
|
|
|
52
|
+
## Vault format versioning
|
|
53
|
+
|
|
54
|
+
Every vault records an on-disk **format version** — `1` for the original SQLite format, `2` for the current tarball format. This version lives inside the file and is **independent of the gemvault gem version**: it changes only when the storage layout changes, so upgrading the gem never invalidates your vaults.
|
|
55
|
+
|
|
56
|
+
gemvault reads any format up to the one it understands and **refuses a vault written by a newer gemvault** with a clear message (rather than silently misreading it).
|
|
57
|
+
|
|
58
|
+
The SQLite format (`1`) is **deprecated and read-only**: you can still read and migrate an existing SQLite vault, but `add`/`remove` are refused and print `gemvault upgrade`. Opening one shows a one-time deprecation notice (silence it with `GEMVAULT_SILENCE_DEPRECATIONS=1`). SQLite support will be removed in a future release (0.3–0.5). To migrate a vault to the current format:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
gemvault upgrade myvault.gemv # e.g. SQLite (v1) -> tarball (v2)
|
|
62
|
+
gemvault upgrade myvault.gemv --dry-run # show the plan, change nothing
|
|
63
|
+
gemvault upgrade myvault.gemv --no-backup # skip the default myvault.gemv.bak copy
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
`upgrade` preserves every gem and its timestamp, writes `myvault.gemv.bak` by default, and is a no-op on an already-current vault.
|
|
67
|
+
|
|
49
68
|
## How It Works
|
|
50
69
|
|
|
51
|
-
A `.gemv` file is a
|
|
70
|
+
A `.gemv` file is a self-contained archive of gem metadata and raw `.gem` blobs. A current-format (tarball) vault has a `manifest.json` first entry followed by the `.gem` files, inspectable with standard tools:
|
|
52
71
|
|
|
53
72
|
```bash
|
|
54
|
-
|
|
73
|
+
tar -tf myvault.gemv # tarball (v2) vault
|
|
74
|
+
sqlite3 myvault.gemv "SELECT name, version, platform FROM gems" # SQLite (v1) vault
|
|
55
75
|
```
|
|
56
76
|
|
|
57
77
|
When Bundler sees `type: :vault` in your Gemfile, it auto-installs the `bundler-source-vault` plugin from rubygems.org. The plugin implements the `Bundler::Plugin::API::Source` interface — it reads gemspecs from the vault, participates in dependency resolution, then extracts and installs gems from the vault's blob storage.
|
data/Rakefile
CHANGED
|
@@ -8,9 +8,7 @@ end
|
|
|
8
8
|
|
|
9
9
|
require "rspec/core/rake_task"
|
|
10
10
|
|
|
11
|
-
RSpec::Core::RakeTask.new(:spec)
|
|
12
|
-
t.exclude_pattern = "spec/integration/**/*_spec.rb"
|
|
13
|
-
end
|
|
11
|
+
RSpec::Core::RakeTask.new(:spec)
|
|
14
12
|
|
|
15
13
|
require "rubocop/rake_task"
|
|
16
14
|
RuboCop::RakeTask.new
|
|
@@ -20,27 +18,49 @@ Gempilot::VersionTask.new
|
|
|
20
18
|
|
|
21
19
|
CACHED_IMAGE = "gemvault-test:latest".freeze
|
|
22
20
|
|
|
21
|
+
def cached_image_exists?
|
|
22
|
+
system("podman", "image", "exists", CACHED_IMAGE, out: File::NULL, err: File::NULL)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def build_cached_image
|
|
26
|
+
sh "podman", "build", "--network=host", "-t", CACHED_IMAGE, "-f", "Dockerfile.test", "."
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def destroy_cached_image
|
|
30
|
+
strays = `podman ps -aq --filter ancestor=#{CACHED_IMAGE}`.split
|
|
31
|
+
sh "podman", "rm", "-f", *strays unless strays.empty?
|
|
32
|
+
sh "podman", "rmi", CACHED_IMAGE
|
|
33
|
+
end
|
|
34
|
+
|
|
23
35
|
namespace :spec do
|
|
24
36
|
desc "Build cached container image with gemvault pre-installed"
|
|
25
|
-
task
|
|
26
|
-
sh "podman", "build",
|
|
27
|
-
"--network=host",
|
|
28
|
-
"-t", CACHED_IMAGE,
|
|
29
|
-
"-f", "Dockerfile.test",
|
|
30
|
-
"."
|
|
31
|
-
end
|
|
37
|
+
task(:build) { build_cached_image }
|
|
32
38
|
|
|
33
|
-
desc "
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
39
|
+
desc "Build the cached image unless it already exists"
|
|
40
|
+
task(:setup) { build_cached_image unless cached_image_exists? }
|
|
41
|
+
|
|
42
|
+
desc "Remove test containers and the cached image"
|
|
43
|
+
task(:teardown) { destroy_cached_image if cached_image_exists? }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
directory "pkg" do
|
|
47
|
+
mkdir "pkg"
|
|
37
48
|
end
|
|
38
49
|
|
|
39
50
|
namespace :shim do
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
51
|
+
Bundler::GemHelper.install_tasks dir: "shim", name: "bundler-source-vault"
|
|
52
|
+
CLOBBER.include "shim/pkg"
|
|
53
|
+
|
|
54
|
+
Rake::Task[:build].enhance ["pkg"] do
|
|
55
|
+
FileList["shim/pkg/*.gem"].each do |g|
|
|
56
|
+
mv g, "pkg", verbose: false
|
|
57
|
+
end
|
|
43
58
|
end
|
|
44
59
|
end
|
|
45
60
|
|
|
61
|
+
Rake::Task[:spec].enhance ["spec:setup"]
|
|
62
|
+
Rake::Task[:build].enhance ["shim:build"]
|
|
63
|
+
Rake::Task[:release].enhance ["shim:release"]
|
|
64
|
+
Rake::Task[:clobber].enhance ["spec:teardown"]
|
|
65
|
+
|
|
46
66
|
task default: [:test, :spec, :rubocop]
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
# Tarvault Spike — Findings
|
|
2
|
+
|
|
3
|
+
Spike for `docs/tarvault.md`: replace/augment the SQLite storage of a `.gemv`
|
|
4
|
+
with a **tarball** ("Tarvault") whose first entry is `manifest.json` and whose
|
|
5
|
+
remaining entries are `.gem` files. Motivation: drop the hard `sqlite3`
|
|
6
|
+
dependency (unportable, no JRuby) using only tooling that already ships with
|
|
7
|
+
rubygems.
|
|
8
|
+
|
|
9
|
+
## Result
|
|
10
|
+
|
|
11
|
+
Tarvault is implemented and wired **behind the existing `Gemvault::Vault`
|
|
12
|
+
facade**, so the CLI, the Bundler `type: :vault` source, and the RubyGems
|
|
13
|
+
`--source` path use it transparently:
|
|
14
|
+
|
|
15
|
+
- `Gemvault::Vault` is now a `Forwardable` delegator that selects a backend by
|
|
16
|
+
file format — existing SQLite files open as `Gemvault::Dbvault` (the old code,
|
|
17
|
+
extracted verbatim); new or tar files as `Gemvault::Tarvault`. New vaults are
|
|
18
|
+
Tarvaults.
|
|
19
|
+
- Backends are **lazily required**, so a process that only touches Tarvaults
|
|
20
|
+
never loads `sqlite3` — the JRuby-portability win.
|
|
21
|
+
- `Gemvault::Manifest` owns `manifest.json`; `Gemvault::TarArchive` owns the
|
|
22
|
+
low-level tar read + atomic rewrite; `Gemvault::GemExtraction` /
|
|
23
|
+
`Gemvault::VaultSession` are shared mixins.
|
|
24
|
+
|
|
25
|
+
**Drop-in proof:** the entire pre-existing suite passes with *no change to any
|
|
26
|
+
`spec/` behavior or integration file* — only new unit specs were added, plus two
|
|
27
|
+
Minitest assertions that had probed a SQLite closed-handle artifact.
|
|
28
|
+
|
|
29
|
+
- Minitest: `111 tests, 0 failures` (the Vault-contract, Bundler-source, and
|
|
30
|
+
RubyGems-plugin tests now run on the **tar** backend via the facade).
|
|
31
|
+
- RSpec incl. podman integration: `88 examples, 0 failures` — real
|
|
32
|
+
`bundle install` (`type: :vault`), `gem install --source myvault.gemv`,
|
|
33
|
+
bundler-inline, and CLI, all against tar-backed vaults. *(Note: the container
|
|
34
|
+
image must be rebuilt (`rake spec:build`) after code changes — a stale image
|
|
35
|
+
silently tests old code and still passes on SQLite.)*
|
|
36
|
+
- New unit specs (`spec/gemvault/**`): `42 examples, 0 failures`.
|
|
37
|
+
|
|
38
|
+
A Tarvault on disk:
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
$ file myvault.gemv # POSIX tar archive
|
|
42
|
+
$ tar -tf myvault.gemv
|
|
43
|
+
manifest.json
|
|
44
|
+
addressable-2.9.0.gem
|
|
45
|
+
rstore-0.3.9.gem
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
```json
|
|
49
|
+
// manifest.json
|
|
50
|
+
{
|
|
51
|
+
"vault_version": "2",
|
|
52
|
+
"format": "tarvault",
|
|
53
|
+
"created_at": "2026-07-11 20:32:25",
|
|
54
|
+
"gems": [
|
|
55
|
+
{ "name": "addressable", "version": "2.9.0", "platform": "ruby",
|
|
56
|
+
"created_at": "…", "sha256": "7fdf6ac3…", "encrypted": false }
|
|
57
|
+
]
|
|
58
|
+
}
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Doc Q1 — Integrity: are checksums used?
|
|
62
|
+
|
|
63
|
+
Yes. Three layers exist; the manifest adds the one that matters for the container:
|
|
64
|
+
|
|
65
|
+
1. **Tar header checksum** — every 512-byte tar header carries an octal
|
|
66
|
+
*byte-sum*. It protects the header only and is not cryptographic.
|
|
67
|
+
2. **Inside each `.gem`** — a `.gem` already contains `checksums.yaml.gz`
|
|
68
|
+
(SHA256/SHA512 of its own members). This protects a gem's internals, not its
|
|
69
|
+
identity within the vault.
|
|
70
|
+
3. **Manifest per-gem digest (added)** — each `manifest.json` record stores a
|
|
71
|
+
cryptographic **SHA256 of the whole `.gem` blob**
|
|
72
|
+
(`OpenSSL::Digest.new("SHA256")`, matching rubygems' own `Gem::Security`
|
|
73
|
+
convention). `Tarvault#gem_data` recomputes and compares on read, raising
|
|
74
|
+
`Vault::Error` on mismatch (verified by a test that flips a byte inside the
|
|
75
|
+
stored blob).
|
|
76
|
+
|
|
77
|
+
**Boundary:** `gem_entries`/`specs` read the manifest and do **not** re-hash
|
|
78
|
+
blobs; only `gem_data` (the byte-serving path used by install/extract) verifies.
|
|
79
|
+
This is a deliberate scope for the spike — list operations stay O(manifest). If
|
|
80
|
+
list-time verification is ever wanted, the same `Manifest.digest` is the hook.
|
|
81
|
+
|
|
82
|
+
## Doc feasibility — the one-liner
|
|
83
|
+
|
|
84
|
+
`docs/tarvault.md`'s one-liner works, but for a subtle reason worth recording:
|
|
85
|
+
|
|
86
|
+
```ruby
|
|
87
|
+
Enumerator.new { |y|
|
|
88
|
+
Gem::Package::TarReader.new(File.open(path)) { |r| r.each_entry { |e| y << Gem::Package.new(e) } }
|
|
89
|
+
}.map(&:spec)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
`TarReader#each` **closes each entry after yielding** (verified: reading a
|
|
93
|
+
stashed entry raises `IOError: closed Entry`). The one-liner still returns
|
|
94
|
+
correct specs *only* because `Enumerator.new{}.map(&:spec)` interleaves
|
|
95
|
+
consumption — `.spec` runs while the entry is still open. Materialize the
|
|
96
|
+
packages first (`.to_a` then `map(&:spec)`, or stash the entries) and every
|
|
97
|
+
`.spec` raises `IOError`.
|
|
98
|
+
|
|
99
|
+
**Rule:** read specs eagerly, inside the iteration. Tarvault sidesteps the trap
|
|
100
|
+
entirely — it lists from the manifest and reads any single gem via
|
|
101
|
+
`TarReader#seek` into a private `StringIO`
|
|
102
|
+
(`Gem::Package.new(StringIO.new(entry.read)).spec`), never depending on
|
|
103
|
+
post-close entry behavior.
|
|
104
|
+
|
|
105
|
+
## Doc Q2 — Encryption: is it possible?
|
|
106
|
+
|
|
107
|
+
Yes (feasibility only — not implemented). `OpenSSL::Cipher` AES-256-GCM
|
|
108
|
+
round-trips raw `.gem` bytes cleanly (verified, incl. auth-tag failure on
|
|
109
|
+
tamper). The design already reserves the extension point: `Manifest::Record`
|
|
110
|
+
carries an `encrypted` flag (currently always `false`). Encryption would:
|
|
111
|
+
|
|
112
|
+
- encrypt the raw `.gem` bytes, store the ciphertext as the tar entry, and set
|
|
113
|
+
`encrypted: true` plus cipher/iv/auth-tag on the record;
|
|
114
|
+
- `gem_data` decrypts (key from env/keyring) before returning, composing with
|
|
115
|
+
the checksum by hashing the plaintext.
|
|
116
|
+
|
|
117
|
+
Because gemvault mediates every read and the manifest signals encryption, this
|
|
118
|
+
is a localized change in `gem_data`. `OpenSSL` ships on JRuby, so it does not
|
|
119
|
+
reintroduce a portability problem.
|
|
120
|
+
|
|
121
|
+
## Portability (the motivation)
|
|
122
|
+
|
|
123
|
+
The tar path uses only pure-Ruby rubygems tooling (`Gem::Package::TarReader`/
|
|
124
|
+
`TarWriter`), `OpenSSL`, and `JSON` — all present on JRuby. The facade lazily
|
|
125
|
+
`require`s `dbvault` (hence `sqlite3`) **only** when opening an existing SQLite
|
|
126
|
+
file, so a JRuby process that only touches Tarvaults never loads `sqlite3`.
|
|
127
|
+
|
|
128
|
+
**Done:** `sqlite3` is no longer a runtime dependency. It is removed from the
|
|
129
|
+
gemspec and loaded lazily only to read a legacy SQLite vault; if it is absent,
|
|
130
|
+
`Vault.build_dbvault` raises a clear "install sqlite3 or upgrade the vault"
|
|
131
|
+
error. A process that only touches Tarvaults never loads it (verified by a
|
|
132
|
+
subprocess spec asserting `defined?(SQLite3)` is `nil`).
|
|
133
|
+
|
|
134
|
+
## Atomicity & locking
|
|
135
|
+
|
|
136
|
+
Tar has no index, and the leading `manifest.json` changes size on every
|
|
137
|
+
mutation, so **every add/remove is a full rewrite** — to a sibling tempfile in
|
|
138
|
+
the target directory (same filesystem → atomic `File.rename`), with `fsync`
|
|
139
|
+
before rename. `flock` (the doc's suggestion) would guard concurrent *gemvault*
|
|
140
|
+
processes only; external tar edits remain unsupported by design and are out of
|
|
141
|
+
spike scope.
|
|
142
|
+
|
|
143
|
+
## Follow-on / productionization notes
|
|
144
|
+
|
|
145
|
+
- Streaming rewrite: the current model reads all gem blobs into memory during a
|
|
146
|
+
rewrite — fine for a spike; large vaults would want a copy-through stream.
|
|
147
|
+
- Encryption (`encrypted` flag → real cipher params).
|
|
148
|
+
- List-time integrity verification, if wanted.
|
|
149
|
+
- Unify the two backends' error taxonomy and temp-file logic once Tarvault is
|
|
150
|
+
confirmed as "the way."
|
|
151
|
+
|
|
152
|
+
There is deliberately **no** format-selection flag on `gemvault new`. Tarvault is
|
|
153
|
+
the way forward; new vaults are always Tarvaults. A user who needs a SQLite vault
|
|
154
|
+
uses an older gemvault.
|
|
155
|
+
|
|
156
|
+
## Update — format versioning & `gemvault upgrade` (implemented)
|
|
157
|
+
|
|
158
|
+
A follow-up shipped explicit format versioning and a migration command:
|
|
159
|
+
|
|
160
|
+
- Each vault records an on-disk **format version** (`1` = Dbvault, `2` = Tarvault)
|
|
161
|
+
read and validated on open. It is **decoupled from the gemvault gem version** —
|
|
162
|
+
it bumps only when the byte layout changes. `Vault::UnsupportedVersionError` is
|
|
163
|
+
raised for a vault newer than `Vault::CURRENT_FORMAT` instead of misreading it,
|
|
164
|
+
closing the "reads fail open" hole.
|
|
165
|
+
- `Vault.backend_for` now positively identifies the container (`:sqlite`/`:tar`/
|
|
166
|
+
`:unknown`) ahead of parsing and refuses unrecognized envelopes.
|
|
167
|
+
- `Gemvault::VaultUpgrade` (+ `gemvault upgrade`) reads a vault through its
|
|
168
|
+
existing backend and rewrites it in the current format with an atomic swap,
|
|
169
|
+
a default `.bak` backup, `--dry-run`, idempotency, and `created_at` preservation
|
|
170
|
+
(the write path gained `add(created_at:)`).
|
|
171
|
+
|
|
172
|
+
`sqlite3` is now optional (removed from the gemspec, loaded lazily, clear error
|
|
173
|
+
when absent). There is no `gemvault new --format` flag by design — Tarvault is
|
|
174
|
+
the only write format.
|
|
175
|
+
|
|
176
|
+
The Dbvault deprecation lifecycle is decided and phase 1 is shipped:
|
|
177
|
+
|
|
178
|
+
- **Now:** SQLite vaults are **read-only** (`Dbvault#add`/`#remove` raise
|
|
179
|
+
`Vault::ReadOnlyError` → `gemvault upgrade`), reads still work, and opening one
|
|
180
|
+
emits a once-per-process notice via `Gemvault::Deprecation` (stderr, silenceable
|
|
181
|
+
with `GEMVAULT_SILENCE_DEPRECATIONS=1`, suppressed during `gemvault upgrade`).
|
|
182
|
+
"Writable" tracks *created-this-session* so migration/legacy tooling that
|
|
183
|
+
constructs a Dbvault directly still works.
|
|
184
|
+
- **0.3–0.5:** Dbvault removed entirely.
|
|
185
|
+
|
|
186
|
+
Still open: streaming rewrite and encryption.
|
data/docs/tarvault.md
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# GEMVAULT TAR SPIKE
|
|
2
|
+
|
|
3
|
+
## Problem
|
|
4
|
+
|
|
5
|
+
Gemvault has been working thus far, but the sqlite3 dependency makes the gem less portable then it should be:
|
|
6
|
+
|
|
7
|
+
1. The whole idea and basis of gemvault is that it _is_ a portable gem server in a file, and you shouldn't need dependencies to use it.
|
|
8
|
+
2. sqlite3 is not supported by jruby
|
|
9
|
+
|
|
10
|
+
## Solution
|
|
11
|
+
|
|
12
|
+
Since gemvault is just reading blobs from sqlite3, and doesn't really need any of its other stuff, I propose using a simpler storage model - a tarball. *.gem files, otherwise known as Packages, already are stored in this format, and all of the necessary equipment for reading them is already in rubygems, so it's a natural fit. If we are really worried about integrity that sqlite3 provides, we can use flock or similar to do locking. It is not intended or recommended to manipulate the tar version of the gemv file without going through gemvault, so anything beyond that is not supported. We shall refer to the tar version of a gemv file as a Tarvault and the sqlite3 version as a Dbvault to ensure communication is clear.
|
|
13
|
+
|
|
14
|
+
One can read all the gemspecs from a Tarvault using this chained oneliner:
|
|
15
|
+
|
|
16
|
+
```ruby
|
|
17
|
+
gemspecs = Enumerator.new { |y| Gem::Package::TarReader.new(File.open("tarvault.gemv")) { |reader| reader.each_entry { y << Gem::Package.new(it) } } }.map(&:spec)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
All of the dependencies are already provided.
|
|
23
|
+
|
|
24
|
+
## Things to think about
|
|
25
|
+
|
|
26
|
+
1. A Dbvault makes it trivial to store additional information about a given Package. I think we can meet any needs for metadata via a json manifest or metadata file to keep as the very first entry in a Tarvault. So structure would resemble something like:
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
$ tar -tf tarvault.gemv
|
|
30
|
+
|
|
31
|
+
manifest.json
|
|
32
|
+
myprivategem-0.1.0.gem
|
|
33
|
+
rails-8.0.0.gem
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
1. For files like this, e.g. files within a file, what is the usual way to maintain integrity? Are checksums used?
|
|
38
|
+
2. Encryption can be supported by encrypting a Package directly, and appending the raw bytes directly to the Tarvault. The manifest can indicate that they are encrypted. OpenSSL can be used all the way through (Encryption is not to be done on this spike, but whether it is possible is a key factor)
|
|
39
|
+
|
|
40
|
+
## Notes
|
|
41
|
+
|
|
42
|
+
1. gemvault is below 1.0.0 at the moment and therefore does not guarantee any backwards compatibility with sqlite3 version or anything for that matter. Keeping this compatitble with the Dbvault is not something you should keep in mind. Move forward. Users can use an old version of gemvault if we decide that Tarvault is _the way_
|
|
@@ -8,16 +8,28 @@ module Bundler
|
|
|
8
8
|
def initialize(opts)
|
|
9
9
|
super
|
|
10
10
|
@vault_path = resolve_vault_path(@uri)
|
|
11
|
+
@allow_remote = false
|
|
11
12
|
end
|
|
12
13
|
|
|
13
14
|
def fetch_gemspec_files
|
|
14
15
|
validate_vault_exists!
|
|
15
16
|
|
|
16
17
|
Gemvault::Vault.open(@vault_path) do |vault|
|
|
17
|
-
vault.gem_entries.
|
|
18
|
+
vault.gem_entries.filter_map { |entry| gemspec_file_for(vault, entry) }
|
|
18
19
|
end
|
|
19
20
|
end
|
|
20
21
|
|
|
22
|
+
# Bundler first asks a source, in local mode, which gems are already
|
|
23
|
+
# unpacked so it can tell what still needs installing, then switches to
|
|
24
|
+
# remote mode to resolve and install the rest. A vault gem is not on the
|
|
25
|
+
# load path until it is unpacked, so advertising one that still lives only
|
|
26
|
+
# inside the archive makes Bundler believe it is installed: it skips the
|
|
27
|
+
# install and the later require fails. Advertise not-yet-unpacked gems
|
|
28
|
+
# only once Bundler has asked for remote specs.
|
|
29
|
+
def remote!
|
|
30
|
+
@allow_remote = true
|
|
31
|
+
end
|
|
32
|
+
|
|
21
33
|
def install(spec, opts = {})
|
|
22
34
|
gem_dir = gem_dir_for(spec.full_name)
|
|
23
35
|
return use_installed_gem(spec, gem_dir) if File.directory?(gem_dir) && !opts[:force]
|
|
@@ -46,6 +58,7 @@ module Bundler
|
|
|
46
58
|
spec = vault.spec_from_blob(entry.name, entry.version, entry.platform)
|
|
47
59
|
gem_dir = gem_dir_for(spec.full_name)
|
|
48
60
|
return anchor_gemspec(gem_dir, spec.full_name, spec.to_ruby) if File.directory?(gem_dir)
|
|
61
|
+
return nil unless @allow_remote
|
|
49
62
|
|
|
50
63
|
write_tmp_gemspec(spec.full_name, spec.to_ruby)
|
|
51
64
|
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
require_relative "../command"
|
|
2
|
+
require_relative "../../vault_upgrade"
|
|
3
|
+
|
|
4
|
+
module Gemvault
|
|
5
|
+
class CLI
|
|
6
|
+
module Commands
|
|
7
|
+
# Upgrades a vault to the current storage format (e.g. a SQLite Dbvault
|
|
8
|
+
# to a Tarvault), preserving every gem and its timestamp.
|
|
9
|
+
class Upgrade < Command
|
|
10
|
+
description "Upgrade a vault to the current storage format"
|
|
11
|
+
|
|
12
|
+
argument :vault, required: true,
|
|
13
|
+
usage: "VAULT",
|
|
14
|
+
desc: "Vault file"
|
|
15
|
+
|
|
16
|
+
option :dry_run, desc: "Show what would change without modifying the vault"
|
|
17
|
+
option :no_backup, desc: "Do not write a .bak copy before upgrading"
|
|
18
|
+
|
|
19
|
+
def run(vault)
|
|
20
|
+
upgrade = Gemvault::VaultUpgrade.new(vault, backup: !options[:no_backup])
|
|
21
|
+
summary = upgrade.plan
|
|
22
|
+
return report_no_op(vault, summary) if summary.no_op?
|
|
23
|
+
return report_dry_run(vault, summary) if options[:dry_run]
|
|
24
|
+
|
|
25
|
+
upgrade.call
|
|
26
|
+
report_upgraded(vault, summary)
|
|
27
|
+
rescue Gemvault::Vault::Error => e
|
|
28
|
+
print_error(e.message)
|
|
29
|
+
exit(1)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def report_no_op(vault, summary)
|
|
35
|
+
puts "#{vault} is already current (format #{summary.to_version})"
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def report_dry_run(vault, summary)
|
|
39
|
+
puts "Would upgrade #{vault}: #{summary} (#{summary.gem_count} gems)"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def report_upgraded(vault, summary)
|
|
43
|
+
puts "Upgraded #{vault}: #{summary} (#{summary.gem_count} gems)"
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
require "sqlite3"
|
|
2
|
+
require_relative "vault"
|
|
3
|
+
require_relative "vault_session"
|
|
4
|
+
require_relative "gem_extraction"
|
|
5
|
+
require_relative "gem_entry"
|
|
6
|
+
require_relative "deprecation"
|
|
7
|
+
|
|
8
|
+
module Gemvault
|
|
9
|
+
# Read-only reader for a SQLite-backed vault (a "Dbvault"). Attempts to write
|
|
10
|
+
# are refused with a pointer to run gemvault upgrade, and opening a vault
|
|
11
|
+
# emits a one-time notice.
|
|
12
|
+
class Dbvault
|
|
13
|
+
extend VaultSession
|
|
14
|
+
include GemExtraction
|
|
15
|
+
|
|
16
|
+
FORMAT_VERSION = 1
|
|
17
|
+
|
|
18
|
+
attr_reader :path
|
|
19
|
+
|
|
20
|
+
def initialize(path)
|
|
21
|
+
@path = File.expand_path(path)
|
|
22
|
+
open_vault!
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def add(*)
|
|
26
|
+
raise Vault::ReadOnlyError, read_only_message
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def remove(*)
|
|
30
|
+
raise Vault::ReadOnlyError, read_only_message
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def gem_data(name, version, platform: "ruby")
|
|
34
|
+
row = @db.execute(
|
|
35
|
+
"SELECT data FROM gems WHERE name = ? AND version = ? AND platform = ?",
|
|
36
|
+
[name, version, platform],
|
|
37
|
+
).first
|
|
38
|
+
raise Vault::NotFoundError, "Gem not found: #{name}-#{version} (#{platform})" unless row
|
|
39
|
+
|
|
40
|
+
row["data"]
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def gem_entries
|
|
44
|
+
@db.execute(
|
|
45
|
+
"SELECT name, version, platform, created_at FROM gems ORDER BY name, version",
|
|
46
|
+
).map { |row| GemEntry.new(**row.transform_keys(&:to_sym)) }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def size
|
|
50
|
+
@db.execute("SELECT COUNT(*) AS count FROM gems").first["count"]
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def close
|
|
54
|
+
@db.close if @db && !@db.closed?
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def closed?
|
|
58
|
+
@db.nil? || @db.closed?
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def format_version
|
|
62
|
+
row = @db.execute("SELECT value FROM metadata WHERE key = 'vault_version'").first
|
|
63
|
+
row ? row["value"].to_i : FORMAT_VERSION
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
def open_vault!
|
|
69
|
+
raise Vault::NotFoundError, "Vault not found: #{@path}" unless File.exist?(@path)
|
|
70
|
+
|
|
71
|
+
validate_sqlite!
|
|
72
|
+
@db = new_database
|
|
73
|
+
Vault.assert_readable!(format_version, @path)
|
|
74
|
+
Deprecation.warn_once(deprecation_message)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def new_database
|
|
78
|
+
db = SQLite3::Database.new(@path)
|
|
79
|
+
db.results_as_hash = true
|
|
80
|
+
db
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def validate_sqlite!
|
|
84
|
+
return if File.binread(@path, Vault::SQLITE_MAGIC.bytesize) == Vault::SQLITE_MAGIC
|
|
85
|
+
|
|
86
|
+
raise Vault::Error, "Not a valid vault file (not SQLite): #{@path}"
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def read_only_message
|
|
90
|
+
"#{@path} uses the deprecated read-only SQLite format. Migrate it first: gemvault upgrade #{@path}"
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def deprecation_message
|
|
94
|
+
"SQLite vaults are deprecated and read-only; support will be removed in a future release (0.3-0.5). " \
|
|
95
|
+
"Migrate #{@path} with: gemvault upgrade #{@path}"
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
module Gemvault
|
|
2
|
+
# Emits deprecation notices once per unique message, to a configurable IO
|
|
3
|
+
# (stderr in production). Silenceable per-block or via an environment opt-out
|
|
4
|
+
# so automated pipelines that consume a legacy vault are not spammed.
|
|
5
|
+
module Deprecation
|
|
6
|
+
ENV_KEY = "GEMVAULT_SILENCE_DEPRECATIONS".freeze
|
|
7
|
+
|
|
8
|
+
class << self
|
|
9
|
+
attr_writer :output
|
|
10
|
+
|
|
11
|
+
def output
|
|
12
|
+
@output ||= $stderr
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def warn_once(message)
|
|
16
|
+
return if silenced? || seen.include?(message)
|
|
17
|
+
|
|
18
|
+
seen << message
|
|
19
|
+
output.puts("gemvault: #{message}")
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def silence
|
|
23
|
+
previous = @silenced
|
|
24
|
+
@silenced = true
|
|
25
|
+
yield
|
|
26
|
+
ensure
|
|
27
|
+
@silenced = previous
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def silenced?
|
|
31
|
+
@silenced || ENV.key?(ENV_KEY)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def reset!
|
|
35
|
+
@seen = []
|
|
36
|
+
@silenced = false
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def seen
|
|
42
|
+
@seen ||= []
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|