thuban 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/README.md +150 -7
- data/lib/thuban/pack_reader.rb +216 -0
- data/lib/thuban/remote/credentials.rb +209 -0
- data/lib/thuban/remote/fetch.rb +287 -0
- data/lib/thuban/remote/http.rb +233 -0
- data/lib/thuban/remote/local.rb +55 -0
- data/lib/thuban/remote/protocol.rb +91 -0
- data/lib/thuban/remote/push.rb +250 -0
- data/lib/thuban/remote/repository.rb +223 -0
- data/lib/thuban/remote/ssh.rb +248 -0
- data/lib/thuban/remote.rb +27 -0
- data/lib/thuban/version.rb +1 -1
- data/lib/thuban.rb +10 -0
- data/sig/thuban.rbs +65 -0
- metadata +14 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7115dc5cf395114ca07172c0746db9d0efb228092455b2a0fc7ec069a01b7e7a
|
|
4
|
+
data.tar.gz: 5ed48fad9aab26581024ca4c96100e53f2072fdab5727acb37cad4ce5b34872c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ab9dd86e88c0cf633d21a6489879415ca329146a7aa6aace8eac91b11280e6fe759ce7c11a453a96f8e409b35d0ddb2530b05f6dc8fa8225c7ce5c3f71a6dfab
|
|
7
|
+
data.tar.gz: e99576b1552149f56fe50b6730743d5afe6efa475b6879e2146ffef81cc106eaa94eb1d907ab3fac53248bbc92c3e32a1aab46783073b6307e2888d68acebf4a
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
## 0.4.0 - 2026-09-15
|
|
6
|
+
|
|
7
|
+
- Add pkt-line parsing and smart HTTP protocol v2/v0 reference discovery
|
|
8
|
+
- Add smart HTTP fetch negotiation and bounded pack ingestion
|
|
9
|
+
- Add configured repository remotes and remote-tracking ref updates
|
|
10
|
+
- Add redacted Basic, Bearer, callback, and Git credential-helper authentication
|
|
11
|
+
- Add bounded SSH ref discovery and fetch through the system SSH client
|
|
12
|
+
- Add local, smart HTTP, and SSH push with refspecs, leases, atomic updates, and progress
|
|
13
|
+
- Add fetch progress, shallow depth negotiation, and `blob:none` partial fetches
|
|
14
|
+
|
|
3
15
|
## 0.3.0 - 2026-09-15
|
|
4
16
|
|
|
5
17
|
- Add loose object, tree, and commit writing
|
data/README.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
<h1 align="center">Thuban</h1>
|
|
2
2
|
|
|
3
3
|
<p align="center">
|
|
4
|
-
<strong>A pure Ruby implementation for local
|
|
4
|
+
<strong>A pure Ruby Git implementation for local repositories and remote transfers</strong>
|
|
5
5
|
</p>
|
|
6
6
|
|
|
7
7
|
<p align="center">
|
|
@@ -23,8 +23,8 @@
|
|
|
23
23
|
---
|
|
24
24
|
|
|
25
25
|
Thuban is a pure Ruby Git implementation for reading and writing local
|
|
26
|
-
repositories
|
|
27
|
-
executable.
|
|
26
|
+
repositories and transferring data with local, smart HTTP, or SSH remotes. Local
|
|
27
|
+
repository operations work directly with Git data without invoking the Git executable.
|
|
28
28
|
|
|
29
29
|
## Features
|
|
30
30
|
|
|
@@ -34,6 +34,11 @@ executable.
|
|
|
34
34
|
- Writes loose objects, index entries, refs, reflogs, trees, and commits
|
|
35
35
|
- Finds merge bases and performs reset, cherry-pick, revert, and stash operations
|
|
36
36
|
- Writes interoperable delta-free Git packfiles to any writable IO
|
|
37
|
+
- Discovers smart HTTP protocol v2 and v0 remote references
|
|
38
|
+
- Fetches smart HTTP packs into the local object database
|
|
39
|
+
- Authenticates smart HTTP with Basic, Bearer, callbacks, or Git credential helpers
|
|
40
|
+
- Pushes refs and packfiles with force-with-lease and atomic update support
|
|
41
|
+
- Discovers, fetches, and pushes SSH remotes through the system `ssh` executable
|
|
37
42
|
- Tracks line history across commits and renames with blame
|
|
38
43
|
- Writes files atomically and checks out branches with collision guards
|
|
39
44
|
- Supports linked worktrees and packed refs
|
|
@@ -43,7 +48,7 @@ executable.
|
|
|
43
48
|
Add Thuban to your Gemfile:
|
|
44
49
|
|
|
45
50
|
```ruby
|
|
46
|
-
gem "thuban", "~> 0.
|
|
51
|
+
gem "thuban", "~> 0.3.0"
|
|
47
52
|
```
|
|
48
53
|
|
|
49
54
|
Then install it:
|
|
@@ -194,6 +199,141 @@ end
|
|
|
194
199
|
|
|
195
200
|
The emitted PACK v2 stream stores complete compressed objects without delta
|
|
196
201
|
generation. It can be consumed by `git index-pack` and `git verify-pack`.
|
|
202
|
+
`Thuban::Pack.read_stream(io, repo.odb)` verifies and expands a received pack,
|
|
203
|
+
including offset and reference deltas, into the supplied object database.
|
|
204
|
+
|
|
205
|
+
### Inspect Remote References
|
|
206
|
+
|
|
207
|
+
Smart HTTP discovery prefers protocol v2 and falls back to v0 when necessary:
|
|
208
|
+
|
|
209
|
+
```ruby
|
|
210
|
+
connection = Thuban::Remote.open("https://example.com/project.git")
|
|
211
|
+
begin
|
|
212
|
+
connection.refs.each do |ref|
|
|
213
|
+
puts [ref.oid, ref.name, ref.symref_target, ref.peeled].compact.join(" ")
|
|
214
|
+
end
|
|
215
|
+
ensure
|
|
216
|
+
connection.close
|
|
217
|
+
end
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
Fetch selected object IDs directly, optionally providing local object IDs for
|
|
221
|
+
negotiation:
|
|
222
|
+
|
|
223
|
+
```ruby
|
|
224
|
+
wanted = connection.refs.find { |ref| ref.name == "refs/heads/main" }.oid
|
|
225
|
+
received_oids = connection.fetch(repo, wants: [wanted], haves: repo.refs.values.compact)
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
Authenticate with fixed Basic or Bearer credentials when appropriate:
|
|
229
|
+
|
|
230
|
+
```ruby
|
|
231
|
+
credentials = Thuban::Remote::Credentials.static(
|
|
232
|
+
username: ENV.fetch("GIT_USERNAME"),
|
|
233
|
+
password: ENV.fetch("GIT_PASSWORD")
|
|
234
|
+
)
|
|
235
|
+
connection = Thuban::Remote.open(remote_url, credentials: credentials)
|
|
236
|
+
|
|
237
|
+
token = Thuban::Remote::Credentials.bearer(token: ENV.fetch("GIT_TOKEN"))
|
|
238
|
+
connection = Thuban::Remote.open(remote_url, credentials: token)
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
For credentials that are selected or refreshed at runtime, return another
|
|
242
|
+
credential object from a callback. The callback receives the remote URL with no
|
|
243
|
+
embedded user information:
|
|
244
|
+
|
|
245
|
+
```ruby
|
|
246
|
+
credentials = Thuban::Remote::Credentials.callback do |url|
|
|
247
|
+
Thuban::Remote::Credentials.bearer(token: token_for(url))
|
|
248
|
+
end
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
Use the normal configured Git credential helpers, or select a helper by name:
|
|
252
|
+
|
|
253
|
+
```ruby
|
|
254
|
+
credentials = Thuban::Remote::Credentials.helper
|
|
255
|
+
credentials = Thuban::Remote::Credentials.helper("store --file=/secure/path")
|
|
256
|
+
connection = Thuban::Remote.open(remote_url, credentials: credentials)
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Helper lookup uses `git credential fill` with terminal prompting disabled and is
|
|
260
|
+
bounded by the connection timeout. Thuban never includes credentials, helper
|
|
261
|
+
output, response bodies, or remote URLs in transport errors. Credential objects
|
|
262
|
+
also redact their inspection output. URL-embedded credentials and HTTP redirects
|
|
263
|
+
remain rejected so an authorization header cannot be forwarded to another
|
|
264
|
+
origin.
|
|
265
|
+
|
|
266
|
+
For a repository with a normal `[remote "origin"]` configuration, the
|
|
267
|
+
high-level operation reads its fetch refspec and updates remote-tracking refs:
|
|
268
|
+
|
|
269
|
+
```ruby
|
|
270
|
+
repo.remotes # => {"origin" => "https://example.com/project.git"}
|
|
271
|
+
remote_refs = repo.fetch("origin")
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
Limit downloaded history or omit blobs while reporting bounded transfer and pack
|
|
275
|
+
progress:
|
|
276
|
+
|
|
277
|
+
```ruby
|
|
278
|
+
repo.fetch("origin", depth: 1) { |event| warn "#{event.phase}: #{event.bytes}" }
|
|
279
|
+
repo.fetch("origin", filter: "blob:none")
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
`depth:` accepts positive 32-bit integers and `filter:` currently accepts only
|
|
283
|
+
`"blob:none"`. Thuban negotiates only features advertised by the server. Shallow
|
|
284
|
+
boundaries are atomically maintained in Git's `shallow` file. For configured
|
|
285
|
+
remotes, partial fetches also set Git's `remote.<name>.promisor` and
|
|
286
|
+
`remote.<name>.partialclonefilter` keys, so the Git executable can retrieve an
|
|
287
|
+
omitted object later. Reading an omitted blob through Thuban raises `KeyError`;
|
|
288
|
+
automatic promisor-object retrieval remains the caller's responsibility.
|
|
289
|
+
Progress `bytes` are cumulative for `:pack` events and the current sideband
|
|
290
|
+
message size for `:remote` events, matching push progress.
|
|
291
|
+
|
|
292
|
+
Push one or more explicit refspecs. Normal updates must be fast-forward; use a
|
|
293
|
+
lease for a guarded rewrite, or `force: true` for an unconditional one:
|
|
294
|
+
|
|
295
|
+
```ruby
|
|
296
|
+
repo.push("origin", refspecs: "refs/heads/main:refs/heads/main")
|
|
297
|
+
repo.push("origin", refspecs: "refs/heads/topic:refs/heads/topic",
|
|
298
|
+
lease: expected_remote_oid)
|
|
299
|
+
repo.push("origin", refspecs: [
|
|
300
|
+
"refs/heads/main:refs/heads/main",
|
|
301
|
+
"refs/tags/v1:refs/tags/v1"
|
|
302
|
+
], atomic: true) { |progress| warn "#{progress.phase}: #{progress.current}/#{progress.total}" }
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
Remote names, direct paths, `file://` URLs, HTTP(S), and SSH URLs are accepted.
|
|
306
|
+
At the lower level, `Connection#push` accepts `[ref, old_oid, new_oid]` updates;
|
|
307
|
+
the old object ID is an optimistic lease, and `nil` uses the advertised value.
|
|
308
|
+
Deletion uses an empty source refspec such as `:refs/heads/topic`.
|
|
309
|
+
|
|
310
|
+
Fetched packs are checksum-verified, bounded by byte/object/expanded-size
|
|
311
|
+
limits, and expanded through the existing object database. Redirects and
|
|
312
|
+
URL-embedded credentials remain rejected.
|
|
313
|
+
|
|
314
|
+
SSH remotes accept both standard URL and scp-like forms. Thuban invokes the
|
|
315
|
+
system SSH client without a local shell and uses `git-upload-pack` or
|
|
316
|
+
`git-receive-pack` over its
|
|
317
|
+
standard input and output:
|
|
318
|
+
|
|
319
|
+
```ruby
|
|
320
|
+
connection = Thuban::Remote.open("ssh://git@example.com/project.git")
|
|
321
|
+
connection = Thuban::Remote.open("git@example.com:project.git")
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
Pass `ssh:` as an argument array or safely parsed command string to select a
|
|
325
|
+
client and options. When omitted, `GIT_SSH_COMMAND` is parsed into arguments, or
|
|
326
|
+
`ssh` is used by default:
|
|
327
|
+
|
|
328
|
+
```ruby
|
|
329
|
+
connection = Thuban::Remote.open(remote_url, ssh: ["ssh", "-F", "/safe/config"])
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
SSH runs in batch mode, is bounded by `timeout:`, and discards stderr so remote
|
|
333
|
+
paths and server diagnostics are not copied into exceptions. User, host, port,
|
|
334
|
+
and path syntax is validated before process startup. Passwords in SSH URLs are
|
|
335
|
+
not supported; use normal SSH agents and configuration instead. SSH fetches use
|
|
336
|
+
the same shallow, partial, sideband progress, and pack validation paths as HTTP.
|
|
197
337
|
|
|
198
338
|
### Match Ignored Paths
|
|
199
339
|
|
|
@@ -216,9 +356,11 @@ continuations, and command-scoped overrides are not evaluated.
|
|
|
216
356
|
|
|
217
357
|
The current write API covers loose objects, the index, refs, reflogs, commits,
|
|
218
358
|
guarded checkout, local history operations, stash, and delta-free pack output.
|
|
219
|
-
Thuban
|
|
220
|
-
|
|
221
|
-
|
|
359
|
+
Thuban can fetch and push local, smart HTTP, and SSH remotes, including shallow
|
|
360
|
+
and blobless fetches. Merges, automatic retrieval of omitted promisor objects,
|
|
361
|
+
and pack delta generation are outside the current scope. It does not provide
|
|
362
|
+
its own diff algorithm; blame delegates line matching to Porrima through the
|
|
363
|
+
injectable `differ:` argument.
|
|
222
364
|
|
|
223
365
|
Support is limited to the index and pack formats covered by the test suite.
|
|
224
366
|
Submodule checkout and optional Git extensions outside that coverage are not
|
|
@@ -233,6 +375,7 @@ ruby tools/check_isolation.rb
|
|
|
233
375
|
bundle exec rbs -I sig -r porrima validate
|
|
234
376
|
gem build --strict thuban.gemspec
|
|
235
377
|
ruby bench/pack_write.rb --assert
|
|
378
|
+
ruby bench/ssh_fetch.rb --assert
|
|
236
379
|
```
|
|
237
380
|
|
|
238
381
|
## Contributing
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Thuban
|
|
4
|
+
class Pack
|
|
5
|
+
MAX_PACK_SIZE = 1024 * 1024 * 1024
|
|
6
|
+
MAX_PACK_OBJECTS = 1_000_000
|
|
7
|
+
MAX_DEFERRED_DELTA_BYTES = 128 * 1024 * 1024
|
|
8
|
+
MAX_EXPANDED_PACK_SIZE = 4 * 1024 * 1024 * 1024
|
|
9
|
+
|
|
10
|
+
def self.read_stream(io, odb)
|
|
11
|
+
raise TypeError, "pack input must respond to read" unless io.respond_to?(:read)
|
|
12
|
+
raise TypeError, "expected Thuban::ObjectDatabase" unless odb.is_a?(ObjectDatabase)
|
|
13
|
+
|
|
14
|
+
Tempfile.create(["thuban-pack-", ".pack"]) do |file|
|
|
15
|
+
file.binmode
|
|
16
|
+
copy_stream(io, file, MAX_PACK_SIZE)
|
|
17
|
+
file.flush
|
|
18
|
+
unpack_file(file, odb) { |current, total| yield current, total if block_given? }
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.copy_stream(input, output, limit)
|
|
23
|
+
total = 0
|
|
24
|
+
loop do
|
|
25
|
+
chunk = input.read(65_536)
|
|
26
|
+
break if chunk.nil?
|
|
27
|
+
raise CorruptObject, "pack input stopped before EOF" unless chunk.is_a?(String) && !chunk.empty?
|
|
28
|
+
|
|
29
|
+
total += chunk.bytesize
|
|
30
|
+
raise CorruptObject, "pack exceeds size limit" if total > limit
|
|
31
|
+
|
|
32
|
+
output.write(chunk)
|
|
33
|
+
end
|
|
34
|
+
total
|
|
35
|
+
end
|
|
36
|
+
private_class_method :copy_stream
|
|
37
|
+
|
|
38
|
+
def self.unpack_file(file, odb)
|
|
39
|
+
size = file.size
|
|
40
|
+
raise CorruptObject, "truncated pack" if size < 32
|
|
41
|
+
pack_end = size - 20
|
|
42
|
+
verify_pack_checksum(file, pack_end)
|
|
43
|
+
file.rewind
|
|
44
|
+
header = file.read(12)
|
|
45
|
+
valid = header.start_with?("PACK") && [2, 3].include?(header[4, 4].unpack1("N"))
|
|
46
|
+
raise CorruptObject, "invalid pack header" unless valid
|
|
47
|
+
|
|
48
|
+
count = header[8, 4].unpack1("N")
|
|
49
|
+
raise CorruptObject, "pack object count exceeds limit" if count > MAX_PACK_OBJECTS
|
|
50
|
+
resolved = {}
|
|
51
|
+
deferred = []
|
|
52
|
+
deferred_bytes = 0
|
|
53
|
+
expanded_bytes = 0
|
|
54
|
+
oids = []
|
|
55
|
+
count.times do
|
|
56
|
+
record = read_record(file, pack_end)
|
|
57
|
+
object = resolve_record(record, resolved, odb, MAX_EXPANDED_PACK_SIZE - expanded_bytes)
|
|
58
|
+
if object
|
|
59
|
+
oid, object_size = object
|
|
60
|
+
resolved[record[:offset]] = oid
|
|
61
|
+
oids << oid
|
|
62
|
+
expanded_bytes += object_size
|
|
63
|
+
yield oids.length, count if block_given?
|
|
64
|
+
else
|
|
65
|
+
deferred << record
|
|
66
|
+
deferred_bytes += record[:data].bytesize
|
|
67
|
+
raise CorruptObject, "deferred deltas exceed memory limit" if deferred_bytes > MAX_DEFERRED_DELTA_BYTES
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
raise CorruptObject, "pack object count mismatch" unless file.pos == pack_end
|
|
71
|
+
|
|
72
|
+
waiting_by_offset = Hash.new { |hash, key| hash[key] = [] }
|
|
73
|
+
waiting_by_oid = Hash.new { |hash, key| hash[key] = [] }
|
|
74
|
+
deferred.each do |record|
|
|
75
|
+
target = record[:base_oid] ? waiting_by_oid[record[:base_oid]] : waiting_by_offset[record[:base_offset]]
|
|
76
|
+
target << record
|
|
77
|
+
end
|
|
78
|
+
deferred.clear
|
|
79
|
+
queue = []
|
|
80
|
+
resolved.each do |offset, oid|
|
|
81
|
+
queue.concat(waiting_by_offset.delete(offset) || [])
|
|
82
|
+
queue.concat(waiting_by_oid.delete(oid) || [])
|
|
83
|
+
end
|
|
84
|
+
waiting_by_oid.keys.each do |oid|
|
|
85
|
+
queue.concat(waiting_by_oid.delete(oid)) if odb.exist?(oid)
|
|
86
|
+
end
|
|
87
|
+
cursor = 0
|
|
88
|
+
while cursor < queue.length
|
|
89
|
+
record = queue[cursor]
|
|
90
|
+
cursor += 1
|
|
91
|
+
object = resolve_record(record, resolved, odb, MAX_EXPANDED_PACK_SIZE - expanded_bytes)
|
|
92
|
+
raise CorruptObject, "unresolved delta base" unless object
|
|
93
|
+
|
|
94
|
+
oid, object_size = object
|
|
95
|
+
resolved[record[:offset]] = oid
|
|
96
|
+
oids << oid
|
|
97
|
+
expanded_bytes += object_size
|
|
98
|
+
yield oids.length, count if block_given?
|
|
99
|
+
queue.concat(waiting_by_offset.delete(record[:offset]) || [])
|
|
100
|
+
queue.concat(waiting_by_oid.delete(oid) || [])
|
|
101
|
+
end
|
|
102
|
+
raise CorruptObject, "unresolved delta base" unless waiting_by_offset.empty? && waiting_by_oid.empty?
|
|
103
|
+
|
|
104
|
+
oids
|
|
105
|
+
end
|
|
106
|
+
private_class_method :unpack_file
|
|
107
|
+
|
|
108
|
+
def self.verify_pack_checksum(file, pack_end)
|
|
109
|
+
digest = Digest::SHA1.new
|
|
110
|
+
file.rewind
|
|
111
|
+
remaining = pack_end
|
|
112
|
+
while remaining.positive?
|
|
113
|
+
chunk = file.read([remaining, 65_536].min)
|
|
114
|
+
raise CorruptObject, "truncated pack" unless chunk&.bytesize&.positive?
|
|
115
|
+
|
|
116
|
+
digest.update(chunk)
|
|
117
|
+
remaining -= chunk.bytesize
|
|
118
|
+
end
|
|
119
|
+
expected = file.read(20)
|
|
120
|
+
raise CorruptObject, "pack checksum mismatch" unless expected&.bytesize == 20 && digest.digest == expected
|
|
121
|
+
end
|
|
122
|
+
private_class_method :verify_pack_checksum
|
|
123
|
+
|
|
124
|
+
def self.read_record(file, pack_end)
|
|
125
|
+
offset = file.pos
|
|
126
|
+
byte = file.getbyte
|
|
127
|
+
raise CorruptObject, "truncated pack object" unless byte
|
|
128
|
+
|
|
129
|
+
type = (byte >> 4) & 7
|
|
130
|
+
size = byte & 15
|
|
131
|
+
shift = 4
|
|
132
|
+
while (byte & 0x80).positive?
|
|
133
|
+
byte = file.getbyte
|
|
134
|
+
raise CorruptObject, "truncated pack object size" unless byte && shift <= 63
|
|
135
|
+
|
|
136
|
+
size |= (byte & 0x7f) << shift
|
|
137
|
+
shift += 7
|
|
138
|
+
end
|
|
139
|
+
raise CorruptObject, "pack object too large" if size > MAX_OBJECT_SIZE
|
|
140
|
+
|
|
141
|
+
record = {offset: offset, type: TYPES[type]}
|
|
142
|
+
if type == 6
|
|
143
|
+
record[:base_offset] = offset - read_delta_distance(file)
|
|
144
|
+
raise CorruptObject, "invalid delta base offset" unless record[:base_offset] >= 12 && record[:base_offset] < offset
|
|
145
|
+
elsif type == 7
|
|
146
|
+
base = file.read(20)
|
|
147
|
+
raise CorruptObject, "truncated delta reference" unless base&.bytesize == 20
|
|
148
|
+
|
|
149
|
+
record[:base_oid] = base.unpack1("H*")
|
|
150
|
+
elsif !record[:type]
|
|
151
|
+
raise CorruptObject, "invalid packed object type #{type}"
|
|
152
|
+
end
|
|
153
|
+
record[:data] = inflate_record(file, size, pack_end)
|
|
154
|
+
record
|
|
155
|
+
end
|
|
156
|
+
private_class_method :read_record
|
|
157
|
+
|
|
158
|
+
def self.read_delta_distance(file)
|
|
159
|
+
byte = file.getbyte
|
|
160
|
+
raise CorruptObject, "truncated delta offset" unless byte
|
|
161
|
+
|
|
162
|
+
distance = byte & 0x7f
|
|
163
|
+
count = 0
|
|
164
|
+
while (byte & 0x80).positive?
|
|
165
|
+
byte = file.getbyte
|
|
166
|
+
count += 1
|
|
167
|
+
raise CorruptObject, "invalid delta offset" unless byte && count <= 9
|
|
168
|
+
|
|
169
|
+
distance = ((distance + 1) << 7) | (byte & 0x7f)
|
|
170
|
+
end
|
|
171
|
+
distance
|
|
172
|
+
end
|
|
173
|
+
private_class_method :read_delta_distance
|
|
174
|
+
|
|
175
|
+
def self.inflate_record(file, size, pack_end)
|
|
176
|
+
inflater = Zlib::Inflate.new
|
|
177
|
+
data = +"".b
|
|
178
|
+
start = file.pos
|
|
179
|
+
until inflater.finished?
|
|
180
|
+
remaining = pack_end - file.pos
|
|
181
|
+
raise CorruptObject, "truncated compressed object" unless remaining.positive?
|
|
182
|
+
|
|
183
|
+
chunk = file.read([remaining, 16_384].min)
|
|
184
|
+
inflater.inflate(chunk) do |part|
|
|
185
|
+
data << part
|
|
186
|
+
raise CorruptObject, "packed object exceeds declared size" if data.bytesize > size
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
file.seek(start + inflater.total_in, IO::SEEK_SET)
|
|
190
|
+
raise CorruptObject, "packed object size mismatch" unless data.bytesize == size
|
|
191
|
+
|
|
192
|
+
data
|
|
193
|
+
rescue Zlib::Error => error
|
|
194
|
+
raise CorruptObject, error.message
|
|
195
|
+
ensure
|
|
196
|
+
inflater&.close
|
|
197
|
+
end
|
|
198
|
+
private_class_method :inflate_record
|
|
199
|
+
|
|
200
|
+
def self.resolve_record(record, resolved, odb, remaining)
|
|
201
|
+
type = record[:type]
|
|
202
|
+
data = record[:data]
|
|
203
|
+
unless type
|
|
204
|
+
base_oid = record[:base_oid] || resolved[record[:base_offset]]
|
|
205
|
+
return unless base_oid && odb.exist?(base_oid)
|
|
206
|
+
|
|
207
|
+
type, base = odb.read(base_oid)
|
|
208
|
+
data = apply_delta(base, data)
|
|
209
|
+
end
|
|
210
|
+
raise CorruptObject, "expanded pack exceeds size limit" if data.bytesize > remaining
|
|
211
|
+
|
|
212
|
+
[odb.write(type, data), data.bytesize]
|
|
213
|
+
end
|
|
214
|
+
private_class_method :resolve_record
|
|
215
|
+
end
|
|
216
|
+
end
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "open3"
|
|
4
|
+
|
|
5
|
+
module Thuban
|
|
6
|
+
module Remote
|
|
7
|
+
module Credentials
|
|
8
|
+
MAX_HELPER_OUTPUT = 1024 * 1024
|
|
9
|
+
|
|
10
|
+
class Provider
|
|
11
|
+
def inspect = "#<#{self.class.name} [REDACTED]>"
|
|
12
|
+
end
|
|
13
|
+
class Basic < Provider
|
|
14
|
+
def initialize(username, password)
|
|
15
|
+
@username = username
|
|
16
|
+
@password = password
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def authorization(_url, timeout:)
|
|
20
|
+
"Basic #{["#{@username}:#{@password}"].pack("m0")}"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
class Bearer < Provider
|
|
24
|
+
def initialize(token) = @token = token
|
|
25
|
+
def authorization(_url, timeout:) = "Bearer #{@token}"
|
|
26
|
+
end
|
|
27
|
+
class Callback < Provider
|
|
28
|
+
def initialize(callback) = @callback = callback
|
|
29
|
+
|
|
30
|
+
def authorization(url, timeout:)
|
|
31
|
+
credential = @callback.call(url.to_s.freeze)
|
|
32
|
+
return if credential.nil?
|
|
33
|
+
raise AuthenticationError, "credential callback returned an invalid value", cause: nil unless credential.is_a?(Provider)
|
|
34
|
+
raise AuthenticationError, "credential callback cannot return itself", cause: nil if credential.equal?(self)
|
|
35
|
+
|
|
36
|
+
credential.authorization(url, timeout: timeout)
|
|
37
|
+
rescue AuthenticationError
|
|
38
|
+
raise
|
|
39
|
+
rescue StandardError
|
|
40
|
+
raise AuthenticationError, "credential callback failed", cause: nil
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
class Helper < Provider
|
|
45
|
+
def initialize(name) = @name = name
|
|
46
|
+
|
|
47
|
+
def authorization(url, timeout:)
|
|
48
|
+
fields = parse(run(input_for(url), timeout))
|
|
49
|
+
if fields["username"] && fields.key?("password")
|
|
50
|
+
return Credentials.static(username: fields["username"], password: fields["password"]).authorization(url, timeout: timeout)
|
|
51
|
+
end
|
|
52
|
+
if fields["authtype"]&.casecmp?("Bearer") && fields["credential"]
|
|
53
|
+
return Credentials.bearer(token: fields["credential"]).authorization(url, timeout: timeout)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
raise AuthenticationError, "credential helper returned no usable credentials", cause: nil
|
|
57
|
+
rescue ArgumentError
|
|
58
|
+
raise AuthenticationError, "credential helper returned invalid credentials", cause: nil
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def command
|
|
64
|
+
return ["git", "credential", "fill"] unless @name
|
|
65
|
+
|
|
66
|
+
["git", "-c", "credential.helper=", "-c", "credential.helper=#{@name}", "credential", "fill"]
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def input_for(url)
|
|
70
|
+
host = url.hostname
|
|
71
|
+
host = "[#{host}]" if host.include?(":")
|
|
72
|
+
host = "#{host}:#{url.port}" unless url.port == url.default_port
|
|
73
|
+
"capability[]=authtype\nprotocol=#{url.scheme}\nhost=#{host}\npath=#{url.path.delete_prefix("/")}\n\n"
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def run(input, timeout)
|
|
77
|
+
grouped = !Gem.win_platform?
|
|
78
|
+
options = grouped ? {pgroup: true} : {}
|
|
79
|
+
stdin, stdout, stderr, waiter = Open3.popen3(helper_environment, *command, **options)
|
|
80
|
+
errors = Thread.new { while stderr.read(4096); end rescue nil }
|
|
81
|
+
output_error = nil
|
|
82
|
+
output = Thread.new do
|
|
83
|
+
read_bounded(stdout)
|
|
84
|
+
rescue AuthenticationError => error
|
|
85
|
+
output_error = error
|
|
86
|
+
nil
|
|
87
|
+
rescue StandardError
|
|
88
|
+
output_error = AuthenticationError.new("credential helper failed")
|
|
89
|
+
nil
|
|
90
|
+
end
|
|
91
|
+
stdin.write(input)
|
|
92
|
+
stdin.close
|
|
93
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
|
|
94
|
+
while waiter.alive? || output.alive?
|
|
95
|
+
raise output_error if output_error
|
|
96
|
+
|
|
97
|
+
remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
98
|
+
raise AuthenticationError, "credential helper timed out", cause: nil unless remaining.positive?
|
|
99
|
+
|
|
100
|
+
(waiter.alive? ? waiter : output).join([remaining, 0.01].min)
|
|
101
|
+
end
|
|
102
|
+
raise output_error if output_error
|
|
103
|
+
raise AuthenticationError, "credential helper failed", cause: nil unless waiter.value.success?
|
|
104
|
+
|
|
105
|
+
output.value
|
|
106
|
+
rescue AuthenticationError
|
|
107
|
+
raise
|
|
108
|
+
rescue StandardError
|
|
109
|
+
raise AuthenticationError, "credential helper failed", cause: nil
|
|
110
|
+
ensure
|
|
111
|
+
terminate(waiter, grouped) if waiter&.alive?
|
|
112
|
+
[stdin, stdout, stderr].compact.each { |io| io.close unless io.closed? }
|
|
113
|
+
output&.join
|
|
114
|
+
errors&.join
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def read_bounded(io)
|
|
118
|
+
result = +"".b
|
|
119
|
+
while (chunk = io.read(4096))
|
|
120
|
+
result << chunk
|
|
121
|
+
raise AuthenticationError, "credential helper output exceeds size limit", cause: nil if result.bytesize > MAX_HELPER_OUTPUT
|
|
122
|
+
end
|
|
123
|
+
result
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def terminate(waiter, grouped)
|
|
127
|
+
unless grouped
|
|
128
|
+
Process.kill("KILL", waiter.pid)
|
|
129
|
+
waiter.join
|
|
130
|
+
return
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
target = -waiter.pid
|
|
134
|
+
Process.kill("TERM", target)
|
|
135
|
+
return if waiter.join(0.2)
|
|
136
|
+
|
|
137
|
+
Process.kill("KILL", target)
|
|
138
|
+
waiter.join
|
|
139
|
+
rescue SystemCallError
|
|
140
|
+
nil
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def helper_environment
|
|
144
|
+
{"GIT_TERMINAL_PROMPT" => "0", "GCM_INTERACTIVE" => "Never",
|
|
145
|
+
"GIT_TRACE" => nil, "GIT_TRACE2" => nil, "GIT_TRACE_CURL" => nil}
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def parse(output)
|
|
149
|
+
fields = {}
|
|
150
|
+
output.b.each_line(chomp: true) do |line|
|
|
151
|
+
line = line.delete_suffix("\r")
|
|
152
|
+
break if line.empty?
|
|
153
|
+
|
|
154
|
+
key, value = line.split("=", 2)
|
|
155
|
+
raise AuthenticationError, "credential helper returned invalid output", cause: nil unless value && /\A[a-zA-Z0-9_-]+(?:\[\])?\z/n.match?(key)
|
|
156
|
+
|
|
157
|
+
if key.end_with?("[]")
|
|
158
|
+
(fields[key] ||= []) << value
|
|
159
|
+
next
|
|
160
|
+
end
|
|
161
|
+
raise AuthenticationError, "credential helper returned duplicate fields", cause: nil if fields.key?(key)
|
|
162
|
+
|
|
163
|
+
fields[key] = value
|
|
164
|
+
end
|
|
165
|
+
fields
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def self.static(username:, password:)
|
|
170
|
+
validate_basic(username, password)
|
|
171
|
+
Basic.new(username.dup.freeze, password.dup.freeze)
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def self.bearer(token:)
|
|
175
|
+
raise ArgumentError, "bearer token must be a non-empty String without whitespace" unless token.is_a?(String) && /\A[^\x00-\x20\x7f]+\z/n.match?(token)
|
|
176
|
+
|
|
177
|
+
Bearer.new(token.dup.freeze)
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def self.callback(&block)
|
|
181
|
+
raise ArgumentError, "credential callback is required" unless block
|
|
182
|
+
|
|
183
|
+
Callback.new(block)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def self.helper(name = nil)
|
|
187
|
+
valid = name.nil? || (name.is_a?(String) && !name.empty? && name.bytesize <= 4096 && !name.match?(/[\r\n\0]/))
|
|
188
|
+
raise ArgumentError, "credential helper name is invalid" unless valid
|
|
189
|
+
|
|
190
|
+
Helper.new(name&.dup&.freeze)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def self.validate(value)
|
|
194
|
+
return if value.nil? || value.is_a?(Provider)
|
|
195
|
+
|
|
196
|
+
raise AuthenticationError, "credentials must be created with Thuban::Remote::Credentials"
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def self.validate_basic(username, password)
|
|
200
|
+
valid_username = username.is_a?(String) && !username.empty? && !username.match?(/[:\r\n\0]/)
|
|
201
|
+
valid_password = password.is_a?(String) && !password.match?(/[\r\n\0]/)
|
|
202
|
+
raise ArgumentError, "basic credentials are invalid" unless valid_username && valid_password
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
private_constant :Provider, :Basic, :Bearer, :Callback, :Helper
|
|
206
|
+
private_class_method :validate_basic
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
end
|