cohere-transcribe 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +9 -7
- data/ext/cohere_transcribe_native/.gitignore +6 -0
- data/ext/cohere_transcribe_native/CMakeLists.txt +8 -0
- data/ext/cohere_transcribe_native/audio_abi.cpp +20 -0
- data/ext/cohere_transcribe_native/audio_exports.macos +1 -0
- data/ext/cohere_transcribe_native/audio_exports.map +1 -0
- data/ext/cohere_transcribe_native/test/abi_smoke.rb +26 -0
- data/lib/cohere/transcribe/alignment/aligner.rb +40 -7
- data/lib/cohere/transcribe/asr/native.rb +43 -97
- data/lib/cohere/transcribe/audio/decoder.rb +184 -75
- data/lib/cohere/transcribe/audio/ffmpeg_native.rb +84 -32
- data/lib/cohere/transcribe/audio/segmentation.rb +1 -0
- data/lib/cohere/transcribe/cli.rb +30 -22
- data/lib/cohere/transcribe/configuration.rb +13 -0
- data/lib/cohere/transcribe/constants.rb +1 -1
- data/lib/cohere/transcribe/doctor.rb +2 -1
- data/lib/cohere/transcribe/hub.rb +446 -58
- data/lib/cohere/transcribe/input.rb +18 -3
- data/lib/cohere/transcribe/internal/interruptible_native_call.rb +70 -0
- data/lib/cohere/transcribe/internal/session_ownership.rb +61 -0
- data/lib/cohere/transcribe/internal/utf8.rb +19 -0
- data/lib/cohere/transcribe/model_identity.rb +10 -1
- data/lib/cohere/transcribe/output/publication.rb +95 -86
- data/lib/cohere/transcribe/pytorch_checkpoint.rb +38 -22
- data/lib/cohere/transcribe/runtime/engine.rb +37 -16
- data/lib/cohere/transcribe/runtime/preparation.rb +212 -35
- data/lib/cohere/transcribe/runtime/resources.rb +28 -62
- data/lib/cohere/transcribe/state/checkpoint.rb +3 -2
- data/lib/cohere/transcribe/state/io.rb +217 -117
- data/lib/cohere/transcribe/state/locking.rb +383 -64
- data/lib/cohere/transcribe/state/manifest.rb +21 -22
- data/lib/cohere/transcribe/types.rb +25 -11
- data/lib/cohere/transcribe/version.rb +1 -1
- data/sig/cohere/transcribe.rbs +1 -0
- metadata +5 -1
|
@@ -14,12 +14,23 @@ module Cohere
|
|
|
14
14
|
# clients are reused without copying multi-gigabyte model weights.
|
|
15
15
|
class Hub
|
|
16
16
|
class Error < StandardError; end
|
|
17
|
-
class
|
|
17
|
+
class TransientError < Error; end
|
|
18
|
+
class ConnectionError < TransientError; end
|
|
18
19
|
class AuthenticationError < Error; end
|
|
19
20
|
class NotFoundError < Error; end
|
|
20
21
|
|
|
21
22
|
COMMIT_PATTERN = /\A[0-9a-f]{40}\z/i
|
|
23
|
+
REF_READ_LIMIT = 41
|
|
22
24
|
DEFAULT_ENDPOINT = "https://huggingface.co"
|
|
25
|
+
RESOLUTION_MEMO_TTL_SECONDS = 5.0
|
|
26
|
+
RESOLUTION_MEMO_LIMIT = 64
|
|
27
|
+
CACHE_MODE_UNSUPPORTED_ERRNOS = %i[EPERM EROFS EOPNOTSUPP ENOTSUP].filter_map do |name|
|
|
28
|
+
Errno.const_get(name)::Errno if Errno.const_defined?(name)
|
|
29
|
+
end.freeze
|
|
30
|
+
ResolutionMemo = Struct.new(:value, :error, :expires_at, keyword_init: true)
|
|
31
|
+
|
|
32
|
+
private_constant :REF_READ_LIMIT, :RESOLUTION_MEMO_TTL_SECONDS, :RESOLUTION_MEMO_LIMIT,
|
|
33
|
+
:CACHE_MODE_UNSUPPORTED_ERRNOS, :ResolutionMemo
|
|
23
34
|
|
|
24
35
|
attr_reader :cache_dir, :endpoint
|
|
25
36
|
|
|
@@ -29,7 +40,13 @@ module Cohere
|
|
|
29
40
|
@endpoint = (endpoint || ENV.fetch("HF_ENDPOINT", DEFAULT_ENDPOINT)).sub(%r{/+\z}, "")
|
|
30
41
|
@endpoint_uri = URI(@endpoint)
|
|
31
42
|
@token = token || ENV["HF_TOKEN"] || cached_token(hf_home)
|
|
32
|
-
@offline = offline.nil?
|
|
43
|
+
@offline = if offline.nil?
|
|
44
|
+
truthy_environment?(ENV.fetch("HF_HUB_OFFLINE", nil))
|
|
45
|
+
else
|
|
46
|
+
offline != false
|
|
47
|
+
end
|
|
48
|
+
@resolution_guard = Mutex.new
|
|
49
|
+
@resolution_memo = {}
|
|
33
50
|
end
|
|
34
51
|
|
|
35
52
|
def offline?
|
|
@@ -42,33 +59,22 @@ module Cohere
|
|
|
42
59
|
validate_revision!(requested)
|
|
43
60
|
return requested.downcase if COMMIT_PATTERN.match?(requested)
|
|
44
61
|
|
|
45
|
-
cached = cached_revision(repo_id, requested, filename: filename)
|
|
46
62
|
if offline?
|
|
63
|
+
cached = cached_revision(repo_id, requested, filename: filename)
|
|
47
64
|
return cached if cached
|
|
48
65
|
|
|
49
66
|
raise Error,
|
|
50
67
|
"Hub offline mode has no cached #{filename} snapshot for #{repo_id.inspect} at #{requested.inspect}"
|
|
51
68
|
end
|
|
52
69
|
|
|
53
|
-
encoded_repo = repo_id.split("/").map { |part| URI.encode_www_form_component(part) }.join("/")
|
|
54
|
-
encoded_revision = URI.encode_www_form_component(requested)
|
|
55
70
|
begin
|
|
56
|
-
|
|
57
|
-
rescue
|
|
71
|
+
resolve_online_revision(repo_id, requested)
|
|
72
|
+
rescue TransientError
|
|
73
|
+
cached = cached_revision(repo_id, requested, filename: filename)
|
|
58
74
|
raise unless cached
|
|
59
75
|
|
|
60
|
-
|
|
61
|
-
end
|
|
62
|
-
payload = JSON.parse(response.body)
|
|
63
|
-
commit = payload["sha"]
|
|
64
|
-
unless commit.is_a?(String) && COMMIT_PATTERN.match?(commit)
|
|
65
|
-
raise Error, "Hub returned no immutable commit for #{repo_id.inspect} at #{requested.inspect}"
|
|
76
|
+
cached
|
|
66
77
|
end
|
|
67
|
-
|
|
68
|
-
write_ref(repo_id, requested, commit.downcase)
|
|
69
|
-
commit.downcase
|
|
70
|
-
rescue JSON::ParserError => e
|
|
71
|
-
raise Error, "Invalid Hub response while resolving #{repo_id.inspect}: #{e.message}"
|
|
72
78
|
end
|
|
73
79
|
|
|
74
80
|
def download(repo_id, filename, revision: nil)
|
|
@@ -76,7 +82,10 @@ module Cohere
|
|
|
76
82
|
commit = resolve_revision(repo_id, revision, filename: filename)
|
|
77
83
|
repository = cache_dir.join(cache_repo_name(repo_id))
|
|
78
84
|
destination = repository.join("snapshots", commit, filename)
|
|
79
|
-
|
|
85
|
+
if (cached = safe_cached_file(repository, destination))
|
|
86
|
+
normalize_cached_payload(repository, cached)
|
|
87
|
+
return cached
|
|
88
|
+
end
|
|
80
89
|
|
|
81
90
|
prepare_cache_directory!(repository, destination.dirname)
|
|
82
91
|
encoded_repo = repo_id.split("/").map { |part| URI.encode_www_form_component(part) }.join("/")
|
|
@@ -84,8 +93,17 @@ module Cohere
|
|
|
84
93
|
uri = URI("#{endpoint}/#{encoded_repo}/resolve/#{commit}/#{encoded_filename}")
|
|
85
94
|
|
|
86
95
|
open_download_lock(download_lock_path(destination)) do |lock|
|
|
87
|
-
|
|
88
|
-
|
|
96
|
+
acquire_download_lock!(lock, download_lock_path(destination), destination)
|
|
97
|
+
|
|
98
|
+
verify_cache_lock_identity!(
|
|
99
|
+
download_lock_path(destination),
|
|
100
|
+
lock,
|
|
101
|
+
purpose: "Hub download"
|
|
102
|
+
)
|
|
103
|
+
if (cached = safe_cached_file(repository, destination))
|
|
104
|
+
normalize_cached_payload(repository, cached)
|
|
105
|
+
return cached
|
|
106
|
+
end
|
|
89
107
|
|
|
90
108
|
cleanup_download_temporaries(destination)
|
|
91
109
|
Tempfile.create(
|
|
@@ -93,6 +111,7 @@ module Cohere
|
|
|
93
111
|
destination.dirname.to_s,
|
|
94
112
|
binmode: true
|
|
95
113
|
) do |temporary|
|
|
114
|
+
apply_cache_mode_if_supported(temporary, cache_payload_mode(destination.dirname))
|
|
96
115
|
request(uri, stream: temporary)
|
|
97
116
|
temporary.flush
|
|
98
117
|
temporary.fsync
|
|
@@ -102,7 +121,7 @@ module Cohere
|
|
|
102
121
|
sync_directory(destination.dirname)
|
|
103
122
|
end
|
|
104
123
|
destination
|
|
105
|
-
rescue
|
|
124
|
+
rescue SystemCallError => e
|
|
106
125
|
raise Error, "Cannot cache #{repo_id}/#{filename}: #{e.message}"
|
|
107
126
|
end
|
|
108
127
|
|
|
@@ -110,16 +129,22 @@ module Cohere
|
|
|
110
129
|
commit = resolve_revision(repo_id, revision)
|
|
111
130
|
encoded_repo = repo_id.split("/").map { |part| URI.encode_www_form_component(part) }.join("/")
|
|
112
131
|
response = request(URI("#{endpoint}/api/models/#{encoded_repo}/revision/#{commit}"))
|
|
113
|
-
payload = JSON.parse(response.body)
|
|
114
|
-
siblings = payload["siblings"]
|
|
132
|
+
payload = JSON.parse(response.body.to_s)
|
|
133
|
+
siblings = payload.is_a?(Hash) ? payload["siblings"] : nil
|
|
115
134
|
raise Error, "Hub returned no repository file list for #{repo_id}@#{commit}" unless siblings.is_a?(Array)
|
|
116
135
|
|
|
117
|
-
siblings.filter_map do |item|
|
|
136
|
+
files = siblings.filter_map do |item|
|
|
118
137
|
name = item.is_a?(Hash) ? item["rfilename"] : nil
|
|
119
|
-
name if name.is_a?(String)
|
|
120
|
-
end
|
|
138
|
+
name if name.is_a?(String) && !name.empty?
|
|
139
|
+
end
|
|
140
|
+
if files.empty?
|
|
141
|
+
detail = siblings.empty? ? "an empty repository file list" : "no valid repository file entries"
|
|
142
|
+
raise Error, "Hub returned #{detail} for #{repo_id}@#{commit}"
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
files.freeze
|
|
121
146
|
rescue JSON::ParserError => e
|
|
122
|
-
raise
|
|
147
|
+
raise TransientError, "Invalid Hub response while listing #{repo_id.inspect}: #{e.message}"
|
|
123
148
|
end
|
|
124
149
|
|
|
125
150
|
def snapshot_path(repo_id, commit)
|
|
@@ -147,6 +172,77 @@ module Cohere
|
|
|
147
172
|
|
|
148
173
|
private
|
|
149
174
|
|
|
175
|
+
def resolve_online_revision(repo_id, revision)
|
|
176
|
+
key = resolution_key(repo_id, revision)
|
|
177
|
+
memo = @resolution_guard.synchronize do
|
|
178
|
+
prune_resolution_memo
|
|
179
|
+
@resolution_memo[key]
|
|
180
|
+
end
|
|
181
|
+
return resolution_value(memo.value, memo.error) if memo
|
|
182
|
+
|
|
183
|
+
# Concurrent misses deliberately perform independent requests. Each
|
|
184
|
+
# caller uses the response it fetched; the short memo only avoids
|
|
185
|
+
# repeated requests that begin after one lookup has completed.
|
|
186
|
+
value = fetch_online_revision(repo_id, revision)
|
|
187
|
+
write_ref_best_effort(repo_id, revision, value)
|
|
188
|
+
@resolution_guard.synchronize { memoize_resolution(key, value, nil) }
|
|
189
|
+
value
|
|
190
|
+
rescue TransientError => e
|
|
191
|
+
memo = @resolution_guard.synchronize do
|
|
192
|
+
prune_resolution_memo
|
|
193
|
+
current = @resolution_memo[key]
|
|
194
|
+
memoize_resolution(key, nil, e) unless current
|
|
195
|
+
current
|
|
196
|
+
end
|
|
197
|
+
return resolution_value(memo.value, memo.error) if memo
|
|
198
|
+
|
|
199
|
+
raise
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def fetch_online_revision(repo_id, revision)
|
|
203
|
+
encoded_repo = repo_id.split("/").map { |part| URI.encode_www_form_component(part) }.join("/")
|
|
204
|
+
encoded_revision = URI.encode_www_form_component(revision)
|
|
205
|
+
response = request(URI("#{endpoint}/api/models/#{encoded_repo}/revision/#{encoded_revision}"))
|
|
206
|
+
payload = JSON.parse(response.body.to_s)
|
|
207
|
+
commit = payload.is_a?(Hash) ? payload["sha"] : nil
|
|
208
|
+
unless commit.is_a?(String) && COMMIT_PATTERN.match?(commit)
|
|
209
|
+
raise TransientError, "Hub returned no immutable commit for #{repo_id.inspect} at #{revision.inspect}"
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
commit.downcase
|
|
213
|
+
rescue JSON::ParserError => e
|
|
214
|
+
raise TransientError, "Invalid Hub response while resolving #{repo_id.inspect}: #{e.message}"
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def memoize_resolution(key, value, error)
|
|
218
|
+
@resolution_memo.delete(key)
|
|
219
|
+
@resolution_memo[key] = ResolutionMemo.new(
|
|
220
|
+
value: value,
|
|
221
|
+
error: error,
|
|
222
|
+
expires_at: monotonic + RESOLUTION_MEMO_TTL_SECONDS
|
|
223
|
+
)
|
|
224
|
+
@resolution_memo.shift while @resolution_memo.length > RESOLUTION_MEMO_LIMIT
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def resolution_value(value, error)
|
|
228
|
+
raise error if error
|
|
229
|
+
|
|
230
|
+
value
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def prune_resolution_memo
|
|
234
|
+
now = monotonic
|
|
235
|
+
@resolution_memo.delete_if { |_key, entry| entry.expires_at <= now }
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def resolution_key(repo_id, revision)
|
|
239
|
+
[repo_id.dup.freeze, revision.dup.freeze].freeze
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def monotonic
|
|
243
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
244
|
+
end
|
|
245
|
+
|
|
150
246
|
def request(uri, stream: nil, redirects: 0)
|
|
151
247
|
raise Error, "Hub offline mode prevents a request to #{uri}" if offline?
|
|
152
248
|
raise Error, "Too many redirects while fetching #{uri}" if redirects > 8
|
|
@@ -180,7 +276,7 @@ module Cohere
|
|
|
180
276
|
end
|
|
181
277
|
expected = response_content_length(incoming, uri)
|
|
182
278
|
if expected && written != expected
|
|
183
|
-
raise
|
|
279
|
+
raise TransientError,
|
|
184
280
|
"Hub returned an incomplete response body for #{uri}: " \
|
|
185
281
|
"received #{written} bytes, expected #{expected}"
|
|
186
282
|
end
|
|
@@ -217,7 +313,16 @@ module Cohere
|
|
|
217
313
|
location = response["location"]
|
|
218
314
|
raise Error, "Hub redirect from #{base} did not include a location" unless location
|
|
219
315
|
|
|
220
|
-
URI.join(base.to_s, location)
|
|
316
|
+
resolved = URI.join(base.to_s, location)
|
|
317
|
+
unless resolved.is_a?(URI::HTTP) && resolved.host
|
|
318
|
+
raise Error, "Hub redirect from #{base} returned an invalid location: #{location.inspect}"
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
resolved
|
|
322
|
+
rescue URI::Error => e
|
|
323
|
+
raise Error,
|
|
324
|
+
"Hub redirect from #{base} returned an invalid location: #{location.inspect}",
|
|
325
|
+
cause: e
|
|
221
326
|
end
|
|
222
327
|
|
|
223
328
|
def trusted_authorization_origin?(uri)
|
|
@@ -228,7 +333,7 @@ module Cohere
|
|
|
228
333
|
def raise_http_error!(response, uri)
|
|
229
334
|
detail = begin
|
|
230
335
|
parsed = JSON.parse(response.body.to_s)
|
|
231
|
-
parsed["error"] || parsed["message"]
|
|
336
|
+
parsed.is_a?(Hash) ? parsed["error"] || parsed["message"] : nil
|
|
232
337
|
rescue JSON::ParserError
|
|
233
338
|
nil
|
|
234
339
|
end
|
|
@@ -237,6 +342,7 @@ module Cohere
|
|
|
237
342
|
case response.code.to_i
|
|
238
343
|
when 401, 403 then raise AuthenticationError, message
|
|
239
344
|
when 404 then raise NotFoundError, message
|
|
345
|
+
when 429, 500..599 then raise TransientError, message
|
|
240
346
|
else raise Error, message
|
|
241
347
|
end
|
|
242
348
|
end
|
|
@@ -246,7 +352,7 @@ module Cohere
|
|
|
246
352
|
candidates = []
|
|
247
353
|
ref = revision_path(repository, revision)
|
|
248
354
|
cached_ref = safe_cached_file(repository, ref)
|
|
249
|
-
candidates << cached_ref
|
|
355
|
+
candidates << read_ref_commit(cached_ref) if cached_ref
|
|
250
356
|
candidates << revision if COMMIT_PATTERN.match?(revision)
|
|
251
357
|
candidates.find do |candidate|
|
|
252
358
|
COMMIT_PATTERN.match?(candidate) &&
|
|
@@ -256,23 +362,50 @@ module Cohere
|
|
|
256
362
|
nil
|
|
257
363
|
end
|
|
258
364
|
|
|
365
|
+
def read_ref_commit(path)
|
|
366
|
+
File.open(path, "rb") do |ref|
|
|
367
|
+
next if ref.stat.size > REF_READ_LIMIT
|
|
368
|
+
|
|
369
|
+
value = ref.read(REF_READ_LIMIT)&.strip
|
|
370
|
+
next if ref.stat.size > REF_READ_LIMIT
|
|
371
|
+
|
|
372
|
+
value if value && COMMIT_PATTERN.match?(value)
|
|
373
|
+
end
|
|
374
|
+
rescue SystemCallError
|
|
375
|
+
nil
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
def write_ref_best_effort(repo_id, revision, commit)
|
|
379
|
+
write_ref(repo_id, revision, commit)
|
|
380
|
+
rescue Error, SystemCallError
|
|
381
|
+
# Hub refs are advisory cache hints. A valid online response remains
|
|
382
|
+
# usable when a shared or read-only cache cannot publish the hint.
|
|
383
|
+
commit
|
|
384
|
+
end
|
|
385
|
+
|
|
259
386
|
def write_ref(repo_id, revision, commit)
|
|
260
387
|
repository = cache_dir.join(cache_repo_name(repo_id))
|
|
261
388
|
path = revision_path(repository, revision)
|
|
262
|
-
prepare_cache_directory!(repository, path.dirname)
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
389
|
+
prepare_cache_directory!(repository, path.dirname, replaceable_entries: true)
|
|
390
|
+
current = safe_cached_file(repository, path)
|
|
391
|
+
if current && read_ref_commit(current) == commit
|
|
392
|
+
normalize_cached_file_mode(current, cache_payload_mode(current.dirname))
|
|
393
|
+
return commit
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
# Standard Hub refs contain only the immutable commit. Concurrent
|
|
397
|
+
# writers may replace one complete hint with another, while Tempfile
|
|
398
|
+
# and rename ensure readers never observe a partial value.
|
|
268
399
|
Tempfile.create([".cohere-transcribe-ref-", ".tmp"], path.dirname.to_s) do |temporary|
|
|
269
|
-
temporary.write(
|
|
400
|
+
temporary.write(commit)
|
|
401
|
+
apply_cache_mode_if_supported(temporary, cache_payload_mode(path.dirname))
|
|
270
402
|
temporary.flush
|
|
271
403
|
temporary.fsync
|
|
272
404
|
temporary.close
|
|
273
405
|
File.rename(temporary.path, path)
|
|
274
406
|
end
|
|
275
407
|
sync_directory(path.dirname)
|
|
408
|
+
commit
|
|
276
409
|
end
|
|
277
410
|
|
|
278
411
|
def safe_cached_file(repository, path)
|
|
@@ -308,42 +441,170 @@ module Cohere
|
|
|
308
441
|
pattern = destination.dirname.join("#{download_temporary_prefix(destination)}*.download")
|
|
309
442
|
Dir.glob(pattern.to_s).each do |path|
|
|
310
443
|
File.unlink(path)
|
|
311
|
-
rescue Errno::ENOENT
|
|
444
|
+
rescue Errno::ENOENT, Errno::EACCES, Errno::EPERM, Errno::EROFS
|
|
312
445
|
nil
|
|
313
446
|
end
|
|
314
447
|
end
|
|
315
448
|
|
|
316
|
-
def open_download_lock(path)
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
end
|
|
449
|
+
def open_download_lock(path, &)
|
|
450
|
+
open_cache_lock(path, purpose: "Hub download", &)
|
|
451
|
+
end
|
|
452
|
+
|
|
453
|
+
def open_cache_lock(path, purpose:)
|
|
454
|
+
path = Pathname(path)
|
|
455
|
+
desired_mode = cache_lock_mode(path.dirname)
|
|
456
|
+
lock, access, created = open_cache_lock_handle(path, purpose: purpose, mode: desired_mode)
|
|
457
|
+
opened = verify_cache_lock_identity!(path, lock, purpose: purpose)
|
|
458
|
+
owned_writable = access == "r+" && cache_file_owned_by_current_process?(opened)
|
|
459
|
+
apply_cache_mode_if_supported(lock, desired_mode) if (created || owned_writable) && (opened.mode & 0o777) != desired_mode
|
|
328
460
|
|
|
329
461
|
yield lock
|
|
330
462
|
rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO => e
|
|
331
|
-
raise Error, "
|
|
463
|
+
raise Error, "#{purpose} lock is not a regular file: #{path}", cause: e
|
|
332
464
|
ensure
|
|
333
465
|
lock&.close
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
def open_cache_lock_handle(path, purpose:, mode:)
|
|
469
|
+
descriptor = nil
|
|
470
|
+
handle = nil
|
|
471
|
+
replacement = nil
|
|
472
|
+
retired = nil
|
|
473
|
+
succeeded = false
|
|
474
|
+
begin
|
|
475
|
+
descriptor, created = open_or_create_cache_lock_descriptor(path, cache_lock_flags, mode)
|
|
476
|
+
access = "r+"
|
|
477
|
+
rescue Errno::EACCES, Errno::EROFS => e
|
|
478
|
+
descriptor = open_readonly_cache_lock_descriptor(path, purpose: purpose, write_error: e)
|
|
479
|
+
created = false
|
|
480
|
+
access = "r"
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
handle = File.new(descriptor, access, autoclose: true)
|
|
484
|
+
descriptor = nil
|
|
485
|
+
opened = verify_cache_lock_identity!(path, handle, purpose: purpose)
|
|
486
|
+
if access == "r" && cache_file_owned_by_current_process?(opened)
|
|
487
|
+
apply_cache_mode_if_supported(handle, mode)
|
|
488
|
+
replacement = reopen_cache_lock_writable(path, purpose: purpose)
|
|
489
|
+
if replacement
|
|
490
|
+
retired = handle
|
|
491
|
+
handle = replacement
|
|
492
|
+
replacement = nil
|
|
493
|
+
access = "r+"
|
|
494
|
+
retired.close
|
|
495
|
+
retired = nil
|
|
496
|
+
end
|
|
497
|
+
end
|
|
498
|
+
succeeded = true
|
|
499
|
+
[handle, access, created]
|
|
500
|
+
rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO => e
|
|
501
|
+
raise Error, "#{purpose} lock is not a regular file: #{path}", cause: e
|
|
502
|
+
ensure
|
|
503
|
+
retired.close if retired && !retired.closed?
|
|
504
|
+
replacement.close if replacement && !replacement.closed?
|
|
505
|
+
handle.close if handle && !succeeded && !handle.closed?
|
|
506
|
+
::IO.new(descriptor).close if descriptor
|
|
507
|
+
end
|
|
508
|
+
|
|
509
|
+
def open_or_create_cache_lock_descriptor(path, flags, mode)
|
|
510
|
+
2.times do
|
|
511
|
+
return [::IO.sysopen(path.to_s, flags, mode), false]
|
|
512
|
+
rescue Errno::ENOENT
|
|
513
|
+
begin
|
|
514
|
+
return [::IO.sysopen(path.to_s, flags | File::CREAT | File::EXCL, mode), true]
|
|
515
|
+
rescue Errno::EEXIST
|
|
516
|
+
next
|
|
517
|
+
end
|
|
518
|
+
end
|
|
519
|
+
|
|
520
|
+
raise Errno::ENOENT, path.to_s
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
def open_readonly_cache_lock_descriptor(path, purpose:, write_error:)
|
|
524
|
+
::IO.sysopen(path.to_s, readonly_cache_lock_flags)
|
|
525
|
+
rescue Errno::EACCES => e
|
|
526
|
+
raise Error,
|
|
527
|
+
"#{purpose} lock is not readable at #{path}; ask the cache owner to grant shared read/write access, " \
|
|
528
|
+
"or remove the stale lock only after all downloads have stopped",
|
|
529
|
+
cause: e
|
|
530
|
+
rescue Errno::ENOENT => e
|
|
531
|
+
raise Error, "Cannot create #{purpose.downcase} lock at #{path}: #{write_error.message}", cause: e
|
|
532
|
+
end
|
|
533
|
+
|
|
534
|
+
def reopen_cache_lock_writable(path, purpose:)
|
|
535
|
+
handle = nil
|
|
536
|
+
succeeded = false
|
|
537
|
+
descriptor = ::IO.sysopen(path.to_s, cache_lock_flags)
|
|
538
|
+
handle = File.new(descriptor, "r+", autoclose: true)
|
|
539
|
+
descriptor = nil
|
|
540
|
+
verify_cache_lock_identity!(path, handle, purpose: purpose)
|
|
541
|
+
succeeded = true
|
|
542
|
+
handle
|
|
543
|
+
rescue Errno::EACCES, Errno::EROFS
|
|
544
|
+
nil
|
|
545
|
+
ensure
|
|
546
|
+
handle&.close unless succeeded
|
|
334
547
|
::IO.new(descriptor).close if descriptor
|
|
335
548
|
end
|
|
336
549
|
|
|
550
|
+
def cache_lock_flags
|
|
551
|
+
flags = File::RDWR
|
|
552
|
+
flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
553
|
+
flags |= File::CLOEXEC if defined?(File::CLOEXEC)
|
|
554
|
+
flags
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
def readonly_cache_lock_flags
|
|
558
|
+
flags = File::RDONLY
|
|
559
|
+
flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
560
|
+
flags |= File::CLOEXEC if defined?(File::CLOEXEC)
|
|
561
|
+
flags
|
|
562
|
+
end
|
|
563
|
+
|
|
564
|
+
def acquire_download_lock!(lock, path, destination)
|
|
565
|
+
return if lock.flock(File::LOCK_EX)
|
|
566
|
+
|
|
567
|
+
raise Error, "Cannot acquire Hub download lock for #{destination} at #{path}"
|
|
568
|
+
rescue SystemCallError => e
|
|
569
|
+
if e.is_a?(Errno::EBADF)
|
|
570
|
+
detail = "the filesystem requires a writable shared lock file"
|
|
571
|
+
elsif cache_locking_unsupported?(e)
|
|
572
|
+
detail = "the filesystem does not provide the required file locking"
|
|
573
|
+
else
|
|
574
|
+
raise
|
|
575
|
+
end
|
|
576
|
+
raise Error, "Cannot acquire Hub download lock for #{destination}: #{detail} at #{path}", cause: e
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
def cache_locking_unsupported?(error)
|
|
580
|
+
names = %i[ENOLCK EOPNOTSUPP ENOTSUP]
|
|
581
|
+
names.any? { |name| Errno.const_defined?(name) && error.errno == Errno.const_get(name)::Errno }
|
|
582
|
+
end
|
|
583
|
+
|
|
584
|
+
def cache_file_owned_by_current_process?(stat)
|
|
585
|
+
Process.respond_to?(:euid) && stat.uid == Process.euid
|
|
586
|
+
end
|
|
587
|
+
|
|
588
|
+
def verify_cache_lock_identity!(path, lock, purpose:)
|
|
589
|
+
opened = lock.stat
|
|
590
|
+
current = path.lstat
|
|
591
|
+
return opened if opened.file? && !current.symlink? && opened.dev == current.dev && opened.ino == current.ino
|
|
592
|
+
|
|
593
|
+
raise Error, "#{purpose} lock changed while held or is not a regular file: #{path}"
|
|
594
|
+
rescue SystemCallError => e
|
|
595
|
+
raise Error, "#{purpose} lock changed while held or is not a regular file: #{path}", cause: e
|
|
596
|
+
end
|
|
597
|
+
|
|
337
598
|
# Build each repository subdirectory only after proving that its parent is
|
|
338
599
|
# still inside this repository. This prevents a poisoned snapshots/refs
|
|
339
600
|
# symlink from redirecting a Tempfile and rename outside the cache.
|
|
340
|
-
def prepare_cache_directory!(repository, directory)
|
|
601
|
+
def prepare_cache_directory!(repository, directory, replaceable_entries: false)
|
|
341
602
|
FileUtils.mkdir_p(cache_dir)
|
|
342
603
|
cache_root = cache_dir.realpath
|
|
343
604
|
begin
|
|
344
|
-
|
|
605
|
+
create_cache_directory(repository, cache_dir)
|
|
345
606
|
rescue Errno::EEXIST
|
|
346
|
-
|
|
607
|
+
normalize_existing_cache_directory(repository, replaceable_entries: false)
|
|
347
608
|
end
|
|
348
609
|
|
|
349
610
|
repository_stat = repository.lstat
|
|
@@ -364,9 +625,9 @@ module Cohere
|
|
|
364
625
|
|
|
365
626
|
current = current.join(part)
|
|
366
627
|
begin
|
|
367
|
-
|
|
628
|
+
create_cache_directory(current, current.dirname, replaceable_entries: replaceable_entries)
|
|
368
629
|
rescue Errno::EEXIST
|
|
369
|
-
|
|
630
|
+
normalize_existing_cache_directory(current, replaceable_entries: replaceable_entries)
|
|
370
631
|
end
|
|
371
632
|
resolved = current.realpath
|
|
372
633
|
unless resolved.directory? && within_directory?(resolved, repository_root)
|
|
@@ -380,6 +641,133 @@ module Cohere
|
|
|
380
641
|
raise Error, "Cannot prepare Hub cache directory #{directory}: #{e.message}"
|
|
381
642
|
end
|
|
382
643
|
|
|
644
|
+
def create_cache_directory(path, parent, replaceable_entries: false)
|
|
645
|
+
# New cache directories inherit the verified parent's ordinary access
|
|
646
|
+
# bits and setgid policy even when this process has a private umask.
|
|
647
|
+
# Owner access keeps a newly-created private cache usable.
|
|
648
|
+
mode = cache_directory_mode(parent, replaceable_entries: replaceable_entries)
|
|
649
|
+
Dir.mkdir(path, mode)
|
|
650
|
+
flags = File::RDONLY
|
|
651
|
+
flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
652
|
+
flags |= File::CLOEXEC if defined?(File::CLOEXEC)
|
|
653
|
+
File.open(path, flags) do |handle|
|
|
654
|
+
opened = handle.stat
|
|
655
|
+
current = path.lstat
|
|
656
|
+
unless opened.directory? && !current.symlink? && opened.dev == current.dev && opened.ino == current.ino
|
|
657
|
+
raise Error, "Hub cache directory changed while it was being created: #{path}"
|
|
658
|
+
end
|
|
659
|
+
|
|
660
|
+
apply_cache_mode_if_supported(handle, mode)
|
|
661
|
+
end
|
|
662
|
+
end
|
|
663
|
+
|
|
664
|
+
def normalize_existing_cache_directory(path, replaceable_entries:)
|
|
665
|
+
return if replaceable_entries
|
|
666
|
+
|
|
667
|
+
flags = File::RDONLY
|
|
668
|
+
flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
669
|
+
flags |= File::CLOEXEC if defined?(File::CLOEXEC)
|
|
670
|
+
File.open(path, flags) do |handle|
|
|
671
|
+
opened = handle.stat
|
|
672
|
+
current = path.lstat
|
|
673
|
+
unless opened.directory? && !current.symlink? && opened.dev == current.dev && opened.ino == current.ino
|
|
674
|
+
raise Error, "Hub cache directory changed while its mode was being updated: #{path}"
|
|
675
|
+
end
|
|
676
|
+
next unless cache_file_owned_by_current_process?(opened)
|
|
677
|
+
|
|
678
|
+
current_mode = opened.mode & 0o3777
|
|
679
|
+
desired_mode = current_mode | (current_mode.anybits?(0o022) ? 0o1000 : 0)
|
|
680
|
+
apply_cache_mode_if_supported(handle, desired_mode) if current_mode != desired_mode
|
|
681
|
+
end
|
|
682
|
+
rescue Errno::EACCES, Errno::EROFS, Errno::ENOENT, Errno::ELOOP
|
|
683
|
+
nil
|
|
684
|
+
end
|
|
685
|
+
|
|
686
|
+
def normalize_cached_file_mode(path, desired_mode)
|
|
687
|
+
return if path.lstat.symlink?
|
|
688
|
+
|
|
689
|
+
flags = File::RDONLY
|
|
690
|
+
flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
691
|
+
flags |= File::CLOEXEC if defined?(File::CLOEXEC)
|
|
692
|
+
File.open(path, flags) do |handle|
|
|
693
|
+
opened = handle.stat
|
|
694
|
+
current = path.lstat
|
|
695
|
+
unless opened.file? && !current.symlink? && opened.dev == current.dev && opened.ino == current.ino
|
|
696
|
+
raise Error, "Hub cache file changed while its mode was being updated: #{path}"
|
|
697
|
+
end
|
|
698
|
+
next unless cache_file_owned_by_current_process?(opened)
|
|
699
|
+
next if (opened.mode & 0o777) == desired_mode
|
|
700
|
+
|
|
701
|
+
apply_cache_mode_if_supported(handle, desired_mode)
|
|
702
|
+
end
|
|
703
|
+
rescue Errno::EACCES, Errno::EROFS, Errno::ENOENT, Errno::ELOOP
|
|
704
|
+
nil
|
|
705
|
+
end
|
|
706
|
+
|
|
707
|
+
def normalize_cached_payload(repository, path)
|
|
708
|
+
repository_root = repository.realpath
|
|
709
|
+
relative = path.dirname.relative_path_from(repository)
|
|
710
|
+
return if relative.absolute? || relative.each_filename.any? { |part| part == ".." }
|
|
711
|
+
|
|
712
|
+
current = repository
|
|
713
|
+
normalize_existing_cache_directory(current, replaceable_entries: false)
|
|
714
|
+
components = relative.each_filename.reject { |part| part == "." }
|
|
715
|
+
until components.empty?
|
|
716
|
+
part = components.shift
|
|
717
|
+
|
|
718
|
+
current = current.join(part)
|
|
719
|
+
resolved = current.realpath
|
|
720
|
+
return unless resolved.directory? && within_directory?(resolved, repository_root)
|
|
721
|
+
|
|
722
|
+
normalize_existing_cache_directory(current, replaceable_entries: false)
|
|
723
|
+
end
|
|
724
|
+
normalize_cached_file_mode(path, cache_payload_mode(path.dirname))
|
|
725
|
+
rescue Error, SystemCallError, ArgumentError
|
|
726
|
+
# Reusing a verified immutable payload does not depend on being able to
|
|
727
|
+
# upgrade access modes in an older or read-only cache.
|
|
728
|
+
nil
|
|
729
|
+
end
|
|
730
|
+
|
|
731
|
+
def cache_directory_mode(parent, replaceable_entries: false)
|
|
732
|
+
parent_mode = parent.stat.mode
|
|
733
|
+
mode = 0o700 | (parent_mode & 0o2000)
|
|
734
|
+
mode |= parent_mode & 0o070 if parent_mode.anybits?(0o010)
|
|
735
|
+
mode |= parent_mode & 0o007 if parent_mode.anybits?(0o001)
|
|
736
|
+
# Snapshot directories keep other users from replacing published
|
|
737
|
+
# payloads. Ref directories omit this bit so collaborators can replace
|
|
738
|
+
# one another's advisory branch hints through atomic rename.
|
|
739
|
+
mode |= 0o1000 if !replaceable_entries && mode.anybits?(0o022)
|
|
740
|
+
mode
|
|
741
|
+
end
|
|
742
|
+
|
|
743
|
+
def cache_payload_mode(parent)
|
|
744
|
+
# Published payloads and refs inherit read access, never collaborator
|
|
745
|
+
# write access. Ref replacement is provided by the parent directory.
|
|
746
|
+
parent_mode = parent.stat.mode
|
|
747
|
+
mode = 0o600
|
|
748
|
+
mode |= 0o040 if parent_mode.allbits?(0o050)
|
|
749
|
+
mode |= 0o004 if parent_mode.allbits?(0o005)
|
|
750
|
+
mode
|
|
751
|
+
end
|
|
752
|
+
|
|
753
|
+
def cache_lock_mode(parent)
|
|
754
|
+
# Lock files must be writable by every collaborator who can create
|
|
755
|
+
# cache entries in the containing directory.
|
|
756
|
+
parent_mode = parent.stat.mode
|
|
757
|
+
mode = 0o600
|
|
758
|
+
mode |= 0o060 if parent_mode.allbits?(0o030)
|
|
759
|
+
mode |= 0o006 if parent_mode.allbits?(0o003)
|
|
760
|
+
mode
|
|
761
|
+
end
|
|
762
|
+
|
|
763
|
+
def apply_cache_mode_if_supported(target, mode)
|
|
764
|
+
target.chmod(mode)
|
|
765
|
+
rescue SystemCallError => e
|
|
766
|
+
raise unless CACHE_MODE_UNSUPPORTED_ERRNOS.include?(e.errno)
|
|
767
|
+
|
|
768
|
+
nil
|
|
769
|
+
end
|
|
770
|
+
|
|
383
771
|
def within_directory?(path, directory)
|
|
384
772
|
path_text = path.to_s
|
|
385
773
|
directory_text = directory.to_s
|