cohere-transcribe 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +9 -0
  3. data/README.md +9 -7
  4. data/ext/cohere_transcribe_native/.gitignore +6 -0
  5. data/ext/cohere_transcribe_native/CMakeLists.txt +8 -0
  6. data/ext/cohere_transcribe_native/audio_abi.cpp +20 -0
  7. data/ext/cohere_transcribe_native/audio_exports.macos +1 -0
  8. data/ext/cohere_transcribe_native/audio_exports.map +1 -0
  9. data/ext/cohere_transcribe_native/test/abi_smoke.rb +26 -0
  10. data/lib/cohere/transcribe/alignment/aligner.rb +40 -7
  11. data/lib/cohere/transcribe/asr/native.rb +43 -97
  12. data/lib/cohere/transcribe/audio/decoder.rb +184 -75
  13. data/lib/cohere/transcribe/audio/ffmpeg_native.rb +84 -32
  14. data/lib/cohere/transcribe/audio/segmentation.rb +1 -0
  15. data/lib/cohere/transcribe/cli.rb +30 -22
  16. data/lib/cohere/transcribe/configuration.rb +13 -0
  17. data/lib/cohere/transcribe/constants.rb +1 -1
  18. data/lib/cohere/transcribe/doctor.rb +2 -1
  19. data/lib/cohere/transcribe/hub.rb +446 -58
  20. data/lib/cohere/transcribe/input.rb +18 -3
  21. data/lib/cohere/transcribe/internal/interruptible_native_call.rb +70 -0
  22. data/lib/cohere/transcribe/internal/session_ownership.rb +61 -0
  23. data/lib/cohere/transcribe/internal/utf8.rb +19 -0
  24. data/lib/cohere/transcribe/model_identity.rb +10 -1
  25. data/lib/cohere/transcribe/output/publication.rb +95 -86
  26. data/lib/cohere/transcribe/pytorch_checkpoint.rb +38 -22
  27. data/lib/cohere/transcribe/runtime/engine.rb +37 -16
  28. data/lib/cohere/transcribe/runtime/preparation.rb +212 -35
  29. data/lib/cohere/transcribe/runtime/resources.rb +28 -62
  30. data/lib/cohere/transcribe/state/checkpoint.rb +3 -2
  31. data/lib/cohere/transcribe/state/io.rb +217 -117
  32. data/lib/cohere/transcribe/state/locking.rb +383 -64
  33. data/lib/cohere/transcribe/state/manifest.rb +21 -22
  34. data/lib/cohere/transcribe/types.rb +25 -11
  35. data/lib/cohere/transcribe/version.rb +1 -1
  36. data/sig/cohere/transcribe.rbs +1 -0
  37. metadata +5 -1
@@ -14,12 +14,23 @@ module Cohere
14
14
  # clients are reused without copying multi-gigabyte model weights.
15
15
  class Hub
16
16
  class Error < StandardError; end
17
- class ConnectionError < Error; end
17
+ class TransientError < Error; end
18
+ class ConnectionError < TransientError; end
18
19
  class AuthenticationError < Error; end
19
20
  class NotFoundError < Error; end
20
21
 
21
22
  COMMIT_PATTERN = /\A[0-9a-f]{40}\z/i
23
+ REF_READ_LIMIT = 41
22
24
  DEFAULT_ENDPOINT = "https://huggingface.co"
25
+ RESOLUTION_MEMO_TTL_SECONDS = 5.0
26
+ RESOLUTION_MEMO_LIMIT = 64
27
+ CACHE_MODE_UNSUPPORTED_ERRNOS = %i[EPERM EROFS EOPNOTSUPP ENOTSUP].filter_map do |name|
28
+ Errno.const_get(name)::Errno if Errno.const_defined?(name)
29
+ end.freeze
30
+ ResolutionMemo = Struct.new(:value, :error, :expires_at, keyword_init: true)
31
+
32
+ private_constant :REF_READ_LIMIT, :RESOLUTION_MEMO_TTL_SECONDS, :RESOLUTION_MEMO_LIMIT,
33
+ :CACHE_MODE_UNSUPPORTED_ERRNOS, :ResolutionMemo
23
34
 
24
35
  attr_reader :cache_dir, :endpoint
25
36
 
@@ -29,7 +40,13 @@ module Cohere
29
40
  @endpoint = (endpoint || ENV.fetch("HF_ENDPOINT", DEFAULT_ENDPOINT)).sub(%r{/+\z}, "")
30
41
  @endpoint_uri = URI(@endpoint)
31
42
  @token = token || ENV["HF_TOKEN"] || cached_token(hf_home)
32
- @offline = offline.nil? ? truthy_environment?(ENV.fetch("HF_HUB_OFFLINE", nil)) : !!offline
43
+ @offline = if offline.nil?
44
+ truthy_environment?(ENV.fetch("HF_HUB_OFFLINE", nil))
45
+ else
46
+ offline != false
47
+ end
48
+ @resolution_guard = Mutex.new
49
+ @resolution_memo = {}
33
50
  end
34
51
 
35
52
  def offline?
@@ -42,33 +59,22 @@ module Cohere
42
59
  validate_revision!(requested)
43
60
  return requested.downcase if COMMIT_PATTERN.match?(requested)
44
61
 
45
- cached = cached_revision(repo_id, requested, filename: filename)
46
62
  if offline?
63
+ cached = cached_revision(repo_id, requested, filename: filename)
47
64
  return cached if cached
48
65
 
49
66
  raise Error,
50
67
  "Hub offline mode has no cached #{filename} snapshot for #{repo_id.inspect} at #{requested.inspect}"
51
68
  end
52
69
 
53
- encoded_repo = repo_id.split("/").map { |part| URI.encode_www_form_component(part) }.join("/")
54
- encoded_revision = URI.encode_www_form_component(requested)
55
70
  begin
56
- response = request(URI("#{endpoint}/api/models/#{encoded_repo}/revision/#{encoded_revision}"))
57
- rescue ConnectionError
71
+ resolve_online_revision(repo_id, requested)
72
+ rescue TransientError
73
+ cached = cached_revision(repo_id, requested, filename: filename)
58
74
  raise unless cached
59
75
 
60
- return cached
61
- end
62
- payload = JSON.parse(response.body)
63
- commit = payload["sha"]
64
- unless commit.is_a?(String) && COMMIT_PATTERN.match?(commit)
65
- raise Error, "Hub returned no immutable commit for #{repo_id.inspect} at #{requested.inspect}"
76
+ cached
66
77
  end
67
-
68
- write_ref(repo_id, requested, commit.downcase)
69
- commit.downcase
70
- rescue JSON::ParserError => e
71
- raise Error, "Invalid Hub response while resolving #{repo_id.inspect}: #{e.message}"
72
78
  end
73
79
 
74
80
  def download(repo_id, filename, revision: nil)
@@ -76,7 +82,10 @@ module Cohere
76
82
  commit = resolve_revision(repo_id, revision, filename: filename)
77
83
  repository = cache_dir.join(cache_repo_name(repo_id))
78
84
  destination = repository.join("snapshots", commit, filename)
79
- return destination if safe_cached_file(repository, destination)
85
+ if (cached = safe_cached_file(repository, destination))
86
+ normalize_cached_payload(repository, cached)
87
+ return cached
88
+ end
80
89
 
81
90
  prepare_cache_directory!(repository, destination.dirname)
82
91
  encoded_repo = repo_id.split("/").map { |part| URI.encode_www_form_component(part) }.join("/")
@@ -84,8 +93,17 @@ module Cohere
84
93
  uri = URI("#{endpoint}/#{encoded_repo}/resolve/#{commit}/#{encoded_filename}")
85
94
 
86
95
  open_download_lock(download_lock_path(destination)) do |lock|
87
- raise Error, "Cannot acquire Hub download lock for #{destination}" unless lock.flock(File::LOCK_EX)
88
- return destination if safe_cached_file(repository, destination)
96
+ acquire_download_lock!(lock, download_lock_path(destination), destination)
97
+
98
+ verify_cache_lock_identity!(
99
+ download_lock_path(destination),
100
+ lock,
101
+ purpose: "Hub download"
102
+ )
103
+ if (cached = safe_cached_file(repository, destination))
104
+ normalize_cached_payload(repository, cached)
105
+ return cached
106
+ end
89
107
 
90
108
  cleanup_download_temporaries(destination)
91
109
  Tempfile.create(
@@ -93,6 +111,7 @@ module Cohere
93
111
  destination.dirname.to_s,
94
112
  binmode: true
95
113
  ) do |temporary|
114
+ apply_cache_mode_if_supported(temporary, cache_payload_mode(destination.dirname))
96
115
  request(uri, stream: temporary)
97
116
  temporary.flush
98
117
  temporary.fsync
@@ -102,7 +121,7 @@ module Cohere
102
121
  sync_directory(destination.dirname)
103
122
  end
104
123
  destination
105
- rescue Errno::EACCES, Errno::ENOSPC, Errno::EROFS => e
124
+ rescue SystemCallError => e
106
125
  raise Error, "Cannot cache #{repo_id}/#{filename}: #{e.message}"
107
126
  end
108
127
 
@@ -110,16 +129,22 @@ module Cohere
110
129
  commit = resolve_revision(repo_id, revision)
111
130
  encoded_repo = repo_id.split("/").map { |part| URI.encode_www_form_component(part) }.join("/")
112
131
  response = request(URI("#{endpoint}/api/models/#{encoded_repo}/revision/#{commit}"))
113
- payload = JSON.parse(response.body)
114
- siblings = payload["siblings"]
132
+ payload = JSON.parse(response.body.to_s)
133
+ siblings = payload.is_a?(Hash) ? payload["siblings"] : nil
115
134
  raise Error, "Hub returned no repository file list for #{repo_id}@#{commit}" unless siblings.is_a?(Array)
116
135
 
117
- siblings.filter_map do |item|
136
+ files = siblings.filter_map do |item|
118
137
  name = item.is_a?(Hash) ? item["rfilename"] : nil
119
- name if name.is_a?(String)
120
- end.freeze
138
+ name if name.is_a?(String) && !name.empty?
139
+ end
140
+ if files.empty?
141
+ detail = siblings.empty? ? "an empty repository file list" : "no valid repository file entries"
142
+ raise Error, "Hub returned #{detail} for #{repo_id}@#{commit}"
143
+ end
144
+
145
+ files.freeze
121
146
  rescue JSON::ParserError => e
122
- raise Error, "Invalid Hub response while listing #{repo_id.inspect}: #{e.message}"
147
+ raise TransientError, "Invalid Hub response while listing #{repo_id.inspect}: #{e.message}"
123
148
  end
124
149
 
125
150
  def snapshot_path(repo_id, commit)
@@ -147,6 +172,77 @@ module Cohere
147
172
 
148
173
  private
149
174
 
175
+ def resolve_online_revision(repo_id, revision)
176
+ key = resolution_key(repo_id, revision)
177
+ memo = @resolution_guard.synchronize do
178
+ prune_resolution_memo
179
+ @resolution_memo[key]
180
+ end
181
+ return resolution_value(memo.value, memo.error) if memo
182
+
183
+ # Concurrent misses deliberately perform independent requests. Each
184
+ # caller uses the response it fetched; the short memo only avoids
185
+ # repeated requests that begin after one lookup has completed.
186
+ value = fetch_online_revision(repo_id, revision)
187
+ write_ref_best_effort(repo_id, revision, value)
188
+ @resolution_guard.synchronize { memoize_resolution(key, value, nil) }
189
+ value
190
+ rescue TransientError => e
191
+ memo = @resolution_guard.synchronize do
192
+ prune_resolution_memo
193
+ current = @resolution_memo[key]
194
+ memoize_resolution(key, nil, e) unless current
195
+ current
196
+ end
197
+ return resolution_value(memo.value, memo.error) if memo
198
+
199
+ raise
200
+ end
201
+
202
+ def fetch_online_revision(repo_id, revision)
203
+ encoded_repo = repo_id.split("/").map { |part| URI.encode_www_form_component(part) }.join("/")
204
+ encoded_revision = URI.encode_www_form_component(revision)
205
+ response = request(URI("#{endpoint}/api/models/#{encoded_repo}/revision/#{encoded_revision}"))
206
+ payload = JSON.parse(response.body.to_s)
207
+ commit = payload.is_a?(Hash) ? payload["sha"] : nil
208
+ unless commit.is_a?(String) && COMMIT_PATTERN.match?(commit)
209
+ raise TransientError, "Hub returned no immutable commit for #{repo_id.inspect} at #{revision.inspect}"
210
+ end
211
+
212
+ commit.downcase
213
+ rescue JSON::ParserError => e
214
+ raise TransientError, "Invalid Hub response while resolving #{repo_id.inspect}: #{e.message}"
215
+ end
216
+
217
+ def memoize_resolution(key, value, error)
218
+ @resolution_memo.delete(key)
219
+ @resolution_memo[key] = ResolutionMemo.new(
220
+ value: value,
221
+ error: error,
222
+ expires_at: monotonic + RESOLUTION_MEMO_TTL_SECONDS
223
+ )
224
+ @resolution_memo.shift while @resolution_memo.length > RESOLUTION_MEMO_LIMIT
225
+ end
226
+
227
+ def resolution_value(value, error)
228
+ raise error if error
229
+
230
+ value
231
+ end
232
+
233
+ def prune_resolution_memo
234
+ now = monotonic
235
+ @resolution_memo.delete_if { |_key, entry| entry.expires_at <= now }
236
+ end
237
+
238
+ def resolution_key(repo_id, revision)
239
+ [repo_id.dup.freeze, revision.dup.freeze].freeze
240
+ end
241
+
242
+ def monotonic
243
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
244
+ end
245
+
150
246
  def request(uri, stream: nil, redirects: 0)
151
247
  raise Error, "Hub offline mode prevents a request to #{uri}" if offline?
152
248
  raise Error, "Too many redirects while fetching #{uri}" if redirects > 8
@@ -180,7 +276,7 @@ module Cohere
180
276
  end
181
277
  expected = response_content_length(incoming, uri)
182
278
  if expected && written != expected
183
- raise Error,
279
+ raise TransientError,
184
280
  "Hub returned an incomplete response body for #{uri}: " \
185
281
  "received #{written} bytes, expected #{expected}"
186
282
  end
@@ -217,7 +313,16 @@ module Cohere
217
313
  location = response["location"]
218
314
  raise Error, "Hub redirect from #{base} did not include a location" unless location
219
315
 
220
- URI.join(base.to_s, location)
316
+ resolved = URI.join(base.to_s, location)
317
+ unless resolved.is_a?(URI::HTTP) && resolved.host
318
+ raise Error, "Hub redirect from #{base} returned an invalid location: #{location.inspect}"
319
+ end
320
+
321
+ resolved
322
+ rescue URI::Error => e
323
+ raise Error,
324
+ "Hub redirect from #{base} returned an invalid location: #{location.inspect}",
325
+ cause: e
221
326
  end
222
327
 
223
328
  def trusted_authorization_origin?(uri)
@@ -228,7 +333,7 @@ module Cohere
228
333
  def raise_http_error!(response, uri)
229
334
  detail = begin
230
335
  parsed = JSON.parse(response.body.to_s)
231
- parsed["error"] || parsed["message"]
336
+ parsed.is_a?(Hash) ? parsed["error"] || parsed["message"] : nil
232
337
  rescue JSON::ParserError
233
338
  nil
234
339
  end
@@ -237,6 +342,7 @@ module Cohere
237
342
  case response.code.to_i
238
343
  when 401, 403 then raise AuthenticationError, message
239
344
  when 404 then raise NotFoundError, message
345
+ when 429, 500..599 then raise TransientError, message
240
346
  else raise Error, message
241
347
  end
242
348
  end
@@ -246,7 +352,7 @@ module Cohere
246
352
  candidates = []
247
353
  ref = revision_path(repository, revision)
248
354
  cached_ref = safe_cached_file(repository, ref)
249
- candidates << cached_ref.read.strip if cached_ref
355
+ candidates << read_ref_commit(cached_ref) if cached_ref
250
356
  candidates << revision if COMMIT_PATTERN.match?(revision)
251
357
  candidates.find do |candidate|
252
358
  COMMIT_PATTERN.match?(candidate) &&
@@ -256,23 +362,50 @@ module Cohere
256
362
  nil
257
363
  end
258
364
 
365
+ def read_ref_commit(path)
366
+ File.open(path, "rb") do |ref|
367
+ next if ref.stat.size > REF_READ_LIMIT
368
+
369
+ value = ref.read(REF_READ_LIMIT)&.strip
370
+ next if ref.stat.size > REF_READ_LIMIT
371
+
372
+ value if value && COMMIT_PATTERN.match?(value)
373
+ end
374
+ rescue SystemCallError
375
+ nil
376
+ end
377
+
378
+ def write_ref_best_effort(repo_id, revision, commit)
379
+ write_ref(repo_id, revision, commit)
380
+ rescue Error, SystemCallError
381
+ # Hub refs are advisory cache hints. A valid online response remains
382
+ # usable when a shared or read-only cache cannot publish the hint.
383
+ commit
384
+ end
385
+
259
386
  def write_ref(repo_id, revision, commit)
260
387
  repository = cache_dir.join(cache_repo_name(repo_id))
261
388
  path = revision_path(repository, revision)
262
- prepare_cache_directory!(repository, path.dirname)
263
- # Resolution can happen concurrently in separate Transcriber facades.
264
- # A PID-only temporary name lets those threads truncate or rename the
265
- # same file out from under one another. Each writer instead publishes a
266
- # complete, durable ref from its own file; every competing value is an
267
- # immutable commit, so the final atomic rename remains consistent.
389
+ prepare_cache_directory!(repository, path.dirname, replaceable_entries: true)
390
+ current = safe_cached_file(repository, path)
391
+ if current && read_ref_commit(current) == commit
392
+ normalize_cached_file_mode(current, cache_payload_mode(current.dirname))
393
+ return commit
394
+ end
395
+
396
+ # Standard Hub refs contain only the immutable commit. Concurrent
397
+ # writers may replace one complete hint with another, while Tempfile
398
+ # and rename ensure readers never observe a partial value.
268
399
  Tempfile.create([".cohere-transcribe-ref-", ".tmp"], path.dirname.to_s) do |temporary|
269
- temporary.write("#{commit}\n")
400
+ temporary.write(commit)
401
+ apply_cache_mode_if_supported(temporary, cache_payload_mode(path.dirname))
270
402
  temporary.flush
271
403
  temporary.fsync
272
404
  temporary.close
273
405
  File.rename(temporary.path, path)
274
406
  end
275
407
  sync_directory(path.dirname)
408
+ commit
276
409
  end
277
410
 
278
411
  def safe_cached_file(repository, path)
@@ -308,42 +441,170 @@ module Cohere
308
441
  pattern = destination.dirname.join("#{download_temporary_prefix(destination)}*.download")
309
442
  Dir.glob(pattern.to_s).each do |path|
310
443
  File.unlink(path)
311
- rescue Errno::ENOENT
444
+ rescue Errno::ENOENT, Errno::EACCES, Errno::EPERM, Errno::EROFS
312
445
  nil
313
446
  end
314
447
  end
315
448
 
316
- def open_download_lock(path)
317
- flags = File::RDWR | File::CREAT
318
- flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
319
- flags |= File::CLOEXEC if defined?(File::CLOEXEC)
320
- descriptor = ::IO.sysopen(path.to_s, flags, 0o600)
321
- lock = File.new(descriptor, "r+", autoclose: true)
322
- descriptor = nil
323
- opened = lock.stat
324
- current = path.lstat
325
- unless opened.file? && !current.symlink? && opened.dev == current.dev && opened.ino == current.ino
326
- raise Error, "Hub download lock changed while it was being opened or is not regular: #{path}"
327
- end
449
+ def open_download_lock(path, &)
450
+ open_cache_lock(path, purpose: "Hub download", &)
451
+ end
452
+
453
+ def open_cache_lock(path, purpose:)
454
+ path = Pathname(path)
455
+ desired_mode = cache_lock_mode(path.dirname)
456
+ lock, access, created = open_cache_lock_handle(path, purpose: purpose, mode: desired_mode)
457
+ opened = verify_cache_lock_identity!(path, lock, purpose: purpose)
458
+ owned_writable = access == "r+" && cache_file_owned_by_current_process?(opened)
459
+ apply_cache_mode_if_supported(lock, desired_mode) if (created || owned_writable) && (opened.mode & 0o777) != desired_mode
328
460
 
329
461
  yield lock
330
462
  rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO => e
331
- raise Error, "Hub download lock is not a regular file: #{path}", cause: e
463
+ raise Error, "#{purpose} lock is not a regular file: #{path}", cause: e
332
464
  ensure
333
465
  lock&.close
466
+ end
467
+
468
+ def open_cache_lock_handle(path, purpose:, mode:)
469
+ descriptor = nil
470
+ handle = nil
471
+ replacement = nil
472
+ retired = nil
473
+ succeeded = false
474
+ begin
475
+ descriptor, created = open_or_create_cache_lock_descriptor(path, cache_lock_flags, mode)
476
+ access = "r+"
477
+ rescue Errno::EACCES, Errno::EROFS => e
478
+ descriptor = open_readonly_cache_lock_descriptor(path, purpose: purpose, write_error: e)
479
+ created = false
480
+ access = "r"
481
+ end
482
+
483
+ handle = File.new(descriptor, access, autoclose: true)
484
+ descriptor = nil
485
+ opened = verify_cache_lock_identity!(path, handle, purpose: purpose)
486
+ if access == "r" && cache_file_owned_by_current_process?(opened)
487
+ apply_cache_mode_if_supported(handle, mode)
488
+ replacement = reopen_cache_lock_writable(path, purpose: purpose)
489
+ if replacement
490
+ retired = handle
491
+ handle = replacement
492
+ replacement = nil
493
+ access = "r+"
494
+ retired.close
495
+ retired = nil
496
+ end
497
+ end
498
+ succeeded = true
499
+ [handle, access, created]
500
+ rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO => e
501
+ raise Error, "#{purpose} lock is not a regular file: #{path}", cause: e
502
+ ensure
503
+ retired.close if retired && !retired.closed?
504
+ replacement.close if replacement && !replacement.closed?
505
+ handle.close if handle && !succeeded && !handle.closed?
506
+ ::IO.new(descriptor).close if descriptor
507
+ end
508
+
509
+ def open_or_create_cache_lock_descriptor(path, flags, mode)
510
+ 2.times do
511
+ return [::IO.sysopen(path.to_s, flags, mode), false]
512
+ rescue Errno::ENOENT
513
+ begin
514
+ return [::IO.sysopen(path.to_s, flags | File::CREAT | File::EXCL, mode), true]
515
+ rescue Errno::EEXIST
516
+ next
517
+ end
518
+ end
519
+
520
+ raise Errno::ENOENT, path.to_s
521
+ end
522
+
523
+ def open_readonly_cache_lock_descriptor(path, purpose:, write_error:)
524
+ ::IO.sysopen(path.to_s, readonly_cache_lock_flags)
525
+ rescue Errno::EACCES => e
526
+ raise Error,
527
+ "#{purpose} lock is not readable at #{path}; ask the cache owner to grant shared read/write access, " \
528
+ "or remove the stale lock only after all downloads have stopped",
529
+ cause: e
530
+ rescue Errno::ENOENT => e
531
+ raise Error, "Cannot create #{purpose.downcase} lock at #{path}: #{write_error.message}", cause: e
532
+ end
533
+
534
+ def reopen_cache_lock_writable(path, purpose:)
535
+ handle = nil
536
+ succeeded = false
537
+ descriptor = ::IO.sysopen(path.to_s, cache_lock_flags)
538
+ handle = File.new(descriptor, "r+", autoclose: true)
539
+ descriptor = nil
540
+ verify_cache_lock_identity!(path, handle, purpose: purpose)
541
+ succeeded = true
542
+ handle
543
+ rescue Errno::EACCES, Errno::EROFS
544
+ nil
545
+ ensure
546
+ handle&.close unless succeeded
334
547
  ::IO.new(descriptor).close if descriptor
335
548
  end
336
549
 
550
+ def cache_lock_flags
551
+ flags = File::RDWR
552
+ flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
553
+ flags |= File::CLOEXEC if defined?(File::CLOEXEC)
554
+ flags
555
+ end
556
+
557
+ def readonly_cache_lock_flags
558
+ flags = File::RDONLY
559
+ flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
560
+ flags |= File::CLOEXEC if defined?(File::CLOEXEC)
561
+ flags
562
+ end
563
+
564
+ def acquire_download_lock!(lock, path, destination)
565
+ return if lock.flock(File::LOCK_EX)
566
+
567
+ raise Error, "Cannot acquire Hub download lock for #{destination} at #{path}"
568
+ rescue SystemCallError => e
569
+ if e.is_a?(Errno::EBADF)
570
+ detail = "the filesystem requires a writable shared lock file"
571
+ elsif cache_locking_unsupported?(e)
572
+ detail = "the filesystem does not provide the required file locking"
573
+ else
574
+ raise
575
+ end
576
+ raise Error, "Cannot acquire Hub download lock for #{destination}: #{detail} at #{path}", cause: e
577
+ end
578
+
579
+ def cache_locking_unsupported?(error)
580
+ names = %i[ENOLCK EOPNOTSUPP ENOTSUP]
581
+ names.any? { |name| Errno.const_defined?(name) && error.errno == Errno.const_get(name)::Errno }
582
+ end
583
+
584
+ def cache_file_owned_by_current_process?(stat)
585
+ Process.respond_to?(:euid) && stat.uid == Process.euid
586
+ end
587
+
588
+ def verify_cache_lock_identity!(path, lock, purpose:)
589
+ opened = lock.stat
590
+ current = path.lstat
591
+ return opened if opened.file? && !current.symlink? && opened.dev == current.dev && opened.ino == current.ino
592
+
593
+ raise Error, "#{purpose} lock changed while held or is not a regular file: #{path}"
594
+ rescue SystemCallError => e
595
+ raise Error, "#{purpose} lock changed while held or is not a regular file: #{path}", cause: e
596
+ end
597
+
337
598
  # Build each repository subdirectory only after proving that its parent is
338
599
  # still inside this repository. This prevents a poisoned snapshots/refs
339
600
  # symlink from redirecting a Tempfile and rename outside the cache.
340
- def prepare_cache_directory!(repository, directory)
601
+ def prepare_cache_directory!(repository, directory, replaceable_entries: false)
341
602
  FileUtils.mkdir_p(cache_dir)
342
603
  cache_root = cache_dir.realpath
343
604
  begin
344
- Dir.mkdir(repository, 0o755)
605
+ create_cache_directory(repository, cache_dir)
345
606
  rescue Errno::EEXIST
346
- nil
607
+ normalize_existing_cache_directory(repository, replaceable_entries: false)
347
608
  end
348
609
 
349
610
  repository_stat = repository.lstat
@@ -364,9 +625,9 @@ module Cohere
364
625
 
365
626
  current = current.join(part)
366
627
  begin
367
- Dir.mkdir(current, 0o755)
628
+ create_cache_directory(current, current.dirname, replaceable_entries: replaceable_entries)
368
629
  rescue Errno::EEXIST
369
- nil
630
+ normalize_existing_cache_directory(current, replaceable_entries: replaceable_entries)
370
631
  end
371
632
  resolved = current.realpath
372
633
  unless resolved.directory? && within_directory?(resolved, repository_root)
@@ -380,6 +641,133 @@ module Cohere
380
641
  raise Error, "Cannot prepare Hub cache directory #{directory}: #{e.message}"
381
642
  end
382
643
 
644
+ def create_cache_directory(path, parent, replaceable_entries: false)
645
+ # New cache directories inherit the verified parent's ordinary access
646
+ # bits and setgid policy even when this process has a private umask.
647
+ # Owner access keeps a newly-created private cache usable.
648
+ mode = cache_directory_mode(parent, replaceable_entries: replaceable_entries)
649
+ Dir.mkdir(path, mode)
650
+ flags = File::RDONLY
651
+ flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
652
+ flags |= File::CLOEXEC if defined?(File::CLOEXEC)
653
+ File.open(path, flags) do |handle|
654
+ opened = handle.stat
655
+ current = path.lstat
656
+ unless opened.directory? && !current.symlink? && opened.dev == current.dev && opened.ino == current.ino
657
+ raise Error, "Hub cache directory changed while it was being created: #{path}"
658
+ end
659
+
660
+ apply_cache_mode_if_supported(handle, mode)
661
+ end
662
+ end
663
+
664
+ def normalize_existing_cache_directory(path, replaceable_entries:)
665
+ return if replaceable_entries
666
+
667
+ flags = File::RDONLY
668
+ flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
669
+ flags |= File::CLOEXEC if defined?(File::CLOEXEC)
670
+ File.open(path, flags) do |handle|
671
+ opened = handle.stat
672
+ current = path.lstat
673
+ unless opened.directory? && !current.symlink? && opened.dev == current.dev && opened.ino == current.ino
674
+ raise Error, "Hub cache directory changed while its mode was being updated: #{path}"
675
+ end
676
+ next unless cache_file_owned_by_current_process?(opened)
677
+
678
+ current_mode = opened.mode & 0o3777
679
+ desired_mode = current_mode | (current_mode.anybits?(0o022) ? 0o1000 : 0)
680
+ apply_cache_mode_if_supported(handle, desired_mode) if current_mode != desired_mode
681
+ end
682
+ rescue Errno::EACCES, Errno::EROFS, Errno::ENOENT, Errno::ELOOP
683
+ nil
684
+ end
685
+
686
+ def normalize_cached_file_mode(path, desired_mode)
687
+ return if path.lstat.symlink?
688
+
689
+ flags = File::RDONLY
690
+ flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
691
+ flags |= File::CLOEXEC if defined?(File::CLOEXEC)
692
+ File.open(path, flags) do |handle|
693
+ opened = handle.stat
694
+ current = path.lstat
695
+ unless opened.file? && !current.symlink? && opened.dev == current.dev && opened.ino == current.ino
696
+ raise Error, "Hub cache file changed while its mode was being updated: #{path}"
697
+ end
698
+ next unless cache_file_owned_by_current_process?(opened)
699
+ next if (opened.mode & 0o777) == desired_mode
700
+
701
+ apply_cache_mode_if_supported(handle, desired_mode)
702
+ end
703
+ rescue Errno::EACCES, Errno::EROFS, Errno::ENOENT, Errno::ELOOP
704
+ nil
705
+ end
706
+
707
+ def normalize_cached_payload(repository, path)
708
+ repository_root = repository.realpath
709
+ relative = path.dirname.relative_path_from(repository)
710
+ return if relative.absolute? || relative.each_filename.any? { |part| part == ".." }
711
+
712
+ current = repository
713
+ normalize_existing_cache_directory(current, replaceable_entries: false)
714
+ components = relative.each_filename.reject { |part| part == "." }
715
+ until components.empty?
716
+ part = components.shift
717
+
718
+ current = current.join(part)
719
+ resolved = current.realpath
720
+ return unless resolved.directory? && within_directory?(resolved, repository_root)
721
+
722
+ normalize_existing_cache_directory(current, replaceable_entries: false)
723
+ end
724
+ normalize_cached_file_mode(path, cache_payload_mode(path.dirname))
725
+ rescue Error, SystemCallError, ArgumentError
726
+ # Reusing a verified immutable payload does not depend on being able to
727
+ # upgrade access modes in an older or read-only cache.
728
+ nil
729
+ end
730
+
731
+ def cache_directory_mode(parent, replaceable_entries: false)
732
+ parent_mode = parent.stat.mode
733
+ mode = 0o700 | (parent_mode & 0o2000)
734
+ mode |= parent_mode & 0o070 if parent_mode.anybits?(0o010)
735
+ mode |= parent_mode & 0o007 if parent_mode.anybits?(0o001)
736
+ # Snapshot directories keep other users from replacing published
737
+ # payloads. Ref directories omit this bit so collaborators can replace
738
+ # one another's advisory branch hints through atomic rename.
739
+ mode |= 0o1000 if !replaceable_entries && mode.anybits?(0o022)
740
+ mode
741
+ end
742
+
743
+ def cache_payload_mode(parent)
744
+ # Published payloads and refs inherit read access, never collaborator
745
+ # write access. Ref replacement is provided by the parent directory.
746
+ parent_mode = parent.stat.mode
747
+ mode = 0o600
748
+ mode |= 0o040 if parent_mode.allbits?(0o050)
749
+ mode |= 0o004 if parent_mode.allbits?(0o005)
750
+ mode
751
+ end
752
+
753
+ def cache_lock_mode(parent)
754
+ # Lock files must be writable by every collaborator who can create
755
+ # cache entries in the containing directory.
756
+ parent_mode = parent.stat.mode
757
+ mode = 0o600
758
+ mode |= 0o060 if parent_mode.allbits?(0o030)
759
+ mode |= 0o006 if parent_mode.allbits?(0o003)
760
+ mode
761
+ end
762
+
763
+ def apply_cache_mode_if_supported(target, mode)
764
+ target.chmod(mode)
765
+ rescue SystemCallError => e
766
+ raise unless CACHE_MODE_UNSUPPORTED_ERRNOS.include?(e.errno)
767
+
768
+ nil
769
+ end
770
+
383
771
  def within_directory?(path, directory)
384
772
  path_text = path.to_s
385
773
  directory_text = directory.to_s