cohere-transcribe 0.1.2 → 0.1.3

data/lib/cohere/transcribe/state/locking.rb

@@ -2,19 +2,28 @@
 
 require "digest"
 require "fileutils"
+require "rubygems/version"
 require "tmpdir"
 
 module Cohere
   module Transcribe
     module State
       LOCK_DIRECTORY_PREFIX = "cohere-transcribe"
+      LOCK_DIRECTORY_NAME = ".cohere-transcribe-locks"
       LOCK_SUFFIX = ".lock"
+      LEGACY_LOCK_REMOVAL_VERSION = "0.2.0"
 
-      OutputLockTarget = Data.define(:path, :identity) do
-        def sort_key
-          [path.to_s.downcase, identity.downcase]
+      OutputLockTarget = Data.define(:path, :identity)
+      LockPath = Data.define(:path, :role) do
+        def canonical?
+          role == :canonical
+        end
+
+        def legacy?
+          role == :legacy
         end
       end
+      private_constant :LockPath
 
       class OutputSetLock
         @guard = Mutex.new
@@ -23,79 +32,173 @@ module Cohere
         class << self
           attr_reader :guard, :active
 
-          def acquire(target, blocking: false, operation_hook: nil)
+          def acquire(target, blocking: false, operation_hook: nil, ownership_hook: nil)
+            State.ensure_lock_acquisition_allowed!(target)
+            completed = false
+            failed = false
             guard.synchronize do
-              handle = nil
-              acquired = false
-              key = target.path.to_s
-              if active.key?(key)
+              handles = []
+              lock = nil
+              lock_paths = State.lock_path_specs_for_target(target)
+              paths = lock_paths.map(&:path)
+              keys = paths.map(&:to_s)
+              if keys.any? { |key| active.key?(key) }
                 raise TranscriptionRuntimeError,
                       "Another transcription job owns output set #{target.identity}"
               end
 
-              handle = State.open_lock_file(target.path)
-              operation = File::LOCK_EX
-              operation |= File::LOCK_NB unless blocking
-              unless handle.flock(operation)
-                handle.close
-                raise TranscriptionRuntimeError,
-                      "Another transcription process owns output set #{target.identity} (lock #{target.path})"
+              lock_paths.each_with_index do |lock_path, index|
+                path = lock_path.path
+                handle = nil
+                Thread.handle_interrupt(Object => :never) do
+                  handle = State.open_lock_file(path, shared: lock_path.canonical?)
+                  handles << handle
+                end
+                operation = File::LOCK_EX
+                operation |= File::LOCK_NB unless blocking
+                unless State.acquire_exclusive_file_lock(handle, operation, target: target, path: path)
+                  raise TranscriptionRuntimeError,
+                        "Another transcription process owns output set #{target.identity} (lock #{path})"
+                end
+                operation_hook&.call(:after_flock, target) if index.zero?
+                State.verify_lock_identity!(
+                  path,
+                  handle,
+                  message: "Output lock changed while acquiring #{target.identity}"
+                )
+                State.refresh_legacy_lock(handle) if lock_path.legacy?
               end
-              operation_hook&.call(:after_flock, target)
-              State.verify_lock_identity!(
-                target.path,
-                handle,
-                message: "Output lock changed while acquiring #{target.identity}"
-              )
 
-              lock = new(target, handle, key)
-              active[key] = lock
-              acquired = true
+              lock = new(target, handles, keys, roles: lock_paths.map(&:role))
+              keys.each do |key|
+                active[key] = lock
+              end
+              ownership_hook&.call(lock)
+              completed = true
               lock
             rescue Errno::EWOULDBLOCK, Errno::EAGAIN
-              handle&.close
+              failed = true
+              raise TranscriptionRuntimeError,
+                    "Another transcription process owns output set #{target.identity}"
+            rescue SystemCallError => e
+              failed = true
               raise TranscriptionRuntimeError,
-                    "Another transcription process owns output set #{target.identity} (lock #{target.path})"
-            rescue Interrupt, SystemExit, StandardError
-              handle&.close
+                    "Cannot acquire output lock for #{target.identity}: #{e.message}",
+                    cause: e
+            rescue Exception # rubocop:disable Lint/RescueException -- record only this acquisition's unwind
+              failed = true
               raise
             ensure
-              handle.close if handle && !acquired && !handle.closed?
+              unless completed && !failed && !State.lock_acquisition_unwinding?
+                Thread.handle_interrupt(Object => :never) do
+                  delete_active_keys(lock, keys || [])
+                  close_lock_handles(handles || [])
+                  lock&.mark_released!
+                end
+              end
             end
           end
 
           def release_all
-            guard.synchronize { active.values.dup }.each(&:release)
+            guard.synchronize { active.values.uniq }.each(&:release)
+          end
+
+          def close_lock_handles(handles)
+            first_error = nil
+            Thread.handle_interrupt(Object => :never) do
+              handles.reverse_each do |handle|
+                begin
+                  next if handle.closed?
+                rescue Exception => e # rubocop:disable Lint/RescueException
+                  first_error ||= e
+                end
+
+                begin
+                  handle.flock(File::LOCK_UN)
+                rescue Exception => e # rubocop:disable Lint/RescueException
+                  first_error ||= e
+                end
+                begin
+                  handle.close
+                rescue Exception => e # rubocop:disable Lint/RescueException
+                  first_error ||= e
+                end
+              end
+            end
+            first_error
+          end
+
+          def delete_active_keys(lock, keys)
+            first_error = nil
+            keys.reverse_each do |key|
+              active.delete(key) if active[key].equal?(lock)
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              first_error ||= e
+            end
+            first_error
           end
         end
 
         attr_reader :target
 
-        def initialize(target, handle, key)
+        def initialize(target, handles, keys, roles: nil)
           @target = target
-          @handle = handle
-          @key = key
+          @handles = handles.freeze
+          @keys = keys.freeze
+          @roles = (roles || Array.new(keys.length, :primary)).freeze
           @released = false
         end
 
-        def release
+        def verify!
           self.class.guard.synchronize do
-            return if @released
+            if @released
+              raise TranscriptionRuntimeError,
+                    "Output lock was released before committing #{target.identity}"
+            end
 
-            begin
-              @handle.flock(File::LOCK_UN)
-            ensure
-              @handle.close
-              @released = true
-              self.class.active.delete(@key)
+            canonical_held = @roles.include?(:canonical)
+            @keys.zip(@handles, @roles).each do |path, handle, role|
+              State.verify_lock_identity!(
+                path,
+                handle,
+                message: "Output lock changed while held for #{target.identity}"
+              )
+            rescue TranscriptionRuntimeError => e
+              # The output-adjacent lock is authoritative. A tmp cleaner may
+              # remove the 0.1.x compatibility path; replacement is still fatal.
+              raise unless canonical_held && role == :legacy && State.missing_lock_path_error?(e)
             end
           end
           nil
         end
 
+        def release
+          cleanup_error = nil
+          Thread.handle_interrupt(Object => :never) do
+            self.class.guard.synchronize do
+              return if @released
+
+              begin
+                handle_error = self.class.close_lock_handles(@handles)
+                registration_error = self.class.delete_active_keys(self, @keys)
+                cleanup_error = handle_error || registration_error
+              ensure
+                @released = true
+              end
+            end
+          end
+          raise cleanup_error if cleanup_error
+
+          nil
+        end
+
         def released?
           @released
         end
+
+        def mark_released!
+          @released = true
+        end
       end
 
       module_function
@@ -103,42 +206,112 @@ module Cohere
       def lock_target_for_outputs(output_paths)
         parent, stem = output_parent_and_stem(output_paths)
         identity = parent.join(stem).expand_path.cleanpath.to_s
-        digest = Digest::SHA256.hexdigest(identity.downcase)
         OutputLockTarget.new(
-          path: lock_registry_directory.join("#{digest}#{LOCK_SUFFIX}").freeze,
+          path: canonical_lock_path(identity).freeze,
           identity: identity.freeze
         )
       end
 
       def with_output_lock(target, blocking: false)
-        lock = OutputSetLock.acquire(target, blocking: blocking)
-        yield lock
-      ensure
-        lock&.release
+        lock = nil
+        primary_error = nil
+        Thread.handle_interrupt(Object => :never) do
+          Thread.handle_interrupt(Object => :on_blocking) do
+            OutputSetLock.acquire(
+              target,
+              blocking: blocking,
+              ownership_hook: ->(acquired_lock) { lock = acquired_lock }
+            )
+          end
+          Thread.handle_interrupt(Object => :immediate) { yield lock }
+        rescue Exception => e # rubocop:disable Lint/RescueException -- preserve protected-work failures
+          primary_error = e
+          raise
+        ensure
+          begin
+            lock&.release
+          rescue Exception # rubocop:disable Lint/RescueException -- preserve protected-work failures
+            raise unless primary_error || lock_acquisition_unwinding?
+          end
+        end
+      end
+
+      def lock_paths_for_target(target)
+        lock_path_specs_for_target(target).map(&:path).freeze
+      end
+
+      def lock_path_specs_for_target(target)
+        primary = Pathname(target.path).expand_path.cleanpath
+        canonical = canonical_lock_path(target.identity)
+        legacy = legacy_temporary_lock_path(target.identity)
+        unless primary == canonical
+          role = primary == legacy ? :legacy : :custom
+          return [LockPath.new(path: primary.freeze, role: role)].freeze
+        end
+
+        # Remove this temporary-registry compatibility path in 0.2.0. Until
+        # then, acquiring both paths keeps released 0.1.2 processes coordinated
+        # with output-adjacent locks.
+        paths = [LockPath.new(path: primary.freeze, role: :canonical)]
+        paths << LockPath.new(path: legacy.freeze, role: :legacy) if legacy_lock_compatibility_enabled? && legacy != primary
+        paths.freeze
       end
 
-      def lock_registry_directory
+      def legacy_lock_compatibility_enabled?(version = VERSION)
+        Gem::Version.new(version) < Gem::Version.new(LEGACY_LOCK_REMOVAL_VERSION)
+      end
+
+      def canonical_lock_path(identity)
+        identity = Pathname(identity).expand_path.cleanpath
+        lock_registry_directory(identity.dirname).join(lock_filename(identity.to_s))
+      end
+
+      def lock_registry_directory(output_parent)
+        Pathname(output_parent).expand_path.cleanpath.join(LOCK_DIRECTORY_NAME)
+      end
+
+      def legacy_temporary_lock_path(identity)
         scope = Process.respond_to?(:uid) ? Process.uid.to_s : Digest::SHA256.hexdigest(Dir.home)[0, 16]
-        Pathname(Dir.tmpdir).join("#{LOCK_DIRECTORY_PREFIX}-#{scope}")
+        Pathname(Dir.tmpdir).expand_path.join("#{LOCK_DIRECTORY_PREFIX}-#{scope}", lock_filename(identity))
+      end
+
+      def lock_filename(identity)
+        "#{Digest::SHA256.hexdigest(identity.downcase)}#{LOCK_SUFFIX}"
       end
 
-      def open_lock_file(path)
+      def open_lock_file(path, shared: false)
         path = Pathname(path)
         succeeded = false
-        validate_lock_directory!(path.dirname)
+        created = false
+        validate_lock_directory!(path.dirname, shared: shared)
         inspect_lock_path!(path)
-        flags = File::RDWR | File::CREAT
+        flags = File::RDWR
         flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
         flags |= File::CLOEXEC if defined?(File::CLOEXEC)
-        descriptor = ::IO.sysopen(path.to_s, flags, 0o600)
-        handle = File.new(descriptor, "r+", autoclose: true)
-        descriptor = nil
+        mode = shared ? shared_lock_file_mode(path.dirname.lstat) : 0o600
+        begin
+          descriptor, created = open_or_create_lock_descriptor(path, flags, mode)
+          access = "r+"
+        rescue Errno::EACCES, Errno::EROFS => e
+          raise unless shared
+
+          handle, access = open_shared_lock_after_write_denial(path, flags, mode, e)
+        end
+        unless handle
+          handle = File.new(descriptor, access, autoclose: true)
+          descriptor = nil
+        end
         opened = verify_lock_identity!(
           path,
           handle,
           message: "Output lock changed while it was being opened or is not regular"
         )
-        validate_owned_private_file!(opened, path)
+        if shared
+          owned_writable = access == "r+" && Process.respond_to?(:uid) && opened.uid == Process.uid
+          apply_lock_mode_if_supported(handle, mode) if (created || owned_writable) && (opened.mode & 0o777) != mode
+        else
+          validate_owned_private_file!(opened, path)
+        end
         succeeded = true
         handle
       rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO => e
@@ -151,17 +324,37 @@ module Cohere
         end
       end
 
-      def validate_lock_directory!(path)
-        begin
-          Dir.mkdir(path, 0o700)
-        rescue Errno::EEXIST
-          nil
-        rescue SystemCallError => e
-          raise TranscriptionRuntimeError, "Cannot prepare output lock directory #{path}: #{e.message}"
+      def open_or_create_lock_descriptor(path, flags, mode)
+        2.times do
+          return [::IO.sysopen(path.to_s, flags, mode), false]
+        rescue Errno::ENOENT
+          begin
+            return [::IO.sysopen(path.to_s, flags | File::CREAT | File::EXCL, mode), true]
+          rescue Errno::EEXIST
+            next
+          end
+        end
+
+        raise Errno::ENOENT, path.to_s
+      end
+
+      def validate_lock_directory!(path, shared: false)
+        if shared
+          create_shared_lock_directory(path)
+        else
+          begin
+            FileUtils.mkdir_p(path, mode: 0o700)
+          rescue SystemCallError => e
+            raise TranscriptionRuntimeError, "Cannot prepare output lock directory #{path}: #{e.message}"
+          end
         end
         stat = path.lstat
         raise TranscriptionRuntimeError, "Output lock directory is not a real directory: #{path}" unless stat.directory? && !stat.symlink?
 
+        if shared
+          validate_shared_lock_directory_mode!(path, stat)
+          return
+        end
         return if Gem.win_platform?
 
         if Process.respond_to?(:uid) && stat.uid != Process.uid
@@ -176,6 +369,132 @@ module Cohere
         raise TranscriptionRuntimeError, "Cannot prepare output lock directory #{path}: #{e.message}"
       end
 
+      def create_shared_lock_directory(path)
+        parent = path.dirname.lstat
+        unless parent.directory? && !parent.symlink?
+          raise TranscriptionRuntimeError, "Output lock parent is not a real directory: #{path.dirname}"
+        end
+
+        mode = (parent.mode & 0o3777) | 0o700
+        mode |= 0o1000 if mode.anybits?(0o022)
+        begin
+          Dir.mkdir(path, mode)
+          apply_lock_mode_if_supported(path, mode)
+        rescue Errno::EEXIST
+          existing = path.lstat
+          owned = Process.respond_to?(:uid) && existing.uid == Process.uid
+          mode_can_be_updated = !Gem.win_platform? && existing.directory? && owned
+          apply_lock_mode_if_supported(path, mode) if mode_can_be_updated && (existing.mode & 0o3777) != mode
+        rescue SystemCallError => e
+          raise TranscriptionRuntimeError, "Cannot prepare output lock directory #{path}: #{e.message}"
+        end
+      end
+
+      def validate_shared_lock_directory_mode!(path, stat)
+        return if Gem.win_platform? || !Process.respond_to?(:uid)
+        return if stat.uid == Process.uid
+        return if stat.mode.nobits?(0o022) || stat.mode.anybits?(0o1000)
+
+        raise TranscriptionRuntimeError,
+              "Output lock directory is writable by multiple users but has no sticky bit: #{path}. " \
+              "Ask its owner or administrator to add the sticky bit, or remove it when no transcription job is running"
+      end
+
+      def shared_lock_file_mode(directory_stat)
+        mode = 0o600
+        mode |= 0o060 if directory_stat.mode.anybits?(0o020) && directory_stat.mode.anybits?(0o010)
+        mode |= 0o006 if directory_stat.mode.anybits?(0o002) && directory_stat.mode.anybits?(0o001)
+        mode
+      end
+
+      def apply_lock_mode_if_supported(target, mode)
+        target.chmod(mode)
+      rescue SystemCallError => e
+        unsupported = [Errno::EPERM::Errno]
+        unsupported << Errno::EROFS::Errno
+        unsupported << Errno::EOPNOTSUPP::Errno if defined?(Errno::EOPNOTSUPP)
+        unsupported << Errno::ENOTSUP::Errno if defined?(Errno::ENOTSUP)
+        raise unless unsupported.include?(e.errno)
+
+        nil
+      end
+
+      def refresh_legacy_lock(handle)
+        handle.rewind
+        handle.write("#{Process.pid}\n")
+        handle.flush
+        handle.truncate(handle.pos)
+        nil
+      end
+
+      def acquire_exclusive_file_lock(handle, operation, target:, path:)
+        handle.flock(operation)
+      rescue Errno::EBADF => e
+        raise TranscriptionRuntimeError,
+              "Cannot acquire an exclusive output lock for #{target.identity}: " \
+              "the filesystem requires a writable lock file at #{path}",
+              cause: e
+      end
+
+      def open_shared_lock_after_write_denial(path, flags, mode, original_error)
+        descriptor = ::IO.sysopen(path.to_s, readonly_lock_flags)
+        readonly = File.new(descriptor, "r", autoclose: true)
+        descriptor = nil
+        opened = verify_lock_identity!(
+          path,
+          readonly,
+          message: "Output lock changed while opening it read-only"
+        )
+        if original_error.is_a?(Errno::EACCES) &&
+           Process.respond_to?(:uid) && opened.uid == Process.uid
+          apply_lock_mode_if_supported(readonly, mode)
+          begin
+            descriptor = ::IO.sysopen(path.to_s, flags, mode)
+            writable = File.new(descriptor, "r+", autoclose: true)
+            descriptor = nil
+            readonly.close
+            readonly = nil
+            handle = writable
+            writable = nil
+            return [handle, "r+"]
+          rescue Errno::EACCES, Errno::EROFS
+            nil
+          end
+        end
+
+        handle = readonly
+        readonly = nil
+        [handle, "r"]
+      ensure
+        readonly&.close unless readonly&.closed?
+        writable&.close unless writable&.closed?
+        ::IO.new(descriptor).close if descriptor
+      end
+
+      def readonly_lock_flags
+        readonly_flags = File::RDONLY
+        readonly_flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
+        readonly_flags |= File::CLOEXEC if defined?(File::CLOEXEC)
+        readonly_flags
+      end
+
+      def ensure_lock_acquisition_allowed!(target)
+        return unless lock_acquisition_unwinding?
+
+        raise TranscriptionRuntimeError,
+              "Cannot acquire output lock for #{target.identity} while the current thread is terminating"
+      end
+
+      # Ruby exposes an in-progress Thread#kill unwind only through #status;
+      # pending_interrupt? is already false while ensure blocks are running.
+      def lock_acquisition_unwinding?
+        Thread.current.status == "aborting"
+      end
+
+      def missing_lock_path_error?(error)
+        error.cause.is_a?(Errno::ENOENT)
+      end
+
       def inspect_lock_path!(path)
         stat = path.lstat
         return if stat.file? && !stat.symlink?
@@ -202,7 +521,7 @@ module Cohere
         current = Pathname(path).lstat
         return opened if opened.file? && same_file?(opened, current)
 
-        raise TranscriptionRuntimeError, "#{message}: #{path}"
+        raise TranscriptionRuntimeError, "#{message}: #{path}", cause: nil
       rescue SystemCallError => e
         raise TranscriptionRuntimeError, "#{message}: #{path}", cause: e
       end

data/lib/cohere/transcribe/state/manifest.rb

@@ -52,8 +52,7 @@ module Cohere
       end
 
       def verify_published_outputs(source_snapshot:, output_paths:, state_path:, asr_contract_key:,
-                                   render_contract_key:, directory_binding: nil,
-                                   guard_bindings: nil)
+                                   render_contract_key:, directory_binding:, guard_bindings: nil)
         payload, reason = decode_state(
           state_path,
           directory_binding: directory_binding,
@@ -72,11 +71,18 @@ module Cohere
         )
         return PublicationVerification.new(verified: false, generation_id: nil, reason: mismatch.freeze) if mismatch
 
+        verify_publication_bindings!(directory_binding, guard_bindings)
         PublicationVerification.new(
           verified: true,
           generation_id: payload.fetch("generation_id").dup.freeze,
           reason: nil
         )
+      rescue PublicationDirectoryChangedError => e
+        PublicationVerification.new(
+          verified: false,
+          generation_id: nil,
+          reason: "publication parent changed during verification (#{e.message})".freeze
+        )
       rescue EncodingError, SystemCallError, TypeError, ArgumentError => e
         PublicationVerification.new(
           verified: false,
@@ -86,8 +92,7 @@ module Cohere
       end
 
       def publication_mismatch_reason(payload, source_snapshot:, output_paths:, asr_contract_key:,
-                                      render_contract_key:, directory_binding: nil,
-                                      guard_bindings: nil)
+                                      render_contract_key:, directory_binding:, guard_bindings: nil)
         return "state is #{payload["kind"].inspect}, not published" unless payload["kind"] == "published"
         return "state marker ASR contract does not match" unless payload["asr_contract_key"] == asr_contract_key
         return "state marker render contract does not match" unless payload["render_contract_key"] == render_contract_key
@@ -105,18 +110,11 @@ module Cohere
           return "state marker path for #{format} does not match" unless record.is_a?(Hash) && record["name"] == path.basename.to_s
 
           begin
-            output_record = if directory_binding
-                              bound_output_record(
-                                path,
-                                directory_binding: directory_binding,
-                                guard_bindings: guard_bindings
-                              )
-                            else
-                              stat = path.lstat
-                              return "#{format} output is missing or not regular" unless stat.file? && !stat.symlink?
-
-                              [stat.size, Digest::SHA256.file(path).hexdigest]
-                            end
+            output_record = bound_output_record(
+              path,
+              directory_binding: directory_binding,
+              guard_bindings: guard_bindings
+            )
             return "#{format} output changed or is not regular" unless output_record
 
             size, sha256 = output_record
@@ -140,18 +138,14 @@ module Cohere
             Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
               handle = bound.open_regular(basename)
             end
-          rescue TranscriptionRuntimeError
+          rescue PublicationEntryError
             bound.verify!
             return nil
           end
           opened = handle.stat
           digest = Digest::SHA256.new
           digest << handle.read(1024 * 1024) until handle.eof?
-          unchanged = begin
-            bound.same_regular_entry?(basename, opened)
-          rescue TranscriptionRuntimeError
-            false
-          end
+          unchanged = bound.same_regular_entry?(basename, opened)
           unless unchanged
             bound.verify!
             return nil
@@ -164,6 +158,11 @@ module Cohere
         end
       end
       private_class_method :bound_output_record
+
+      def verify_publication_bindings!(directory_binding, guard_bindings)
+        (Array(guard_bindings).compact + [directory_binding].compact).uniq.each(&:verify!)
+      end
+      private_class_method :verify_publication_bindings!
     end
   end
 end