cohere-transcribe 0.1.2 → 0.1.3

data/lib/cohere/transcribe/input.rb

@@ -4,6 +4,7 @@ require "find"
 
 require_relative "constants"
 require_relative "errors"
+require_relative "internal/utf8"
 require_relative "python_text"
 
 module Cohere
@@ -31,9 +32,11 @@ module Cohere
 
           text = value.is_a?(String) ? value : value.to_path
           raise TranscriptionInputError, "#{label} must resolve to a text path" unless text.is_a?(String)
+
+          text = utf8_path(text, label)
           raise TranscriptionInputError, "#{label} must not be empty" if PythonText.blank?(text)
 
-          text.dup.freeze
+          text.freeze
         rescue EncodingError, TypeError, ArgumentError, SystemCallError => e
           raise e if e.is_a?(TranscriptionInputError)
 
@@ -83,8 +86,11 @@ module Cohere
                     path.file? && AUDIO_EXTENSIONS.include?(path.extname.downcase)
                   end
                 end
+        paths = paths.map { |path| Pathname(utf8_path(path.to_s, "discovered audio")) }
         paths.sort_by { |path| path.to_s.downcase(:fold) }.map do |path|
-          [strict_realpath(path), path.relative_path_from(source)]
+          relative_path = path.relative_path_from(source)
+          relative_text = utf8_path(relative_path.to_s, "discovered audio")
+          [strict_realpath(path), Pathname(relative_text)]
         end
       rescue EncodingError, SystemCallError, ArgumentError => e
         raise TranscriptionInputError, "Cannot inspect input #{source}: #{e.message}"
@@ -92,7 +98,8 @@ module Cohere
       private_class_method :directory_candidates
 
       def strict_realpath(value)
-        Pathname(value).expand_path.realpath
+        resolved = Pathname(value).expand_path.realpath
+        Pathname(utf8_path(resolved.to_s, "resolved input"))
       rescue Errno::ENOENT
         raise TranscriptionInputError, "Input does not exist: #{Pathname(value).expand_path}"
       rescue EncodingError, SystemCallError, ArgumentError => e
@@ -105,6 +112,14 @@ module Cohere
         value.is_a?(String) || value.respond_to?(:to_path)
       end
       private_class_method :path_like?
+
+      def utf8_path(value, label)
+        text = Internal::UTF8.normalize(value)
+        return text if text
+
+        raise TranscriptionInputError, "#{label} path must contain valid UTF-8"
+      end
+      private_class_method :utf8_path
     end
   end
 end

data/lib/cohere/transcribe/internal/interruptible_native_call.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Cohere
+  module Transcribe
+    module Internal
+      # Runs a blocking foreign call on a dedicated worker so the caller remains
+      # interruptible. Callbacks are retained only for this invocation; callers
+      # remain responsible for supplying non-poisoning cancellation behavior.
+      module InterruptibleNativeCall
+        module_function
+
+        def run(cancel:, join_interval:, missing_outcome:, thread_name: nil)
+          raise ArgumentError, "an operation block is required" unless block_given?
+
+          outcome = Queue.new
+          worker = nil
+          Thread.handle_interrupt(Object => :on_blocking) do
+            worker = Thread.new do
+              outcome << [:returned, yield]
+            rescue Exception => e # rubocop:disable Lint/RescueException -- transfer native cancellation intact
+              outcome << [:raised, e]
+            end
+            worker.name = thread_name if thread_name && worker.respond_to?(:name=)
+            worker.report_on_exception = false
+            worker.join
+          ensure
+            if worker&.alive?
+              Thread.handle_interrupt(Object => :never) do
+                cancel_and_hard_join(worker, cancel, join_interval)
+              end
+            end
+          end
+
+          status, value = begin
+            outcome.pop(true)
+          rescue ThreadError
+            raise missing_outcome
+          end
+          raise value if status == :raised
+
+          value
+        end
+
+        # Cancellation can arrive before the worker enters the foreign call.
+        # Retry until it exits, suppressing every secondary exception so the
+        # first caller exception remains intact.
+        def cancel_and_hard_join(worker, cancel, join_interval)
+          loop do
+            begin
+              return if worker.join(0)
+            rescue Exception # rubocop:disable Lint/RescueException -- preserve the first caller exception
+              nil
+            end
+            begin
+              cancel.call
+            rescue Exception # rubocop:disable Lint/RescueException -- preserve the first caller exception
+              nil
+            end
+            begin
+              return if worker.join(join_interval)
+            rescue Exception # rubocop:disable Lint/RescueException -- preserve the first caller exception
+              nil
+            end
+          end
+        end
+        private_class_method :cancel_and_hard_join
+      end
+    end
+  end
+end

data/lib/cohere/transcribe/internal/session_ownership.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative "../errors"
+
+module Cohere
+  module Transcribe
+    module Internal
+      # Detached, thread-safe session state for explicit close and ObjectSpace
+      # finalization. The close callable must not retain the owner whose
+      # finalizer holds this object.
+      class SessionOwnership
+        CLOSE_RUBY_SESSION = :close.to_proc.freeze
+
+        def self.finalizer_for(ownership)
+          proc { |_object_id| ownership.finalize }
+        end
+
+        def initialize(close: CLOSE_RUBY_SESSION, installed_error: "Session ownership is already installed")
+          raise ArgumentError, "close must respond to call" unless close.respond_to?(:call)
+
+          @close = close
+          @installed_error = installed_error.freeze
+          @mutex = Mutex.new
+          @session = nil
+        end
+
+        def session
+          @mutex.synchronize { @session }
+        end
+
+        def install(session)
+          @mutex.synchronize do
+            raise TranscriptionRuntimeError, @installed_error if @session
+
+            @session = session
+          end
+        end
+
+        def close
+          Thread.handle_interrupt(Object => :never) do
+            session = @mutex.synchronize do
+              current = @session
+              @session = nil
+              current
+            end
+            return unless session
+
+            @close.call(session)
+          end
+          nil
+        end
+
+        def finalize
+          close
+        rescue Exception # rubocop:disable Lint/RescueException -- finalizers must not escape during GC or shutdown
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/cohere/transcribe/internal/utf8.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Cohere
+  module Transcribe
+    module Internal
+      # Normalizes path/reference bytes without applying locale-dependent
+      # transcoding. Callers retain control over their public error type and
+      # message when the bytes are not valid UTF-8.
+      module UTF8
+        module_function
+
+        def normalize(value)
+          text = value.b.force_encoding(Encoding::UTF_8)
+          text if text.valid_encoding?
+        end
+      end
+    end
+  end
+end

data/lib/cohere/transcribe/model_identity.rb

@@ -2,6 +2,7 @@
 
 require "json"
 require_relative "constants"
+require_relative "internal/utf8"
 
 module Cohere
   module Transcribe
@@ -42,7 +43,7 @@ module Cohere
           raise ArgumentError,
                 "Cannot resolve #{description.downcase} path #{reference.inspect}: #{e.message}"
         end
-        return path.realpath.to_s if path.directory?
+        return utf8_resolved_path(path.realpath.to_s, description) if path.directory?
         if path.exist? || path.symlink?
           raise ArgumentError,
                 "#{description} path #{reference.inspect} is not a directory"
@@ -59,6 +60,14 @@ module Cohere
         raise ArgumentError, "Cannot resolve #{description.downcase} path #{reference.inspect}: #{e.message}"
       end
 
+      def utf8_resolved_path(value, description)
+        text = Internal::UTF8.normalize(value)
+        return text if text
+
+        raise ArgumentError, "#{description} resolved path must contain valid UTF-8"
+      end
+      private_class_method :utf8_resolved_path
+
       # Path.expanduser() expands only the leading home-directory component;
       # unlike Pathname#expand_path, it does not lexically erase a missing
       # component followed by `..` before the filesystem probe.

data/lib/cohere/transcribe/output/publication.rb

@@ -269,7 +269,7 @@ module Cohere
           plan.directory_bindings.each(&:verify!) if plan&.paths&.any?
         end
 
-        def write(plan, result, options, generation_id: nil, speech_spans: nil)
+        def write(plan, result, options, generation_id: nil, speech_spans: nil, lock: nil)
           return [].freeze if plan.paths.empty?
 
           contents = plan.paths.to_h do |format, _path|
@@ -291,7 +291,8 @@ module Cohere
             transaction_paths,
             transaction_contents,
             before_commit: -> { State.ensure_source_unchanged!(snapshot) },
-            directory_bindings: plan.directory_bindings
+            directory_bindings: plan.directory_bindings,
+            commit_guard: lock&.method(:verify!)
           )
           plan.paths.values.freeze
         end
@@ -844,13 +845,15 @@ module Cohere
         private_class_method :quantile
 
         def atomic_write_set(paths, contents, before_commit: nil, rename: nil,
-                             directory_bindings: nil, operation_hook: nil)
+                             directory_bindings: nil, operation_hook: nil, commit_guard: nil)
           staged = {}
           backups = {}
           committed = []
           preserved_backups = {}
           open_handles = []
-          failed = false
+          completed = false
+          primary_error = nil
+          recovery_errors = []
           bound_directories = publication_bound_directories(paths, directory_bindings)
           operation_hook&.call(:directories_opened, nil)
           paths.each do |format, supplied_destination|
@@ -900,90 +903,126 @@ module Cohere
               backups[destination] = [bound, backup_name].freeze
               open_handles << backup
             end
+            backup_copy_error = nil
             begin
               backup.chmod(source.stat.mode & 0o7777)
               operation_hook&.call(:before_backup_copy, destination)
               IO.copy_stream(source, backup)
               backup.flush
               backup.fsync
+            rescue Exception => e # rubocop:disable Lint/RescueException -- retain the copy failure through close
+              backup_copy_error = e
+              raise
             ensure
-              source.close
-              backup.close
+              close_failures = State.close_atomic_resources(
+                [source, backup],
+                recovery_errors,
+                label: ->(handle) { handle.respond_to?(:path) ? handle.path : destination }
+              )
+              raise close_failures.first if close_failures.any? && backup_copy_error.nil?
             end
             bound.verify!
           end
           before_commit&.call
           bound_directories.each_value(&:verify!)
+          Thread.handle_interrupt(State::DEFERRED_PUBLICATION_EXCEPTIONS) { commit_guard&.call }
           staged.each do |destination, (bound, temporary_name)|
             operation_hook&.call(:before_rename, destination)
             if rename
+              # The caller-supplied operation may block. Record its rollback
+              # responsibility first, then leave it interruptible; if it is
+              # interrupted before or after moving the entry, the same backup
+              # restoration path is safe in both cases.
+              Thread.handle_interrupt(State::DEFERRED_PUBLICATION_EXCEPTIONS) { commit_guard&.call }
+              committed << destination
+              rename.call(bound.display_path(temporary_name), destination)
               Thread.handle_interrupt(State::DEFERRED_PUBLICATION_EXCEPTIONS) do
-                committed << destination
-                rename.call(bound.display_path(temporary_name), destination)
                 bound.rename(temporary_name, destination.basename.to_s) if bound.regular_entry?(temporary_name)
+                commit_guard&.call
               end
             else
               Thread.handle_interrupt(State::DEFERRED_PUBLICATION_EXCEPTIONS) do
+                commit_guard&.call
                 bound.rename(temporary_name, destination.basename.to_s)
                 committed << destination
               end
             end
             operation_hook&.call(:after_rename, destination)
             bound.verify!
+            Thread.handle_interrupt(State::DEFERRED_PUBLICATION_EXCEPTIONS) { commit_guard&.call }
           end
-          bound_directories.each_value(&:fsync)
-          bound_directories.each_value(&:verify!)
-        rescue Exception => e # rubocop:disable Lint/RescueException -- rollback must include interrupts
-          failed = true
-          rollback_errors = []
-          committed.reverse_each do |destination|
-            backup = backups[destination]
-            bound, backup_name = backup
-            bound ||= staged.fetch(destination).first
-            begin
-              Thread.handle_interrupt(State::DEFERRED_PUBLICATION_EXCEPTIONS) do
-                bound.unlink(destination.basename.to_s, missing_ok: true)
-                bound.rename(backup_name, destination.basename.to_s) if backup_name
-              end
-            rescue SystemCallError, TranscriptionRuntimeError => rollback_error
-              preserved_backups[backup] = true if backup_name
-              rollback_errors << "#{destination}: #{rollback_error.message}"
+          bound_directories.each_value do |bound|
+            Thread.handle_interrupt(State::DEFERRED_PUBLICATION_EXCEPTIONS) do
+              commit_guard&.call
+              bound.fsync
+              commit_guard&.call
             end
+            bound.verify!
           end
-          begin
-            bound_directories&.each_value(&:fsync)
-          rescue SystemCallError, TranscriptionRuntimeError => rollback_error
-            rollback_errors << "directory sync: #{rollback_error.message}"
-          end
-          if rollback_errors.any?
-            detail = rollback_errors.join("; ")
-            retained = preserved_backups.keys.filter_map do |backup|
-              backup&.then { |bound, name| bound.display_path(name).to_s }
-            end.sort
-            raise TranscriptionRuntimeError,
-                  "Output commit failed and rollback was incomplete (#{detail}); " \
-                  "preserved backups: #{retained}",
-                  cause: e
+          Thread.handle_interrupt(State::DEFERRED_PUBLICATION_EXCEPTIONS) do
+            commit_guard&.call
+            completed = true
           end
-          raise e
+        rescue Exception => e # rubocop:disable Lint/RescueException -- rollback must include interrupts
+          primary_error = e
+          raise
         ensure
-          cleanup_errors = []
-          staged&.each_value do |bound, name|
-            bound.unlink(name, missing_ok: true)
-          rescue SystemCallError, TranscriptionRuntimeError => e
-            cleanup_errors << e
-          end
-          backups&.each_value do |backup|
-            next unless backup && !preserved_backups&.key?(backup)
-
-            bound, name = backup
-            bound.unlink(name, missing_ok: true)
-          rescue SystemCallError, TranscriptionRuntimeError => e
-            cleanup_errors << e
+          Thread.handle_interrupt(State::DEFERRED_PUBLICATION_EXCEPTIONS) do
+            unless completed
+              committed.reverse_each do |destination|
+                backup = backups[destination]
+                bound, backup_name = backup
+                bound ||= staged.fetch(destination).first
+                begin
+                  bound.unlink(destination.basename.to_s, missing_ok: true)
+                  bound.rename(backup_name, destination.basename.to_s) if backup_name
+                  backups[destination] = nil
+                rescue SystemCallError, TranscriptionRuntimeError => e
+                  preserved_backups[backup] = true if backup_name
+                  recovery_errors << "#{destination}: #{e.message}"
+                end
+              end
+              begin
+                bound_directories&.each_value(&:fsync)
+              rescue SystemCallError, TranscriptionRuntimeError => e
+                recovery_errors << "directory sync: #{e.message}"
+              end
+            end
+            staged&.each_value do |bound, name|
+              bound.unlink(name, missing_ok: true)
+            rescue SystemCallError, TranscriptionRuntimeError => e
+              recovery_errors << "cleanup #{bound.display_path(name)}: #{e.message}"
+            end
+            backups&.each_value do |backup|
+              next unless backup && !preserved_backups&.key?(backup)
+
+              bound, name = backup
+              bound.unlink(name, missing_ok: true)
+            rescue SystemCallError, TranscriptionRuntimeError => e
+              preserved_backups[backup] = true
+              recovery_errors << "cleanup #{bound.display_path(name)}: #{e.message}"
+            end
+            State.close_atomic_resources(
+              open_handles,
+              recovery_errors,
+              label: ->(handle) { handle.respond_to?(:path) ? handle.path : handle.inspect }
+            )
+            State.close_atomic_resources(
+              bound_directories&.values,
+              recovery_errors,
+              label: ->(bound) { bound.binding.canonical_path }
+            )
+            retained = preserved_backups.keys.filter_map do |backup|
+              backup&.then { |bound, name| bound.display_path(name) }
+            end
+            State.finish_atomic_recovery!(
+              subject: "Output commit",
+              completed: completed,
+              primary_error: primary_error,
+              recovery_errors: recovery_errors,
+              retained_backups: retained
+            )
           end
-          open_handles&.each { |handle| handle.close unless handle.closed? }
-          bound_directories&.each_value(&:close)
-          raise cleanup_errors.first if cleanup_errors.any? && !failed
         end
 
         def publication_bound_directories(paths, directory_bindings)
@@ -1021,36 +1060,6 @@ module Cohere
         end
         private_class_method :bound_output_mode
 
-        def output_mode(destination)
-          return destination.stat.mode & 0o7777 if destination.exist?
-
-          current_umask = File.umask
-          File.umask(current_umask)
-          0o666 & ~current_umask
-        end
-        private_class_method :output_mode
-
-        def fsync_directories(directories)
-          directories.each do |directory|
-            File.open(directory, File::RDONLY, &:fsync)
-          rescue Errno::EACCES, Errno::EBADF, Errno::EINVAL, Errno::EISDIR,
-                 Errno::ENOTSUP, Errno::EPERM
-            next
-          end
-        end
-        private_class_method :fsync_directories
-
-        def verified_publication?(source_snapshot, paths, state_path, options)
-          State.verify_published_outputs(
-            source_snapshot: source_snapshot,
-            output_paths: paths,
-            state_path: state_path,
-            asr_contract_key: State.asr_contract_key(options),
-            render_contract_key: State.render_contract_key(options)
-          ).verified?
-        end
-        private_class_method :verified_publication?
-
         def source_record(path)
           State::SourceSnapshot.capture(path)
         end

data/lib/cohere/transcribe/pytorch_checkpoint.rb

@@ -184,8 +184,8 @@ module Cohere
                          total_entries == ZIP64_UINT16_MARKER ||
                          central_size == ZIP64_UINT32_MARKER ||
                          central_offset == ZIP64_UINT32_MARKER
-          total_entries, central_size, central_offset = zip64_directory(eocd_offset) if
-            zip64_marker && zip64_locator_present?(eocd_offset)
+          zip64_values = zip64_directory(eocd_offset) if zip64_marker
+          total_entries, central_size, central_offset = zip64_values if zip64_values
           if total_entries > ZIP_ENTRY_LIMIT || central_size > ZIP_CENTRAL_LIMIT ||
              central_size > size || central_offset > size - central_size
             raise Error, "PyTorch archive #{path} has invalid central-directory bounds"
@@ -204,11 +204,12 @@ module Cohere
         end
 
         def zip64_directory(eocd_offset)
-          raise Error, "PyTorch archive #{path} has no ZIP64 locator" if eocd_offset < 20
+          return if eocd_offset < 20
 
           locator = read_range(eocd_offset - 20, 20)
           signature, disk, record_offset, disks = locator.unpack("VVQ<V")
-          raise Error, "PyTorch archive #{path} has an invalid ZIP64 locator" unless signature == ZIP64_LOCATOR && disk.zero? && disks == 1
+          return unless signature == ZIP64_LOCATOR
+          raise Error, "PyTorch archive #{path} has an invalid ZIP64 locator" unless disk.zero? && disks == 1
 
           record = read_range(record_offset, 56)
           signature, record_size, _made, _needed, disk, central_disk,
@@ -221,12 +222,6 @@ module Cohere
           [total_entries, central_size, central_offset]
         end
 
-        def zip64_locator_present?(eocd_offset)
-          return false if eocd_offset < 20
-
-          read_range(eocd_offset - 20, 4).unpack1("V") == ZIP64_LOCATOR
-        end
-
         def parse_central_directory(bytes, expected_entries:, archive_size:, data_limit:)
           offset = 0
           result = {}
@@ -783,7 +778,6 @@ module Cohere
           @path = Pathname(path).expand_path
           @storages = {}
           @storage_payloads = {}
-          @verified_storage_payloads = {}
           @storage_verification_mutex = Mutex.new
           @tensors = read_checkpoint.freeze
         end
@@ -812,20 +806,22 @@ module Cohere
           end
           raise ArgumentError, "chunk_bytes must be positive" unless chunk_bytes.positive?
 
-          verify_storage_payload!(tensor)
-
           source_width = Safetensors::DTYPE_BYTES.fetch(tensor.dtype)
           elements_per_chunk = [chunk_bytes / source_width, 1].max
           bytes_per_chunk = elements_per_chunk * source_width
           unless contiguous?(tensor)
+            verify_storage_payload!(tensor)
             return write_strided_tensor(
               tensor, output, target_dtype: target_dtype, converter: converter,
                               elements_per_chunk: elements_per_chunk, source_width: source_width
             )
           end
 
+          inline_payload = inline_storage_verification(tensor)
+          verify_storage_payload!(tensor) unless inline_payload
           remaining = tensor.nbytes
           written = 0
+          crc32 = 0 if inline_payload
           File.open(path, "rb") do |source|
             source.seek(tensor.data_start)
             while remaining.positive?
@@ -833,16 +829,18 @@ module Cohere
               chunk = source.read(requested)
               raise Error, "Unexpected end of #{path} while reading #{tensor.name.inspect}" if chunk.nil? || chunk.bytesize != requested
 
+              crc32 = Zlib.crc32(chunk, crc32) if inline_payload
               converted = converter.convert(chunk, from: tensor.dtype, to: target_dtype)
               output.write(converted)
               written += converted.bytesize
               remaining -= requested
             end
           end
+          mark_inline_storage_verified!(tensor.storage_key, inline_payload, crc32) if inline_payload
           expected = tensor.element_count * Safetensors::DTYPE_BYTES.fetch(target_dtype)
-          return written if written == expected
+          raise Error, "Converted tensor #{tensor.name.inspect} wrote #{written} bytes; expected #{expected}" unless written == expected
 
-          raise Error, "Converted tensor #{tensor.name.inspect} wrote #{written} bytes; expected #{expected}"
+          written
         end
 
         private
@@ -1036,14 +1034,28 @@ module Cohere
         end
 
         def verify_storage_payload!(tensor)
-          payload = @storage_payloads[tensor.storage_key]
-          return unless payload
-
           @storage_verification_mutex.synchronize do
-            return if @verified_storage_payloads.key?(tensor.storage_key)
+            payload = @storage_payloads[tensor.storage_key]
+            return unless payload
 
             verify_storage_crc!(tensor.storage_key, payload)
-            @verified_storage_payloads[tensor.storage_key] = true
+            @storage_payloads.delete(tensor.storage_key)
+          end
+        end
+
+        def inline_storage_verification(tensor)
+          payload = @storage_verification_mutex.synchronize do
+            @storage_payloads[tensor.storage_key]
+          end
+          return unless payload && tensor.data_start == payload.data_offset && tensor.nbytes == payload.size
+
+          payload
+        end
+
+        def mark_inline_storage_verified!(storage_key, payload, crc32)
+          verify_storage_crc_value!(storage_key, payload, crc32)
+          @storage_verification_mutex.synchronize do
+            @storage_payloads.delete(storage_key) if @storage_payloads[storage_key].equal?(payload)
           end
         end
 
@@ -1063,11 +1075,15 @@ module Cohere
               remaining -= requested
             end
           end
+          verify_storage_crc_value!(storage_key, payload, crc32)
+        rescue Errno::ENOENT, Errno::EACCES => e
+          raise Error, "Cannot read PyTorch checkpoint #{path}: #{e.message}"
+        end
+
+        def verify_storage_crc_value!(storage_key, payload, crc32)
           return if crc32 == payload.crc32
 
           raise Error, "PyTorch tensor storage #{storage_key.inspect} failed its CRC check"
-        rescue Errno::ENOENT, Errno::EACCES => e
-          raise Error, "Cannot read PyTorch checkpoint #{path}: #{e.message}"
         end
 
         # General strided tensors occur in legitimate torch state dictionaries