cohere-transcribe 0.1.0

data/lib/cohere/transcribe/state/locking.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "tmpdir"
+
+module Cohere
+  module Transcribe
+    module State
+      LOCK_DIRECTORY_PREFIX = "cohere-transcribe"
+      LOCK_SUFFIX = ".lock"
+
+      OutputLockTarget = Data.define(:path, :identity) do
+        def sort_key
+          [path.to_s.downcase, identity.downcase]
+        end
+      end
+
+      class OutputSetLock
+        @guard = Mutex.new
+        @active = {}
+
+        class << self
+          attr_reader :guard, :active
+
+          def acquire(target, blocking: false, operation_hook: nil)
+            guard.synchronize do
+              handle = nil
+              acquired = false
+              key = target.path.to_s
+              if active.key?(key)
+                raise TranscriptionRuntimeError,
+                      "Another transcription job owns output set #{target.identity}"
+              end
+
+              handle = State.open_lock_file(target.path)
+              operation = File::LOCK_EX
+              operation |= File::LOCK_NB unless blocking
+              unless handle.flock(operation)
+                handle.close
+                raise TranscriptionRuntimeError,
+                      "Another transcription process owns output set #{target.identity} (lock #{target.path})"
+              end
+              operation_hook&.call(:after_flock, target)
+              State.verify_lock_identity!(
+                target.path,
+                handle,
+                message: "Output lock changed while acquiring #{target.identity}"
+              )
+
+              lock = new(target, handle, key)
+              active[key] = lock
+              acquired = true
+              lock
+            rescue Errno::EWOULDBLOCK, Errno::EAGAIN
+              handle&.close
+              raise TranscriptionRuntimeError,
+                    "Another transcription process owns output set #{target.identity} (lock #{target.path})"
+            rescue Interrupt, SystemExit, StandardError
+              handle&.close
+              raise
+            ensure
+              handle.close if handle && !acquired && !handle.closed?
+            end
+          end
+
+          def release_all
+            guard.synchronize { active.values.dup }.each(&:release)
+          end
+        end
+
+        attr_reader :target
+
+        def initialize(target, handle, key)
+          @target = target
+          @handle = handle
+          @key = key
+          @released = false
+        end
+
+        def release
+          self.class.guard.synchronize do
+            return if @released
+
+            begin
+              @handle.flock(File::LOCK_UN)
+            ensure
+              @handle.close
+              @released = true
+              self.class.active.delete(@key)
+            end
+          end
+          nil
+        end
+
+        def released?
+          @released
+        end
+      end
+
+      module_function
+
+      def lock_target_for_outputs(output_paths)
+        parent, stem = output_parent_and_stem(output_paths)
+        identity = parent.join(stem).expand_path.cleanpath.to_s
+        digest = Digest::SHA256.hexdigest(identity.downcase)
+        OutputLockTarget.new(
+          path: lock_registry_directory.join("#{digest}#{LOCK_SUFFIX}").freeze,
+          identity: identity.freeze
+        )
+      end
+
+      def with_output_lock(target, blocking: false)
+        lock = OutputSetLock.acquire(target, blocking: blocking)
+        yield lock
+      ensure
+        lock&.release
+      end
+
+      def lock_registry_directory
+        scope = Process.respond_to?(:uid) ? Process.uid.to_s : Digest::SHA256.hexdigest(Dir.home)[0, 16]
+        Pathname(Dir.tmpdir).join("#{LOCK_DIRECTORY_PREFIX}-#{scope}")
+      end
+
+      def open_lock_file(path)
+        path = Pathname(path)
+        succeeded = false
+        validate_lock_directory!(path.dirname)
+        inspect_lock_path!(path)
+        flags = File::RDWR | File::CREAT
+        flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
+        flags |= File::CLOEXEC if defined?(File::CLOEXEC)
+        descriptor = ::IO.sysopen(path.to_s, flags, 0o600)
+        handle = File.new(descriptor, "r+", autoclose: true)
+        descriptor = nil
+        opened = verify_lock_identity!(
+          path,
+          handle,
+          message: "Output lock changed while it was being opened or is not regular"
+        )
+        validate_owned_private_file!(opened, path)
+        succeeded = true
+        handle
+      rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO => e
+        raise TranscriptionRuntimeError, "Output lock is not a regular file: #{path}", cause: e
+      ensure
+        if !succeeded && handle
+          handle.close
+        elsif descriptor
+          ::IO.new(descriptor).close
+        end
+      end
+
+      def validate_lock_directory!(path)
+        begin
+          Dir.mkdir(path, 0o700)
+        rescue Errno::EEXIST
+          nil
+        rescue SystemCallError => e
+          raise TranscriptionRuntimeError, "Cannot prepare output lock directory #{path}: #{e.message}"
+        end
+        stat = path.lstat
+        raise TranscriptionRuntimeError, "Output lock directory is not a real directory: #{path}" unless stat.directory? && !stat.symlink?
+
+        return if Gem.win_platform?
+
+        if Process.respond_to?(:uid) && stat.uid != Process.uid
+          raise TranscriptionRuntimeError,
+                "Output lock directory is not owned by the current user: #{path}"
+        end
+        return if stat.mode.nobits?(0o077)
+
+        raise TranscriptionRuntimeError,
+              "Output lock directory permissions must be private (0700): #{path}"
+      rescue Errno::ENOENT, Errno::ENOTDIR => e
+        raise TranscriptionRuntimeError, "Cannot prepare output lock directory #{path}: #{e.message}"
+      end
+
+      def inspect_lock_path!(path)
+        stat = path.lstat
+        return if stat.file? && !stat.symlink?
+
+        raise TranscriptionRuntimeError, "Output lock is not a regular file: #{path}"
+      rescue Errno::ENOENT
+        nil
+      end
+
+      def validate_owned_private_file!(stat, path)
+        return if Gem.win_platform?
+
+        if Process.respond_to?(:uid) && stat.uid != Process.uid
+          raise TranscriptionRuntimeError, "Output lock is not owned by the current user: #{path}"
+        end
+        return if stat.mode.nobits?(0o077)
+
+        raise TranscriptionRuntimeError,
+              "Output lock permissions must be private (0600): #{path}"
+      end
+
+      def verify_lock_identity!(path, handle, message:)
+        opened = handle.stat
+        current = Pathname(path).lstat
+        return opened if opened.file? && same_file?(opened, current)
+
+        raise TranscriptionRuntimeError, "#{message}: #{path}"
+      rescue SystemCallError => e
+        raise TranscriptionRuntimeError, "#{message}: #{path}", cause: e
+      end
+    end
+  end
+end

data/lib/cohere/transcribe/state/manifest.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Cohere
+  module Transcribe
+    module State
+      PublicationVerification = Data.define(:verified, :generation_id, :reason) do
+        def verified?
+          verified
+        end
+      end
+
+      module_function
+
+      def published_manifest_content(source_snapshot:, output_paths:, contents:, asr_contract_key:,
+                                     render_contract_key:, generation_id:)
+        payload = published_payload(
+          source_snapshot: source_snapshot,
+          output_paths: output_paths,
+          contents: contents,
+          asr_contract_key: asr_contract_key,
+          render_contract_key: render_contract_key,
+          generation_id: generation_id
+        )
+        JSON.pretty_generate(envelope(payload)) << "\n"
+      end
+
+      def published_payload(source_snapshot:, output_paths:, contents:, asr_contract_key:,
+                            render_contract_key:, generation_id:)
+        raise ArgumentError, "generation ID is invalid" unless generation_id.is_a?(String) && !generation_id.empty?
+
+        {
+          "kind" => "published",
+          "generation_id" => generation_id,
+          "asr_contract_key" => asr_contract_key,
+          "render_contract_key" => render_contract_key,
+          "source" => source_snapshot.payload,
+          "updated_unix_seconds" => Time.now.to_f,
+          "outputs" => output_paths.keys.sort.to_h do |format|
+            content = contents.fetch(format)
+            [
+              format,
+              {
+                "name" => Pathname(output_paths.fetch(format)).basename.to_s,
+                "size" => content.bytesize,
+                "sha256" => Digest::SHA256.hexdigest(content)
+              }
+            ]
+          end
+        }
+      end
+
+      def verify_published_outputs(source_snapshot:, output_paths:, state_path:, asr_contract_key:,
+                                   render_contract_key:, directory_binding: nil,
+                                   guard_bindings: nil)
+        payload, reason = decode_state(
+          state_path,
+          directory_binding: directory_binding,
+          guard_bindings: guard_bindings
+        )
+        return PublicationVerification.new(verified: false, generation_id: nil, reason: reason.freeze) unless payload
+
+        mismatch = publication_mismatch_reason(
+          payload,
+          source_snapshot: source_snapshot,
+          output_paths: output_paths,
+          asr_contract_key: asr_contract_key,
+          render_contract_key: render_contract_key,
+          directory_binding: directory_binding,
+          guard_bindings: guard_bindings
+        )
+        return PublicationVerification.new(verified: false, generation_id: nil, reason: mismatch.freeze) if mismatch
+
+        PublicationVerification.new(
+          verified: true,
+          generation_id: payload.fetch("generation_id").dup.freeze,
+          reason: nil
+        )
+      rescue EncodingError, SystemCallError, TypeError, ArgumentError => e
+        PublicationVerification.new(
+          verified: false,
+          generation_id: nil,
+          reason: "cannot verify output generation (#{e.class}: #{e.message})".freeze
+        )
+      end
+
+      def publication_mismatch_reason(payload, source_snapshot:, output_paths:, asr_contract_key:,
+                                      render_contract_key:, directory_binding: nil,
+                                      guard_bindings: nil)
+        return "state is #{payload["kind"].inspect}, not published" unless payload["kind"] == "published"
+        return "state marker ASR contract does not match" unless payload["asr_contract_key"] == asr_contract_key
+        return "state marker render contract does not match" unless payload["render_contract_key"] == render_contract_key
+        return "state marker source snapshot does not match" unless payload["source"] == source_snapshot.payload
+
+        generation_id = payload["generation_id"]
+        return "state marker generation ID is invalid" unless generation_id.is_a?(String) && !generation_id.empty?
+
+        records = payload["outputs"]
+        return "state marker output formats do not match" unless records.is_a?(Hash) && records.keys.sort == output_paths.keys.sort
+
+        output_paths.each do |format, path_value|
+          path = Pathname(path_value)
+          record = records[format]
+          return "state marker path for #{format} does not match" unless record.is_a?(Hash) && record["name"] == path.basename.to_s
+
+          begin
+            size, sha256 = if directory_binding
+                             bound_output_record(
+                               path,
+                               directory_binding: directory_binding,
+                               guard_bindings: guard_bindings
+                             )
+                           else
+                             stat = path.lstat
+                             return "#{format} output is missing or not regular" unless stat.file? && !stat.symlink?
+
+                             [stat.size, Digest::SHA256.file(path).hexdigest]
+                           end
+            return "#{format} output does not match its state marker" if record["size"] != size || record["sha256"] != sha256
+          rescue Errno::ENOENT
+            return "#{format} output is missing or not regular"
+          end
+        end
+        nil
+      end
+
+      def bound_output_record(path, directory_binding:, guard_bindings:)
+        with_bound_parent(
+          path,
+          directory_binding: directory_binding,
+          guard_bindings: guard_bindings
+        ) do |bound, basename|
+          bound.verify!
+          handle = nil
+          Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
+            handle = bound.open_regular(basename)
+          end
+          opened = handle.stat
+          digest = Digest::SHA256.new
+          digest << handle.read(1024 * 1024) until handle.eof?
+          unless bound.same_regular_entry?(basename, opened)
+            raise TranscriptionRuntimeError, "Output changed while it was being verified: #{path}"
+          end
+
+          bound.verify!
+          [opened.size, digest.hexdigest]
+        ensure
+          handle&.close
+        end
+      end
+      private_class_method :bound_output_record
+    end
+  end
+end

data/lib/cohere/transcribe/state.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require_relative "state/io"
+require_relative "state/contracts"
+require_relative "state/checkpoint"
+require_relative "state/manifest"
+require_relative "state/locking"