cohere-transcribe 0.1.0

data/lib/cohere/transcribe/runtime/model_provider.rb

@@ -0,0 +1,390 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+require "tempfile"
+
+module Cohere
+  module Transcribe
+    module Runtime
+      # Resolves native Transformers Dense checkpoints, downloads only their
+      # required artifacts, converts them once to CrispASR GGUF, and retains no
+      # Python-side state.
+      class ModelProvider
+        # Increment whenever conversion semantics or the native tensor types
+        # written to GGUF change. Version 2 adds first-class BF16 artifacts.
+        CACHE_LAYOUT_VERSION = 2
+
+        attr_reader :cache_dir, :hub
+
+        def initialize(hub: Hub.new, cache_dir: nil, native_session_class: nil,
+                       converter: nil)
+          @hub = hub
+          root = cache_dir || ENV["COHERE_TRANSCRIBE_CACHE"] ||
+                 File.join(ENV.fetch("XDG_CACHE_HOME", File.expand_path("~/.cache")), "cohere-transcribe")
+          @cache_dir = Pathname(root).expand_path
+          @native_session_class = native_session_class
+          @converter = converter
+        end
+
+        # Planning may discover that every requested output can be resumed from
+        # a checkpoint. Defer multi-gigabyte weight discovery until #open unless
+        # a caller explicitly asks to validate inference artifacts now.
+        def resolve(options, verify_model_weights: false)
+          identity = ModelIdentity.resolve(
+            path_text(options.model),
+            options.model_revision,
+            path_text(options.adapter),
+            options.adapter_revision,
+            hub: hub,
+            verify_weight_artifacts: verify_model_weights
+          )
+          unless identity.model_format == :dense
+            raise TranscriptionConfigurationError,
+                  "Saved #{identity.model_format} checkpoints are outside the core Dense Ruby inference path"
+          end
+          if identity.adapter_id
+            raise TranscriptionConfigurationError,
+                  "PEFT/LoRA adapters are not supported by the native Ruby Dense runtime"
+          end
+
+          identity
+        end
+
+        def open(identity, options)
+          ModelIdentity.verify_model_weight_artifacts(identity, hub: hub)
+          path = converted_model_path(identity, options)
+          (@native_session_class || ASR::NativeSession).new(path, options)
+        rescue ArgumentError, TypeError, Hub::Error => e
+          raise TranscriptionRuntimeError, e.message
+        end
+
+        def converted_model_path(identity, options)
+          model_directory = materialize_source(identity)
+          output_type = { "fp32" => :f32, "bf16" => :bf16 }.fetch(options.dtype, :f16)
+          fingerprint = source_fingerprint(identity, model_directory, output_type)
+          directory = conversion_cache_directory
+          output = directory.join("#{fingerprint}-#{output_type}.gguf")
+          return output if valid_gguf?(output, fingerprint: fingerprint, output_type: output_type)
+
+          lock_path = Pathname("#{output}.lock")
+          open_conversion_lock(lock_path) do |lock|
+            raise TranscriptionRuntimeError, "Cannot acquire Dense conversion lock for #{output}" unless lock.flock(File::LOCK_EX)
+            return output if valid_gguf?(output, fingerprint: fingerprint, output_type: output_type)
+
+            cleanup_conversion_temporaries(output)
+            remove_cache_entry(output)
+            remove_cache_entry(conversion_marker(output))
+            begin
+              converter.convert(
+                model_dir: model_directory,
+                output_path: output,
+                output_type: output_type,
+                overwrite: false,
+                fsync: true
+              )
+              unless source_fingerprint(identity, model_directory, output_type) == fingerprint
+                raise TranscriptionRuntimeError,
+                      "Dense checkpoint changed during Dense conversion; retry with a stable model directory"
+              end
+              write_conversion_marker(output, fingerprint: fingerprint, output_type: output_type)
+            rescue Exception # rubocop:disable Lint/RescueException -- clean partial artifacts before propagating interrupts
+              remove_cache_entry(output)
+              remove_cache_entry(conversion_marker(output))
+              raise
+            end
+          end
+          unless valid_gguf?(output, fingerprint: fingerprint, output_type: output_type)
+            raise TranscriptionRuntimeError,
+                  "Dense conversion did not produce a valid GGUF file at #{output}"
+          end
+
+          output
+        rescue DenseConverter::Error, Safetensors::Error, PyTorchCheckpoint::Error, GGUF::Error => e
+          raise TranscriptionRuntimeError, "Cannot convert Dense Cohere checkpoint: #{e.message}"
+        rescue Hub::Error => e
+          raise TranscriptionRuntimeError, e.message
+        rescue SystemCallError => e
+          raise TranscriptionRuntimeError, "Cannot prepare native Dense model: #{e.message}"
+        end
+
+        def materialize_source(identity)
+          local = Pathname(identity.model_id).expand_path
+          return local.realpath if local.directory?
+
+          revision = identity.model_revision
+          raise TranscriptionRuntimeError, "Remote model identity has no immutable revision" unless revision
+
+          DenseConverter::REQUIRED_ARTIFACT_FILENAMES.each do |filename|
+            hub.download(identity.model_id, filename, revision: revision)
+          end
+          snapshot = hub.snapshot_path(identity.model_id, revision)
+          return snapshot if hub.cached_file(identity.model_id, "model.safetensors", revision: revision)
+          return snapshot if hub.cached_file(identity.model_id, "pytorch_model.bin", revision: revision)
+
+          safetensors_index = hub.cached_file(
+            identity.model_id, "model.safetensors.index.json", revision: revision
+          )
+          if safetensors_index
+            download_index_shards(identity, safetensors_index, extension: ".safetensors", label: "Safetensors")
+            return snapshot
+          end
+          pytorch_index = hub.cached_file(
+            identity.model_id, "pytorch_model.bin.index.json", revision: revision
+          )
+          if pytorch_index
+            download_index_shards(identity, pytorch_index, extension: ".bin", label: "PyTorch")
+            return snapshot
+          end
+
+          files = hub.list_files(identity.model_id, revision: revision)
+          if files.include?("model.safetensors")
+            hub.download(identity.model_id, "model.safetensors", revision: revision)
+          elsif files.include?("model.safetensors.index.json")
+            index = hub.download(identity.model_id, "model.safetensors.index.json", revision: revision)
+            download_index_shards(identity, index, extension: ".safetensors", label: "Safetensors")
+          elsif files.include?("pytorch_model.bin")
+            hub.download(identity.model_id, "pytorch_model.bin", revision: revision)
+          elsif files.include?("pytorch_model.bin.index.json")
+            index = hub.download(identity.model_id, "pytorch_model.bin.index.json", revision: revision)
+            download_index_shards(identity, index, extension: ".bin", label: "PyTorch")
+          else
+            raise TranscriptionRuntimeError,
+                  "#{identity.model_id}@#{revision} has no supported Dense Safetensors or PyTorch weights"
+          end
+          snapshot
+        end
+
+        private
+
+        def converter
+          @converter ||= DenseConverter
+        end
+
+        def conversion_cache_directory
+          FileUtils.mkdir_p(cache_dir)
+          cache_root = cache_dir.realpath
+          directory = cache_dir.join("dense-v#{CACHE_LAYOUT_VERSION}")
+          begin
+            Dir.mkdir(directory, 0o700)
+          rescue Errno::EEXIST
+            nil
+          end
+          stat = directory.lstat
+          unless stat.directory? && !stat.symlink? && directory.parent.realpath == cache_root
+            raise TranscriptionRuntimeError,
+                  "Dense conversion cache directory is not a regular directory: #{directory}"
+          end
+
+          directory
+        rescue TranscriptionRuntimeError
+          raise
+        rescue SystemCallError => e
+          raise TranscriptionRuntimeError,
+                "Cannot prepare Dense conversion cache directory: #{e.message}"
+        end
+
+        def open_conversion_lock(path)
+          flags = File::RDWR | File::CREAT
+          flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
+          flags |= File::CLOEXEC if defined?(File::CLOEXEC)
+          descriptor = ::IO.sysopen(path.to_s, flags, 0o600)
+          lock = File.new(descriptor, "r+", autoclose: true)
+          descriptor = nil
+          opened = lock.stat
+          current = path.lstat
+          unless opened.file? && !current.symlink? && opened.dev == current.dev && opened.ino == current.ino
+            raise TranscriptionRuntimeError,
+                  "Dense conversion lock changed while it was being opened or is not regular: #{path}"
+          end
+
+          yield lock
+        rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO => e
+          raise TranscriptionRuntimeError, "Dense conversion lock is not a regular file: #{path}", cause: e
+        ensure
+          lock&.close
+          ::IO.new(descriptor).close if descriptor
+        end
+
+        def path_text(value)
+          value.respond_to?(:to_path) ? value.to_path : value
+        end
+
+        def download_index_shards(identity, index_path, extension:, label:)
+          payload = JSON.parse(Pathname(index_path).read(encoding: "UTF-8"))
+          weights = payload.is_a?(Hash) ? payload["weight_map"] : nil
+          unless weights.is_a?(Hash) && !weights.empty? &&
+                 weights.all? { |name, filename| name.is_a?(String) && !name.empty? && filename.is_a?(String) }
+            raise TranscriptionRuntimeError, "#{label} index #{index_path} has an invalid weight_map"
+          end
+
+          filenames = weights.values.uniq
+          filenames.each do |filename|
+            if filename.empty? || filename.include?("\0")
+              raise TranscriptionRuntimeError, "#{label} index contains an invalid shard name: #{filename.inspect}"
+            end
+
+            path = Pathname(filename)
+            unless !path.absolute? &&
+                   path.each_filename.none? { |part| part == ".." } && filename.end_with?(extension)
+              raise TranscriptionRuntimeError, "#{label} index contains an invalid shard name: #{filename.inspect}"
+            end
+
+            hub.download(identity.model_id, filename, revision: identity.model_revision)
+          end
+        rescue JSON::ParserError, EncodingError => e
+          raise TranscriptionRuntimeError, "Cannot parse #{label} index #{index_path}: #{e.message}"
+        end
+
+        def source_fingerprint(identity, directory, output_type)
+          digest = Digest::SHA256.new
+          digest << "cohere-transcribe-dense\0#{CACHE_LAYOUT_VERSION}\0#{output_type}\0"
+          digest << identity.model_id.to_s << "\0" << identity.model_revision.to_s << "\0"
+          source_files(directory).each do |path|
+            stat = path.stat
+            digest << path.relative_path_from(directory).to_s << "\0"
+            digest << stat.size.to_s << "\0" << stat.mtime.to_r.to_s << "\0"
+            # A same-size rewrite can deliberately preserve mtime. ctime is
+            # kernel-managed and still invalidates that stale conversion.
+            digest << stat.ctime.to_r.to_s << "\0"
+          end
+          digest.hexdigest
+        end
+
+        def source_files(directory)
+          files = DenseConverter::REQUIRED_ARTIFACT_FILENAMES.map { |name| directory.join(name) }
+          single = directory.join("model.safetensors")
+          if single.file?
+            files << single
+          elsif directory.join("model.safetensors.index.json").file?
+            index = directory.join("model.safetensors.index.json")
+            files << index
+            payload = JSON.parse(index.read(encoding: "UTF-8"))
+            weights = payload.is_a?(Hash) ? payload["weight_map"] : nil
+            unless weights.is_a?(Hash) && !weights.empty? &&
+                   weights.all? { |name, file| name.is_a?(String) && !name.empty? && file.is_a?(String) }
+              raise TranscriptionRuntimeError,
+                    "Cannot fingerprint Dense checkpoint: " \
+                    "Safetensors index #{index} has an invalid weight_map"
+            end
+
+            shard_names = weights.values.uniq
+            shard_names.each do |name|
+              next if valid_safetensors_shard_name?(name)
+
+              raise TranscriptionRuntimeError,
+                    "Cannot fingerprint Dense checkpoint: " \
+                    "Safetensors index #{index} contains an invalid shard path #{name.inspect}"
+            end
+            shard_names.sort.each { |name| files << directory.join(name) }
+          elsif directory.join("pytorch_model.bin").file?
+            files << directory.join("pytorch_model.bin")
+          else
+            index = directory.join("pytorch_model.bin.index.json")
+            files << index
+            tensor_set = PyTorchCheckpoint::TensorSet.from_directory(directory)
+            files.concat(tensor_set.readers.map(&:path))
+          end
+          missing = files.reject(&:file?)
+          raise TranscriptionRuntimeError, "Dense checkpoint is missing: #{missing.join(", ")}" unless missing.empty?
+
+          files.sort_by(&:to_s)
+        rescue JSON::ParserError, KeyError => e
+          raise TranscriptionRuntimeError, "Cannot fingerprint Dense checkpoint: #{e.message}"
+        end
+
+        def valid_safetensors_shard_name?(name)
+          return false if name.empty? || name.include?("\0")
+
+          candidate = Pathname(name)
+          !candidate.absolute? && candidate.each_filename.none? { |part| part == ".." }
+        end
+
+        def conversion_marker(path)
+          Pathname("#{path}.complete.json")
+        end
+
+        def remove_cache_entry(path)
+          path.delete
+        rescue Errno::ENOENT
+          nil
+        end
+
+        def cleanup_conversion_temporaries(output)
+          [output, conversion_marker(output)].each do |target|
+            pattern = target.dirname.join(".#{target.basename}*.tmp")
+            Dir.glob(pattern.to_s).each do |path|
+              File.unlink(path)
+            rescue Errno::ENOENT
+              nil
+            end
+          end
+        end
+
+        def valid_gguf?(path, fingerprint:, output_type:)
+          stat = path.lstat
+          return false unless stat.file? && !stat.symlink? && stat.size > 32
+          return false unless path.open("rb") { |file| file.read(4) == "GGUF" }
+
+          marker = conversion_marker(path)
+          marker_stat = marker.lstat
+          return false unless marker_stat.file? && !marker_stat.symlink? && marker_stat.size.between?(1, 16_384)
+
+          payload = JSON.parse(marker.read(encoding: "UTF-8"))
+          payload == conversion_marker_payload(
+            stat,
+            fingerprint: fingerprint,
+            output_type: output_type
+          )
+        rescue JSON::ParserError, EncodingError, SystemCallError
+          false
+        end
+
+        def write_conversion_marker(path, fingerprint:, output_type:)
+          stat = path.lstat
+          unless stat.file? && !stat.symlink? && stat.size > 32 &&
+                 path.open("rb") { |file| file.read(4) == "GGUF" }
+            raise TranscriptionRuntimeError, "Dense conversion produced an invalid GGUF file at #{path}"
+          end
+
+          marker = conversion_marker(path)
+          payload = conversion_marker_payload(
+            stat,
+            fingerprint: fingerprint,
+            output_type: output_type
+          )
+          Tempfile.create([".#{marker.basename}", ".tmp"], marker.dirname, binmode: true) do |temporary|
+            temporary.chmod(0o600)
+            temporary.write(JSON.generate(payload))
+            temporary.write("\n")
+            temporary.flush
+            temporary.fsync
+            temporary.close
+            File.rename(temporary.path, marker)
+          end
+          begin
+            File.open(marker.dirname, File::RDONLY, &:fsync)
+          rescue Errno::EINVAL, Errno::ENOTSUP, Errno::EISDIR
+            nil
+          end
+          marker
+        end
+
+        def conversion_marker_payload(stat, fingerprint:, output_type:)
+          {
+            "schema_version" => 1,
+            "cache_layout_version" => CACHE_LAYOUT_VERSION,
+            "source_fingerprint" => fingerprint,
+            "output_type" => output_type.to_s,
+            "device" => stat.dev,
+            "inode" => stat.ino,
+            "size" => stat.size,
+            "mtime_ns" => (stat.mtime.to_i * 1_000_000_000) + stat.mtime.nsec,
+            "ctime_ns" => (stat.ctime.to_i * 1_000_000_000) + stat.ctime.nsec
+          }
+        end
+      end
+    end
+  end
+end

data/lib/cohere/transcribe/runtime/precision.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "fiddle"
+
+module Cohere
+  module Transcribe
+    module Runtime
+      # Resolves public device/precision requests against the native runtime
+      # before any input is decoded. The returned options describe what will
+      # actually execute; an unavailable explicit accelerator is never allowed
+      # to fall through to ggml's command-line-oriented CPU fallback.
+      module Precision
+        module_function
+
+        def resolve(options, native_library: nil)
+          library = native_library
+          device = if options.device == "cpu"
+                     "cpu"
+                   else
+                     library ||= ASR::NativeLibrary.load
+                     library.resolve_device(options.device)
+                   end
+          dtype = resolved_dtype(options.dtype, device, library)
+          if options.alignment == "word" && options.align_dtype == "fp16" && device != "cuda"
+            raise TranscriptionRuntimeError,
+                  "--align-dtype fp16 is supported only with CUDA"
+          end
+
+          vad_engine = options.vad == "silero" ? "onnx" : options.vad_engine
+          options.with(device: device, dtype: dtype, vad_engine: vad_engine)
+        rescue TranscriptionError
+          raise
+        rescue Fiddle::DLError, LoadError, SystemCallError => e
+          raise TranscriptionRuntimeError,
+                "Cannot resolve the native inference device: #{e.class}: #{e.message}"
+        end
+
+        def resolved_dtype(requested, device, native_library)
+          return "fp32" if device == "cpu"
+
+          resolved = if requested == "auto"
+                       device == "cuda" && native_library&.supports_bf16?(device) ? "bf16" : "fp16"
+                     else
+                       requested
+                     end
+          if resolved == "bf16" && !native_library&.supports_bf16?(device)
+            label = device == "mps" ? "MPS device/runtime" : "CUDA device"
+            raise TranscriptionRuntimeError,
+                  "This #{label} does not support BF16; use --dtype fp16"
+          end
+          resolved
+        end
+        private_class_method :resolved_dtype
+      end
+    end
+  end
+end

data/lib/cohere/transcribe/runtime/preparation.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require "etc"
+
+require_relative "../audio/ffmpeg_native"
+
+module Cohere
+  module Transcribe
+    module Runtime
+      module Preparation
+        MAX_PIPELINE_GROUP_BYTES = 512 * (1024**2)
+        MAX_GROUP_JOBS = 128
+
+        # The result of decode and VAD preparation. Timings are accumulated by
+        # the engine on the consuming thread, keeping statistics deterministic.
+        PreparedEntry = Data.define(
+          :item,
+          :snapshot,
+          :decoded,
+          :duration,
+          :segment_times,
+          :speech_spans,
+          :vad_details,
+          :decode_seconds,
+          :vad_seconds,
+          :error
+        )
+
+        # Bounded ordered preparation with at most one group ahead of ASR.
+        #
+        # A pipelined group receives at most half of the configured decoded-PCM
+        # budget (and never more than 512 MiB). Jobs inside it divide that cap,
+        # so the current and one in-flight next group cannot together exceed the
+        # configured PCM budget. Native decoder implementation transients are
+        # outside this retained-PCM accounting, as they are in the Python path.
+        class Pipeline
+          include Enumerable
+
+          class WorkerPool
+            STOP = Object.new.freeze
+
+            def initialize(size, &prepare)
+              @prepare = prepare
+              @queues = Array.new(size) { Queue.new }
+              @threads = @queues.each_with_index.map do |queue, slot|
+                Thread.new do
+                  Thread.current.name = "cohere-audio-prep-#{slot}" if Thread.current.respond_to?(:name=)
+                  loop do
+                    task = queue.pop
+                    break if task.equal?(STOP)
+
+                    item, limit, response = task
+                    begin
+                      response << [:ok, @prepare.call(item, limit, slot)]
+                    rescue Exception => e # rubocop:disable Lint/RescueException -- transport fatal worker errors
+                      response << [:error, e]
+                    end
+                  end
+                end.tap { |thread| thread.report_on_exception = false }
+              end
+            end
+
+            def prepare(group, per_file_limit)
+              responses = group.each_with_index.map do |item, slot|
+                response = Queue.new
+                @queues.fetch(slot) << [item, per_file_limit, response]
+                response
+              end
+              responses.map do |response|
+                status, value = response.pop
+                raise value if status == :error
+
+                value
+              end
+            end
+
+            def close(cancel:)
+              if cancel
+                Audio::FFmpegNative.cancel_active!
+                @threads.each { |thread| thread.kill if thread.alive? }
+              else
+                @queues.each { |queue| queue << STOP }
+              end
+              @threads.each(&:join)
+            end
+          end
+          private_constant :WorkerPool
+
+          attr_reader :effective_workers, :group_byte_limit, :memory_byte_limit, :wait_seconds
+
+          def initialize(items, memory_byte_limit:, requested_workers:, enabled:, worker_limit: nil, &prepare)
+            raise ArgumentError, "prepare block is required" unless prepare
+
+            @items = items.to_a.freeze
+            @memory_byte_limit = Integer(memory_byte_limit)
+            raise ArgumentError, "memory_byte_limit must be positive" unless @memory_byte_limit.positive?
+
+            @prepare = prepare
+            @wait_seconds = 0.0
+            @enabled = enabled && @items.length > 1
+            @worker_limit = worker_limit.nil? ? nil : Integer(worker_limit)
+            raise ArgumentError, "worker_limit must be positive" if @worker_limit && !@worker_limit.positive?
+
+            @group_byte_limit = if @enabled
+                                  [[@memory_byte_limit / 2, 1].max, MAX_PIPELINE_GROUP_BYTES].min
+                                else
+                                  @memory_byte_limit
+                                end
+            @effective_workers = resolve_workers(requested_workers)
+          end
+
+          def pipelined?
+            @enabled
+          end
+
+          def each(&block)
+            return enum_for(__method__) unless block
+            return if @items.empty?
+
+            unless pipelined?
+              @items.each { |item| block.call(@prepare.call(item, @memory_byte_limit, 0)) }
+              return
+            end
+
+            groups = @items.each_slice(@effective_workers).to_a
+            pool = WorkerPool.new(@effective_workers, &@prepare)
+            pending = submit(groups.first, 0, pool)
+            groups.each_with_index do |_group, group_index|
+              prepared = resolve(pending)
+              pending = nil
+              pending = submit(groups.fetch(group_index + 1), group_index + 1, pool) if group_index + 1 < groups.length
+              consume(prepared, &block)
+            end
+            completed = true
+          ensure
+            cancel(pending) if defined?(pending) && pending
+            pool&.close(cancel: !completed) if defined?(pool) && pool
+          end
+
+          private
+
+          def resolve_workers(requested)
+            return 1 if @items.empty?
+
+            processors = Etc.nprocessors
+            processors = 1 unless processors.is_a?(Integer) && processors.positive?
+            automatic = if @items.length == 1
+                          1
+                        else
+                          [2, @items.length, [processors / 2, 1].max].min
+                        end
+            value = requested.nil? ? automatic : Integer(requested)
+            raise ArgumentError, "requested_workers must be positive" unless value.positive?
+
+            return 1 unless @enabled
+
+            limits = [value, @items.length, processors, MAX_GROUP_JOBS, @group_byte_limit]
+            limits << @worker_limit if @worker_limit
+            limits.min
+          rescue SystemCallError
+            value = requested.nil? ? 2 : Integer(requested)
+            limits = [value, @items.length, MAX_GROUP_JOBS, @group_byte_limit]
+            limits << @worker_limit if @worker_limit
+            limits.min
+          end
+
+          def submit(group, group_index, pool)
+            Thread.new do
+              Thread.current.name = "cohere-audio-group-#{group_index}" if Thread.current.respond_to?(:name=)
+              prepare_group(group, pool)
+            end.tap { |thread| thread.report_on_exception = false }
+          end
+
+          def resolve(thread)
+            started = monotonic
+            thread.value
+          ensure
+            @wait_seconds += monotonic - started if started
+          end
+
+          def prepare_group(group, pool)
+            per_file_limit = [@group_byte_limit / group.length, 1].max
+            pool.prepare(group, per_file_limit)
+          end
+
+          def consume(prepared, &block)
+            prepared.each_index do |index|
+              entry = prepared.fetch(index)
+              begin
+                block.call(entry)
+              ensure
+                prepared[index] = nil
+              end
+            end
+          ensure
+            prepared&.clear
+            GC.start(full_mark: false, immediate_mark: true, immediate_sweep: true)
+          end
+
+          def cancel(thread)
+            Audio::FFmpegNative.cancel_active!
+            thread.kill if thread.alive?
+            thread.join
+          rescue Exception # rubocop:disable Lint/RescueException -- preserve the caller's active exception
+            nil
+          end
+
+          def monotonic
+            Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          end
+        end
+      end
+    end
+  end
+end