carray-netcdf 2.0.0 → 2.1.0

data/lib/carray/netcdf/file.rb

@@ -19,16 +19,16 @@ module CA
     class File < Group
 
       # Kwargs on {.create} that become file-wide defaults for every
-      # subsequent {Group#def_var} call. See docs/writing.md for details.
+      # subsequent {Group#def_var} call. See docs/03_writing.md for details.
       # @return [Array<Symbol>]
-      FILE_VAR_DEFAULTS = %i[chunks deflate shuffle fletcher32 endian no_fill].freeze
+      FILE_VAR_DEFAULTS = VAR_OPTION_KEYS
 
       class << self
         # Open an existing NetCDF file.
         #
         # @param path [String]
         # @param mode [:read, :write, :append]
-        # @param codec [Symbol, Array, Object] see docs/codec.md
+        # @param codec [Symbol, Array, Object] see docs/04_codec.md
         # @yieldparam file [File]
         # @return [File] the file (or block's return value if a block is
         #   given; the file is closed after the block)
@@ -45,16 +45,19 @@ module CA
         # Create a new NetCDF file.
         #
         # @param path [String] final file path
-        # @param codec [Symbol, Array, Object] see docs/codec.md
+        # @param codec [Symbol, Array, Object] see docs/04_codec.md
         # @param clobber [true, false, :skip] overwrite policy when `path`
         #   already exists. `true` (default) overwrites; `false` raises
         #   `Errno::EEXIST`; `:skip` returns `nil` without calling the
         #   block — handy for resumable batches (`Dir[...]` picks up
         #   completed outputs unchanged).
-        # @param netcdf4 [Boolean] use the NetCDF-4 (HDF5) format. When a
-        #   `template:` File / Group is provided, this defaults to
-        #   whatever the template's format was, so a NetCDF-4 template
-        #   produces a NetCDF-4 output unless explicitly overridden.
+        # @param netcdf4 [Boolean, nil] use the NetCDF-4 (HDF5) format.
+        #   Left nil, it follows a `template:` File / Group's format, so a
+        #   NetCDF-4 template produces a NetCDF-4 output; `false` asks for
+        #   the classic format whatever the template.
+        # @param format [Symbol, nil] `:classic`, `:offset64` (CDF-2),
+        #   `:cdf5`, `:netcdf4` or `:netcdf4_classic`; instead of
+        #   `netcdf4:`. Left nil, as for `netcdf4:`.
         # @param atomic [Boolean] write to a same-dir tmp file, rename to
         #   `path` only on successful close; unlink the tmp on any
         #   exception from the block form. Rename is atomic on POSIX.
@@ -73,7 +76,7 @@ module CA
         #   template's atts merged with these overrides (last-writer
         #   wins).
         # @param vars [Array<String>] template variable names to copy in
-        #   full (definition AND data, bit-identical via raw + layer 1).
+        #   full (definition AND data, bit-identical via raw + the low-level API).
         #   Typical use: copy coordinate vars like `%w[time lat lon]` once
         #   per batch iteration.
         # @param var_defaults [Hash] file-wide defaults picked up by every
@@ -82,7 +85,7 @@ module CA
         # @yieldparam file [File]
         # @return [File, nil] the file (or block's return value); `nil`
         #   when `clobber: :skip` and `path` already exists.
-        def create(path, codec: :auto, clobber: true, netcdf4: false,
+        def create(path, codec: :auto, clobber: true, netcdf4: nil, format: nil,
                    atomic: false, template: nil,
                    dims: true, atts: true, vars: [],
                    **var_defaults, &block)
@@ -102,22 +105,33 @@ module CA
           if clobber == :skip && ::File.exist?(path)
             return nil
           end
-          # Format inheritance from the template unless netcdf4: is set
-          # explicitly.  A NetCDF-4 template silently upgrades the output
-          # so vars using groups / chunking / deflate stay defineable.
-          if template && !netcdf4
-            src_file = template.is_a?(File) ? template : template.file
-            # nc_inq_format returns the NC_FORMAT_* Integer constant.
-            fmt = src_file.format
-            netcdf4 = true if fmt == ::NetCDF::NC_FORMAT_NETCDF4 ||
-                              fmt == ::NetCDF::NC_FORMAT_NETCDF4_CLASSIC
+          # The format: `format:`, else `netcdf4:` (true / false), else the
+          # template's, else classic.
+          unless format.nil? || netcdf4.nil?
+            raise ArgumentError, "give format: or netcdf4:, not both"
           end
-          # Pre-flight clobber check for atomic: refuse before creating a tmp.
-          if atomic && clobber == false && ::File.exist?(path)
+          format ||= case netcdf4
+                     when true  then :netcdf4
+                     when false then :classic
+                     else
+                       if template
+                         src_file = template.is_a?(File) ? template : template.file
+                         FORMAT_NAMES.fetch(src_file.format, :classic)
+                       else
+                         :classic
+                       end
+                     end
+          format_flag = FORMAT_FLAGS.fetch(format) do
+            raise ArgumentError,
+                  "format: expected one of #{FORMAT_FLAGS.keys.inspect}, got #{format.inspect}"
+          end
+          # Pre-flight clobber check: refuse before creating anything, so an
+          # atomic and a direct write raise the same error.
+          if clobber == false && ::File.exist?(path)
             raise Errno::EEXIST, path
           end
           flag  = clobber ? ::NetCDF::NC_CLOBBER : ::NetCDF::NC_NOCLOBBER
-          flag |= ::NetCDF::NC_NETCDF4 if netcdf4
+          flag |= format_flag
           write_path = atomic ? atomic_tmp_path_for(path) : path
           after_new  = template ?
                        ->(f) { apply_template(f, template, dims: dims, atts: atts, vars: vars) } :
@@ -131,6 +145,39 @@ module CA
 
         private
 
+        # @api private
+        NC_64BIT_DATA = ::NetCDF.const_defined?(:NC_64BIT_DATA) ? ::NetCDF::NC_64BIT_DATA : 0x0020
+
+        # `format:` names -> the `nc_create` flags that make them.
+        # @api private
+        FORMAT_FLAGS = {
+          classic:         0,
+          offset64:        ::NetCDF::NC_64BIT_OFFSET,
+          cdf5:            NC_64BIT_DATA,
+          netcdf4:         ::NetCDF::NC_NETCDF4,
+          netcdf4_classic: ::NetCDF::NC_NETCDF4 | ::NetCDF::NC_CLASSIC_MODEL,
+        }.freeze
+
+        # `NC_FORMAT_*` (what {#format} returns) -> the `format:` name.
+        # @api private
+        FORMAT_NAMES = {
+          ::NetCDF::NC_FORMAT_CLASSIC         => :classic,
+          ::NetCDF::NC_FORMAT_64BIT_OFFSET    => :offset64,
+          ::NetCDF::NC_FORMAT_64BIT_DATA      => :cdf5,
+          ::NetCDF::NC_FORMAT_NETCDF4         => :netcdf4,
+          ::NetCDF::NC_FORMAT_NETCDF4_CLASSIC => :netcdf4_classic,
+        }.freeze
+
+        # The file an atomic write replaces: the target of a symlink, so
+        # the link is kept and the tmp is made beside the real file.
+        # @api private
+        def atomic_target(path)
+          return path unless ::File.symlink?(path)
+          ::File.realpath(path)
+        rescue SystemCallError
+          path                      # a dangling link is replaced as it is
+        end
+
         # libnetcdf takes a "scheme://..." string as a URL, not a path.
         def url?(path)
           path.to_s.match?(%r{\A[A-Za-z][A-Za-z0-9+.-]*://})
@@ -142,6 +189,7 @@ module CA
         # `find … -name '*.tmp'` picks up orphans. PID + random so
         # parallel writers don't collide.
         def atomic_tmp_path_for(path)
+          path = atomic_target(path)
           dir  = ::File.dirname(path)
           base = ::File.basename(path)
           ::File.join(dir, "#{base}.#{Process.pid}.#{rand(2**32).to_s(16)}.tmp")
@@ -149,7 +197,14 @@ module CA
 
         def build(ncid, filename, mode:, codec:, defining:,
                   tmp_path: nil, var_defaults: {}, after_new: nil, &block)
-          f = new(ncid, filename, mode, codec, defining, tmp_path, var_defaults)
+          begin
+            f = new(ncid, filename, mode, codec, defining, tmp_path, var_defaults)
+          rescue Exception
+            # Without a File there is nothing to close it later.
+            ::NetCDF.nc_close(ncid) rescue nil
+            ::File.unlink(tmp_path) if tmp_path && ::File.exist?(tmp_path)
+            raise
+          end
           # after_new runs BEFORE the user's block, but still inside the same
           # try/close so a failure there also aborts an atomic write.
           begin
@@ -186,15 +241,17 @@ module CA
             raise ArgumentError,
                   "File.create dims: expected true / false / Array, got #{dims.inspect}"
           end
+          # The template's attributes are copied as stored, keeping their
+          # types; Hash overrides are written by value.
           case atts
           when true
-            template.atts.each { |k, v| f.def_att(k, v) }
+            template.atts.each_key { |k| f.send(:copy_att_from, template, ::NetCDF::NC_GLOBAL, k) }
           when false, nil
             # skip
           when Hash
-            template.atts
-                    .merge(atts.transform_keys(&:to_s))
-                    .each { |k, v| f.def_att(k, v) }
+            overrides = atts.transform_keys(&:to_s)
+            (template.atts.keys - overrides.keys).each { |k| f.send(:copy_att_from, template, ::NetCDF::NC_GLOBAL, k) }
+            overrides.each { |k, v| f.def_att(k, v) }
           else
             raise ArgumentError,
                   "File.create atts: expected true / false / Hash, got #{atts.inspect}"
@@ -214,28 +271,37 @@ module CA
         @codec_request = codec
         @tmp_path      = tmp_path
         @var_defaults  = var_defaults.freeze
+        @dim_index     = {}        # dimid -> Dim, file-wide (see Group#dim_by_id)
         # We are the root: pass self as `file`, nil as `parent`.
         super(ncid, self, nil)
         # In create mode the Conventions attribute is not defined yet, so
         # resolve the codec lazily (on first read/write, once definitions are
         # in place).  In read mode it can be resolved eagerly.
         @codec = defining ? nil : Codec.resolve(self, codec)
-        # Safety net: if the user forgets close on an atomic-write File,
-        # at least unlink the tmp (never publish half-written data via a
-        # finalizer — rename is close's job, not GC's).
-        if @tmp_path
-          ObjectSpace.define_finalizer(self,
-            self.class.send(:atomic_tmp_cleaner, @tmp_path))
-        end
+        # Safety net for a File never closed, at GC or process exit: close
+        # the ncid, or a NetCDF-3 file loses what was written and the ids
+        # run out. An atomic write's tmp is unlinked rather than published
+        # (rename is close's job, not GC's). `@state` is shared with the
+        # finalizer, so a File closed by hand is not closed again — its id
+        # may by then name another file.
+        @state = [false]
+        ObjectSpace.define_finalizer(self,
+          self.class.send(:unclosed_file_closer, ncid, @state, @tmp_path))
       end
 
-      # Registers the finalizer as a separate class method so the proc
-      # does NOT capture `self` (that would pin the File and defeat GC).
+      # Builds the finalizer in a class method so the proc does NOT capture
+      # `self` (that would pin the File and defeat GC).
       # @api private
-      def self.atomic_tmp_cleaner(tmp)
-        proc { ::File.unlink(tmp) if ::File.exist?(tmp) rescue nil }
+      def self.unclosed_file_closer(ncid, state, tmp)
+        proc do
+          unless state[0]
+            state[0] = true
+            ::NetCDF.nc_close(ncid) rescue nil
+            ::File.unlink(tmp) if tmp && ::File.exist?(tmp) rescue nil
+          end
+        end
       end
-      private_class_method :atomic_tmp_cleaner
+      private_class_method :unclosed_file_closer
 
       # The final (logical) file path. For an atomic-write File before
       # close, the bytes are actually at a same-dir tmp path (see
@@ -271,40 +337,94 @@ module CA
         super.tap { @codec = nil if @codec_request == :auto && name.to_s == "Conventions" }
       end
 
+      # {Group#reload}, and the codec chosen again from `Conventions`
+      # (unless one was given with `codec:`).
+      # @return [self]
+      def reload
+        super
+        @codec = nil if @codec_request == :auto
+        self
+      end
+
       # @return [Integer] `NC_FORMAT_*` constant
-      def format = nc_inq_format(@ncid)
+      def format = (ensure_open; nc_inq_format(@ncid))
+
+      # Whether the file is in a NetCDF-4 format (NetCDF-4, or NetCDF-4
+      # classic model), the formats with chunking and filters.
+      # @return [Boolean]
+      def netcdf4? = [NC_FORMAT_NETCDF4, NC_FORMAT_NETCDF4_CLASSIC].include?(format)
 
       # Close and, for an atomic-write File, atomically rename the tmp to
       # the final path. Idempotent: a second call is a no-op.
+      #
+      # The File counts as closed even when `nc_close` raises: libnetcdf
+      # has let go of the id by then, and closing it again crashes the
+      # process. An atomic write whose close or rename fails discards
+      # its tmp.
       # @return [void]
       def close
         return if @closed
-        nc_close(@ncid)
-        @closed = true
-        if @tmp_path
-          ::File.rename(@tmp_path, @filename)
-          @tmp_path = nil
+        @closed = @state[0] = true
+        begin
+          nc_close(@ncid)
+          publish_tmp if @tmp_path
+        rescue Exception
+          discard_tmp
+          raise
         end
+        @tmp_path = nil
       end
 
-      # Discard a partially-written atomic file. Closes without renaming
-      # and unlinks the tmp. Used internally by the block-form `ensure`
-      # clause when the block raises; can be called explicitly to abandon
-      # a write in progress. Idempotent.
+      # Discard what has not been committed, and close. Used by the block
+      # form when the block raises; can be called explicitly to abandon a
+      # write in progress. Idempotent.
+      #
+      # It calls `nc_abort`: definitions made since the file last left
+      # define mode are rolled back, and a file created here that is still
+      # in define mode is removed. Data already written stays. An atomic
+      # write's tmp is unlinked.
       # @return [void]
       def abort
         return if @closed
-        nc_close(@ncid) rescue nil
-        @closed = true
-        if @tmp_path
-          ::File.unlink(@tmp_path) if ::File.exist?(@tmp_path)
-          @tmp_path = nil
+        @closed = @state[0] = true
+        nc_abort(@ncid) rescue nil
+        discard_tmp
+      end
+
+      # Move an atomic write's tmp into place: over the target of a
+      # symlink rather than the link, with the replaced file's permissions.
+      # @api private
+      private def publish_tmp
+        target = self.class.send(:atomic_target, @filename)
+        if ::File.exist?(target)
+          ::File.chmod(::File.stat(target).mode & 0o7777, @tmp_path)
         end
+        ::File.rename(@tmp_path, target)
+      end
+
+      # Unlink an atomic write's tmp, if there is one.
+      # @api private
+      private def discard_tmp
+        ::File.unlink(@tmp_path) if @tmp_path && ::File.exist?(@tmp_path)
+        @tmp_path = nil
+      end
+
+      # Raise if the file has been closed: libnetcdf reuses ids, so a
+      # closed File's ncid may already name another open file.
+      # @raise [IOError]
+      # @api private
+      def ensure_open
+        raise IOError, "closed NetCDF file #{@filename}" if @closed
       end
 
       # @return [Boolean]
       def closed? = @closed == true
 
+      # The file's dimensions by id, for {Group#dim_by_id}.
+      # @return [Hash{Integer => Dim}]
+      # @api private
+      attr_reader :dim_index
+
       # The `Conventions` global attribute split into tokens (empty if
       # the attribute is missing). Provided so a higher-level convenience
       # layer can dispatch on the same string this gem uses to pick a
@@ -312,8 +432,10 @@ module CA
       # `"COARDS/CF-1.0"` decompose the way you'd expect.
       # @return [Array<String>]
       def conventions
-        s = att("Conventions") or return []
-        s.split(/[\s,\/]+/).reject(&:empty?)
+        # An NC_STRING array gives an Array; a numeric attribute is ignored.
+        Array(att("Conventions")).grep(String)
+                                 .flat_map { |s| s.split(/[\s,\/]+/) }
+                                 .reject(&:empty?)
       end
 
       # @!group Define / data mode management (also used by every Group / Var)
@@ -321,6 +443,7 @@ module CA
       # Switch to define mode if not already in it.
       # @return [void]
       def ensure_define_mode
+        ensure_open
         return if @defining
         nc_redef(@ncid)
         @defining = true
@@ -329,6 +452,7 @@ module CA
       # Switch to data mode if not already in it.
       # @return [void]
       def ensure_data_mode
+        ensure_open
         return unless @defining
         nc_enddef(@ncid)
         @defining = false
@@ -348,7 +472,10 @@ module CA
       # @!endgroup
 
       # @return [String]
-      def inspect = "#<#{self.class} #{@filename.inspect} vars=#{vars.keys} groups=#{groups.keys}>"
+      def inspect
+        return "#<#{self.class} #{@filename.inspect} (closed)>" if closed?
+        "#<#{self.class} #{@filename.inspect} vars=#{vars.keys} groups=#{groups.keys}>"
+      end
     end
   end
 end