carray-netcdf 2.0.0 → 2.1.0

data/lib/carray/netcdf/var.rb

@@ -18,6 +18,9 @@ module CA
     class Var
 
       include CA::NetCDF
+      # The low-level calls are this class's helpers, not its interface.
+      private :nc_get_var, :nc_get_vara, :nc_get_vars, :nc_get_varm,
+              :nc_put_var, :nc_put_vara, :nc_put_vars, :nc_put_varm
 
       # @param group [Group] containing group (root or subgroup)
       # @param varid [Integer] NetCDF variable id within `group`
@@ -53,6 +56,34 @@ module CA
       # @return [Integer] number of dimensions
       def ndim      = shape.length
 
+      # Sizes asked of the file, not by reading the variable (a CArray
+      # method forwarded to {#to_ca} would read all of it).
+      # @return [Integer]
+      def elements  = shape.inject(1, :*)
+      # @return [Integer]
+      def size      = elements
+      # @return [Integer, nil] the length of the first dimension
+      def dim0      = shape[0]
+      # @return [Integer, nil] the length of the second dimension
+      def dim1      = shape[1]
+      # @return [Integer, nil] the length of the third dimension
+      def dim2      = shape[2]
+      # @return [Integer, nil] the length of the fourth dimension
+      def dim3      = shape[3]
+
+      # The first / last element, read as a point.
+      # @return [Numeric, UNDEF, CATime::Element, nil]
+      def first
+        return self[] if scalar?
+        elements.zero? ? nil : self[0]
+      end
+
+      # @return [Numeric, UNDEF, CATime::Element, nil]
+      def last
+        return self[] if scalar?
+        elements.zero? ? nil : self[elements - 1]
+      end
+
       # @return [Boolean] true if this is a 0-D scalar variable
       def scalar?   = @dimids.empty?
 
@@ -60,7 +91,17 @@ module CA
       def nc_type   = @nc_type
 
       # @return [Symbol] CArray `data_type` symbol
-      def data_type = NC_TO_CA[@nc_type]
+      # An enum variable has its base type's; `nil` for a type with no
+      # CArray counterpart (`NC_STRING`, opaque, ...).
+      def data_type
+        return @data_type if defined?(@data_type)
+        t = @nc_type
+        if t >= NC_FIRSTUSERTYPEID
+          _, _, base, _, klass = nc_inq_user_type(@ncid, t)
+          t = base if klass == NC_ENUM
+        end
+        @data_type = NC_TO_CA[t]
+      end
 
       # @return [File] root file
       def file      = @group.file
@@ -70,14 +111,86 @@ module CA
       # @return [Object, nil]
       def att(name) = @atts[name.to_s]
 
+      # The `NC_*` type an attribute is stored in, or nil if there is none.
+      # @param name [#to_s]
+      # @return [Integer, nil]
+      def att_type(name)
+        return nil unless @atts.key?(name.to_s)
+        @group.file.ensure_open
+        nc_inq_atttype(@ncid, @varid, name.to_s)
+      end
+
       # The variable's dimensions, in order. A dim may live in an ancestor
       # group (netcdf-4 dim inheritance), so resolve by name up the tree.
       # @return [Array<Dim>]
-      def dims = @dimids.map { |d| @group.resolve_dim(nc_inq_dimname(@ncid, d)) }
+      # A dimension is found by its id, so a same-named one defined later
+      # in a subgroup does not stand in for it.
+      def dims
+        @group.file.ensure_open
+        @dimids.map { |d| @group.dim_by_id(d) || @group.resolve_dim(nc_inq_dimname(@ncid, d)) }
+      end
 
       # @return [Object] the codec object (Identity / CF / COARDS / custom)
       def codec = @group.codec
 
+      # The variable's storage and filter settings, in the keyword form
+      # {Group#def_var} takes, so that another variable can be defined
+      # with the same ones. Only what differs from a new variable's
+      # default is listed: chunk sizes, the compressors and shuffle,
+      # fletcher32, a byte order other than the host's, and no_fill. A
+      # file in a classic format has none of these, and gives `{}`.
+      # @return [Hash{Symbol => Object}]
+      def storage_options
+        file.ensure_open
+        return {} unless file.netcdf4?
+        opts = {}
+        storage, sizes = nc_inq_var_chunking(@ncid, @varid)
+        case storage
+        when NC_CHUNKED then opts[:chunks] = sizes
+        when NC_COMPACT then opts[:chunks] = :compact
+        end
+        shuffle, deflate, level = nc_inq_var_deflate(@ncid, @varid)
+        opts[:deflate] = level if deflate
+        opts[:shuffle] = true if shuffle
+        if respond_to?(:nc_inq_var_zstandard, true)
+          has, level = nc_inq_var_zstandard(@ncid, @varid)
+          opts[:zstandard] = level if has
+        end
+        if respond_to?(:nc_inq_var_bzip2, true)
+          has, level = nc_inq_var_bzip2(@ncid, @varid)
+          opts[:bzip2] = level if has
+        end
+        if respond_to?(:nc_inq_var_szip, true)
+          mask, pixels = nc_inq_var_szip(@ncid, @varid)
+          if mask != 0
+            coding = (mask & NC_SZIP_NN) != 0 ? :nn : :ec
+            opts[:szip] = { coding: coding, pixels_per_block: pixels }
+          end
+        end
+        if respond_to?(:nc_inq_var_blosc, true)
+          has, compressor, level, _blocksize, shuffle = nc_inq_var_blosc(@ncid, @varid)
+          if has
+            # blocksize is left out: libnetcdf's blosc filter stores the
+            # chunk's byte size in its place, not the value given.
+            opts[:blosc] = { compressor: BLOSC_COMPRESSOR_NAMES.fetch(compressor, compressor),
+                             level: level,
+                             shuffle: BLOSC_SHUFFLE_NAMES.fetch(shuffle, shuffle) }
+          end
+        end
+        opts[:fletcher32] = true if nc_inq_var_fletcher32(@ncid, @varid)
+        endian = nc_inq_var_endian(@ncid, @varid)
+        if endian != NC_ENDIAN_NATIVE && endian != HOST_ENDIAN
+          opts[:endian] = (endian == NC_ENDIAN_BIG) ? :big : :little
+        end
+        no_fill = begin
+                    nc_inq_var_fill(@ncid, @varid)[0]
+                  rescue NotImplementedError
+                    false                 # NC_STRING: no fill setting to read
+                  end
+        opts[:no_fill] = true if no_fill
+        return opts
+      end
+
       # @!group Read
 
       # Read (a hyperslab of) this variable with codec decoding applied.
@@ -108,7 +221,8 @@ module CA
       #
       # @param writable [Boolean] refused when true (see above).
       # @return [CArray, Numeric] decoded values (scalar for 0-D vars)
-      # @raise [RuntimeError] when `writable: true` is given.
+      # @raise [RuntimeError] when `writable: true` is given, as CArray's
+      #   own to_ca does.
       def to_ca(writable: false)
         if writable
           raise "#{self.class}#to_ca reads into a new array; it can't " \
@@ -174,7 +288,9 @@ module CA
         when CA_REG_POINT
           put_var1(info.index, encoded_scalar(value))
         when CA_REG_FLATTEN
-          nc_put_var(@ncid, @varid, encoded_array(value))
+          ca = encoded_array(value)
+          check_write_elements(ca, @shape.inject(1, :*))
+          nc_put_var(@ncid, @varid, ca)
         when CA_REG_ALL
           ca = encoded_array(value)
           # Use ca.shape (not @shape) so a whole-variable write into an
@@ -185,8 +301,10 @@ module CA
           start = ca.shape.map { 0 }
           nc_put_vara(@ncid, @varid, start, ca.shape, ca)
         when CA_REG_BLOCK
-          start, count, stride, _, reversed = block_spec(info)
-          ca = reverse_axes(encoded_array(value, count), count, reversed)
+          start, count, stride, kept, reversed = block_spec(info)
+          encoded = encoded_array(value, count)
+          check_block_write_shape(encoded, kept) if value.is_a?(CArray)
+          ca = reverse_axes(encoded, count, reversed)
           if stride.all? { |s| s == 1 }
             nc_put_vara(@ncid, @varid, start, count, ca)
           else
@@ -208,23 +326,30 @@ module CA
       # @return [Object] the value written
       def def_att(name, value)
         @group.__transaction__ do
-          put_var_att(@ncid, @varid, name.to_s, value, @nc_type)
+          put_var_att(@ncid, @varid, name.to_s, value, @nc_type, unsigned: unsigned_flag?(@atts["_Unsigned"]))
         end
         @atts = read_attributes
         value
       end
 
       # @return [String]
-      def inspect = "#<#{self.class} #{@name}#{shape.inspect}>"
+      def inspect
+        return "#<#{self.class} #{@name} (closed)>" if @group.file.closed?
+        "#<#{self.class} #{@name}#{shape.inspect}>"
+      end
 
       # CArray methods that change their receiver without a `!` / `=` in
       # the name. Delegated, they would change a throwaway copy.
       # @api private
       IN_PLACE_METHODS = %i[
-        fill unmask paste inherit_mask inherit_mask_replace attach
+        fill unmask invert_mask paste inherit_mask inherit_mask_replace attach
         load_binary set_attr
       ].freeze
 
+      # Operators that end in `=` but only compare.
+      # @api private
+      COMPARISON_METHODS = %i[== != <= >= ===].freeze
+
       # @api private
       def respond_to_missing?(id, include_private = false)
         (CArray.method_defined?(id) && !in_place?(id)) || super
@@ -236,7 +361,7 @@ module CA
       # refused: it would change the fresh array {#to_ca} returns, not the
       # file. Write with `v[…] = …`, or call it on `v.to_ca` explicitly.
       # @raise [NoMethodError] for an in-place CArray method
-      def method_missing(id, *argv, &block)
+      private def method_missing(id, *argv, &block)
         return super unless CArray.method_defined?(id)
         if in_place?(id)
           raise NoMethodError.new(
@@ -249,6 +374,7 @@ module CA
 
       # @api private
       private def in_place?(id)
+        return false if COMPARISON_METHODS.include?(id)
         id.end_with?("!", "=") || IN_PLACE_METHODS.include?(id)
       end
 
@@ -279,9 +405,13 @@ module CA
       #   through {#encoded_scalar}.
       # @api private
       private def encoded_array(value, shape = @shape)
-        return codec.encode(self, value) if value.is_a?(CArray)
-        ca = broadcast_scalar_to_shape(value, shape)
-        codec.encode(self, ca)
+        ca = case value
+             when CArray          then value
+             when CATime::Element then element_array(value, shape)
+             else                      broadcast_scalar_to_shape(value, shape)
+             end
+        refuse_timedelta(ca)
+        round_for_integer_variable(codec.encode(self, ca))
       end
 
       # Encode a value to a plain Numeric usable by `nc_put_var1`. Accepts
@@ -300,7 +430,7 @@ module CA
                   when CATime::Element
                     CA_INT64([value.value]).time(unit: value.unit)
                   when CATimedelta::Element
-                    CA_INT64([value.value]).timedelta(unit: value.unit)
+                    refuse_timedelta(value)
                   when CArray
                     unless value.elements == 1
                       raise ArgumentError,
@@ -313,16 +443,7 @@ module CA
                     w[0] = value                # handles UNDEF, coerces numerics
                     w
                   end
-        encoded = codec.encode(self, wrapped)
-        # Consistency with the array-write path: for an integer target, a
-        # float-typed CArray gets rounded (encode_scale does this when
-        # scale/offset is present; do it unconditionally here so the point
-        # path also rounds when the user writes 3.7 to an :int var).
-        if encoded.is_a?(CArray) &&
-           INTEGER_TYPES.include?(data_type) &&
-           !INTEGER_TYPES.include?(encoded.data_type)
-          encoded = encoded.round
-        end
+        encoded = round_for_integer_variable(codec.encode(self, wrapped))
         # If the codec did NOT encode the mask into a sentinel (Identity,
         # or CF for a var without _FillValue), guard against silently
         # writing CArray's internal UNDEF representation to disk.
@@ -335,30 +456,92 @@ module CA
         encoded[0]
       end
 
+      # A float-typed CArray bound for an integer variable is rounded, as
+      # `encode_scale` rounds a packed one (CF §8.1), rather than left to
+      # libnetcdf's conversion, which truncates. Shared by the array and
+      # point write paths so `3.7` lands as `4` either way.
+      # @api private
+      private def round_for_integer_variable(encoded)
+        if encoded.is_a?(CArray) &&
+           INTEGER_TYPES.include?(data_type) &&
+           !INTEGER_TYPES.include?(encoded.data_type)
+          # A NaN left here (no codec to make it the fill) would be cast to
+          # a valid-looking 0.
+          if encoded.float? && encoded.is_nan.any
+            raise ArgumentError,
+                  "cannot write NaN to integer variable #{@name}; give it a " \
+                  "_FillValue under a CF or COARDS codec, or replace the NaNs"
+          end
+          return encoded.round
+        end
+        encoded
+      end
+
+      # @api private
+      HOST_ENDIAN = ([1].pack("S") == [1].pack("v")) ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG
+
+      # `nc_inq_var_blosc` values back to the names `def_var(blosc:)` takes
+      # (empty when libnetcdf is older than 4.9 and has no blosc constants).
+      # @api private
+      BLOSC_COMPRESSOR_NAMES = %i[lz lz4 lz4hc snappy zlib zstd].filter_map { |k|
+        c = "BLOSC_#{k.upcase}"
+        [::NetCDF.const_get(c), k] if ::NetCDF.const_defined?(c)
+      }.to_h.freeze
+      # @api private
+      BLOSC_SHUFFLE_NAMES = { NOSHUFFLE: :none, SHUFFLE: :byte, BITSHUFFLE: :bit }.filter_map { |c, k|
+        [::NetCDF.const_get("BLOSC_#{c}"), k] if ::NetCDF.const_defined?("BLOSC_#{c}")
+      }.to_h.freeze
+
       # @api private
       INTEGER_TYPES = %i[int8 uint8 int16 uint16 int32 uint32 int64 uint64].freeze
 
       # Widest natural type for a Ruby scalar, so wrapping a Float in a
-      # 1-elem CArray doesn't truncate against an int variable's data_type.
+      # 1-elem CArray doesn't truncate against an int variable's data_type,
+      # and an Integer can be unpacked or reinterpreted before it is
+      # narrowed. An Integer outside int64 keeps the variable's type
+      # (uint64), and so does anything for an NC_CHAR or enum variable,
+      # whose buffers libnetcdf takes only in their own type.
       # @api private
+      INT64_RANGE = (-2**63)..(2**63 - 1)
       private def preserve_scalar_type(value)
+        return data_type if @nc_type == NC_CHAR || @nc_type >= NC_FIRSTUSERTYPEID
         case value
         when Float   then :float64
-        when Integer then :int64
+        when Integer then INT64_RANGE.cover?(value) ? :int64 : data_type
         else              data_type   # UNDEF, nil, ...  (mask-capable of any type)
         end
       end
 
+      # A CATime::Element repeated over `shape`, as a CATime.
+      # @api private
+      private def element_array(elem, shape)
+        CArray.int64(*(shape.empty? ? [1] : shape)) { elem.value }.time(unit: elem.unit)
+      end
+
+      # A duration has no CF encoding here: say so rather than writing its
+      # ticks in whatever unit it happens to have.
+      # @api private
+      private def refuse_timedelta(value)
+        return unless value.is_a?(CATimedelta) || value.is_a?(CATimedelta::Element)
+        raise ArgumentError,
+              "cannot write a CATimedelta to #{@name}; write its count in the " \
+              "variable's units (e.g. td.ticks)"
+      end
+
       # Build a shape-matching CArray filled with `value`. CArray's
       # arity-0 block form is a documented single-evaluation fill idiom.
       # @api private
       private def broadcast_scalar_to_shape(value, shape)
+        # A Float goes in as float64 and an Integer as int64, so the codec
+        # unpacks the true value, an unsigned one is reinterpreted before
+        # it is narrowed, and an integer variable gets a Float rounded.
+        type = preserve_scalar_type(value)
         if shape.empty?                         # 0-D scalar variable
-          w = CArray.new(data_type, [1])
+          w = CArray.new(type, [1])
           w[0] = value
           w
         else
-          CArray.new(data_type, shape) { value }
+          CArray.new(type, shape) { value }
         end
       end
 
@@ -381,6 +564,11 @@ module CA
         unless argv.empty?
           raise ArgumentError, "scalar variable #{@name.inspect} takes no index"
         end
+        if value.is_a?(CArray) && value.elements != 1
+          raise ArgumentError,
+                "scalar variable #{@name.inspect} needs a single value; " \
+                "got a #{value.shape.inspect} array"
+        end
         ca = encoded_array(value)               # 1-element CArray of var type
         nc_put_var(@ncid, @varid, ca)
       end
@@ -406,18 +594,79 @@ module CA
           out = reverse_axes(out, count, reversed)
           # drop point-indexed axes (size 1) by reshaping to the kept shape
           (kept.empty? || kept.length == count.length) ? out : out.reshape(*kept)
-        when CA_REG_SELECT, CA_REG_GRID
+        when CA_REG_GRID
+          read_grid(argv) || nc_get_var(@ncid, @varid)[*argv]
+        when CA_REG_MAPPING
+          read_mapping(argv[0]) || nc_get_var(@ncid, @varid)[*argv]
+        when CA_REG_SELECT
+          # A boolean mask covers the whole variable.
           nc_get_var(@ncid, @varid)[*argv]
         else
           raise ArgumentError, "unsupported index for reading #{@name}"
         end
       end
 
+      # Index arrays per axis (`v[CA_INT([0, 7]), nil, 2..5]`): read the
+      # block that holds what they pick, and pick from it, rather than the
+      # whole variable. nil when an axis takes a form not handled here
+      # (the caller then reads the whole variable).
+      # @api private
+      private def read_grid(argv)
+        return nil unless argv.size == @shape.size
+        start, count, local = [], [], []
+        argv.each_with_index do |idx, axis|
+          n = @shape[axis]
+          case idx
+          when nil
+            start << 0; count << n; local << nil
+          when CArray
+            return nil unless INTEGER_TYPES.include?(idx.data_type)
+            i = idx.to_ca
+            i = i + (i.lt(0) * n) if i.lt(0).any      # negative from the end
+            lo, hi = i.min, i.max
+            return nil unless lo.is_a?(Integer) && lo >= 0 && hi < n
+            start << lo; count << hi - lo + 1; local << i - lo
+          when Range
+            b = CArray.scan_index([n], [idx])
+            return nil unless b.type == CA_REG_BLOCK
+            first, len, step = b.index[0]
+            return nil unless step == 1
+            start << first; count << len; local << (0...len)
+          else
+            return nil
+          end
+        end
+        return nil if count.include?(0)
+        nc_get_vara(@ncid, @varid, start, count)[*local]
+      end
+
+      # A flat array of addresses (`v[CA_INT([1, 7, -1])]`) picking a few
+      # elements: read those one by one. nil for many of them.
+      # @api private
+      MAPPING_POINT_LIMIT = 1024
+      private def read_mapping(addrs)
+        return nil unless addrs.is_a?(CArray) && INTEGER_TYPES.include?(addrs.data_type)
+        return nil if addrs.elements > MAPPING_POINT_LIMIT || data_type.nil?
+        total = @shape.inject(1, :*)
+        out = CArray.new(data_type, addrs.shape)
+        addrs.to_a.flatten.each_with_index do |a, k|
+          a += total if a < 0
+          raise IndexError, "index #{a} is out of range for #{@name}" unless (0...total).cover?(a)
+          out[k] = get_var1(addr_to_index(a))
+        end
+        out
+      end
+
       # One element. An NC_CHAR element travels as a 1-char String in the
       # netcdf gem's var1 calls; here it is the byte, as it is in an array
       # read (NC_CHAR maps to :uint8).
       # @api private
       private def get_var1(index)
+        if @nc_type == NC_STRING
+          raise NotImplementedError,
+                "no CArray type for an NC_STRING variable; read it with " \
+                "NetCDF.nc_get_var, which returns Strings"
+        end
         v = nc_get_var1(@ncid, @varid, index)
         @nc_type == NC_CHAR && v.is_a?(String) ? (v.getbyte(0) || 0) : v
       end
@@ -439,6 +688,23 @@ module CA
               "got #{shape.inspect}"
       end
 
+      # A block write takes a CArray shaped like the block, as CArray's own
+      # block assignment does (size-1 axes aside); otherwise the cells
+      # would land in the wrong places, or some be dropped.
+      # @api private
+      private def check_block_write_shape(ca, kept)
+        return if ca.shape.reject { |n| n == 1 } == kept.reject { |n| n == 1 }
+        raise ArgumentError,
+              "shape mismatch writing #{@name}: block #{kept.inspect} <- #{ca.shape.inspect}"
+      end
+
+      # @api private
+      private def check_write_elements(ca, n)
+        return if ca.elements == n
+        raise ArgumentError,
+              "elements mismatch writing #{@name}: #{n} <- #{ca.elements}"
+      end
+
       # -> [start, count, stride, kept_shape, reversed]; kept_shape drops
       # point axes. libnetcdf takes only a positive stride, so a reversed
       # range (`(9..0).step(-1)`) becomes the same cells walked forward,
@@ -488,6 +754,7 @@ module CA
       # effect.
       # @api private
       private def refresh_shape
+        @group.file.ensure_open
         @shape = @dimids.map { |d| nc_inq_dimlen(@ncid, d) }
       end
 
@@ -503,7 +770,19 @@ module CA
       # meaning there, so it is refused instead of landing near 2**31.
       # @api private
       private def scan_index_for_write(argv)
-        [CArray.scan_index(@shape, argv), @shape]
+        info = CArray.scan_index(@shape, argv)
+        # A range scanned against the current length is clipped to it, so
+        # `v[0..1] = ...` on an empty unlimited axis would write nothing.
+        # An index that scans the same whatever the unlimited length is
+        # means what it says; take it when it reaches further.
+        if @unlimited.any?
+          wide  = CArray.scan_index(shape_for_write(INT_MAX_SCAN), argv) rescue nil
+          wider = CArray.scan_index(shape_for_write(INT_MAX_SCAN - 1), argv) rescue nil
+          if wide && wide == wider && wide != info
+            return [wide, shape_for_write(INT_MAX_SCAN)]
+          end
+        end
+        [info, @shape]
       rescue IndexError
         raise unless @unlimited.any?
         shape = shape_for_write(INT_MAX_SCAN)
@@ -530,7 +809,8 @@ module CA
         atts  = {}
         natts.times do |i|
           name = nc_inq_attname(@ncid, @varid, i)
-          atts[name] = nc_get_att(@ncid, @varid, name)
+          value = readable_att(@ncid, @varid, name, @name)
+          atts[name] = deep_freeze(value) unless value.equal?(UNREADABLE_ATT)
         end
         atts.freeze
       end

data/lib/carray/netcdf/version.rb

@@ -1,5 +1,5 @@
 module CA
   module NetCDF
-    VERSION = "2.0.0"
+    VERSION = "2.1.0"
   end
 end