carray-netcdf 1.0.13 → 2.0.0

data/lib/carray/netcdf/var.rb

@@ -0,0 +1,539 @@
+module CA
+  module NetCDF
+    # A NetCDF variable: a lazy handle. Indexing issues a hyperslab read
+    # and runs the file's {Codec} (decode on read, encode on write).
+    # Delegates unknown methods to the decoded CArray (`var.mean`,
+    # `var.sum`, …).
+    #
+    # @example Reading
+    #   var = nc["temp"]
+    #   var[]              # whole variable, decoded
+    #   var[0, 0..9, ..]   # hyperslab
+    #   var.to_ca          # whole variable
+    #   var.raw            # bypass codec
+    #
+    # @example Writing
+    #   nc["temp"][] = ca             # whole variable
+    #   nc["temp"][0, ..] = row       # hyperslab
+    class Var
+
+      include CA::NetCDF
+
+      # @param group [Group] containing group (root or subgroup)
+      # @param varid [Integer] NetCDF variable id within `group`
+      def initialize(group, varid)
+        @group   = group
+        @ncid    = group.ncid
+        @varid   = varid
+        @name    = nc_inq_varname(@ncid, @varid)
+        @nc_type = nc_inq_vartype(@ncid, @varid)
+        @dimids  = nc_inq_vardimid(@ncid, @varid)
+        # Per axis: may a write extend it? The dim can live in an ancestor
+        # group, and a group can hold several unlimited dims (netcdf-4).
+        @unlimited = @dimids.map { |d| group.dim_by_id(d)&.unlimited? || false }
+        refresh_shape
+        @atts    = read_attributes
+      end
+
+      # @!attribute [r] name
+      #   @return [String]
+      # @!attribute [r] atts
+      #   @return [Hash{String => Object}] frozen attribute map
+      # @!attribute [r] group
+      #   @return [Group]
+      # @!attribute [r] varid
+      #   @return [Integer]
+      attr_reader :name, :atts, :group, :varid
+
+      # Current shape. Always freshly inquired — the unlimited dim may have
+      # grown since this Var was created.
+      # @return [Array<Integer>]
+      def shape     = refresh_shape
+
+      # @return [Integer] number of dimensions
+      def ndim      = shape.length
+
+      # @return [Boolean] true if this is a 0-D scalar variable
+      def scalar?   = @dimids.empty?
+
+      # @return [Integer] on-disk `NC_*` type constant
+      def nc_type   = @nc_type
+
+      # @return [Symbol] CArray `data_type` symbol
+      def data_type = NC_TO_CA[@nc_type]
+
+      # @return [File] root file
+      def file      = @group.file
+
+      # Look up a variable attribute.
+      # @param name [#to_s]
+      # @return [Object, nil]
+      def att(name) = @atts[name.to_s]
+
+      # The variable's dimensions, in order. A dim may live in an ancestor
+      # group (netcdf-4 dim inheritance), so resolve by name up the tree.
+      # @return [Array<Dim>]
+      def dims = @dimids.map { |d| @group.resolve_dim(nc_inq_dimname(@ncid, d)) }
+
+      # @return [Object] the codec object (Identity / CF / COARDS / custom)
+      def codec = @group.codec
+
+      # @!group Read
+
+      # Read (a hyperslab of) this variable with codec decoding applied.
+      # Accepts any CArray indexing form (range, integer, `..`, `nil`).
+      # Like a write, a read leaves define mode first, so a file being
+      # created can be read back between definitions.
+      #
+      # @param argv [Array] CArray-style index arguments
+      # @return [CArray, Numeric, CATime::Element] decoded values
+      def [](*argv)
+        @group.ensure_data_mode
+        refresh_shape
+        return read_scalar_var(argv) if scalar?
+        info = CArray.scan_index(@shape, argv)
+        if info.type == CA_REG_METHOD_CALL
+          return decode(nc_get_var(@ncid, @varid))[*argv]
+        end
+        decode(read_raw(info, argv))
+      end
+
+      # Read the whole variable with codec decoding.
+      #
+      # The values are read out of the file into a fresh array, which shares
+      # nothing with the variable. `writable: true` — a caller asking for an
+      # array whose writes reach the source — is therefore refused rather
+      # than answered with one that would swallow them; write with
+      # `v[…] = …` instead.
+      #
+      # @param writable [Boolean] refused when true (see above).
+      # @return [CArray, Numeric] decoded values (scalar for 0-D vars)
+      # @raise [RuntimeError] when `writable: true` is given.
+      def to_ca(writable: false)
+        if writable
+          raise "#{self.class}#to_ca reads into a new array; it can't " \
+                "satisfy `writable: true' (write with `v[...] = ...')"
+        end
+        @group.ensure_data_mode
+        refresh_shape
+        return read_scalar_var([]) if scalar?
+        decode(nc_get_var(@ncid, @varid))
+      end
+
+      # Raw values — the public descent point that bypasses the codec.
+      #
+      # `v[…]` and `v.to_ca` compose as
+      #
+      #   v[…]    == v.codec.decode(v, v.raw(…))
+      #   v.to_ca == v.codec.decode(v, v.raw)
+      #
+      # so a caller (or an upper convenience layer) can reach the on-disk
+      # bytes without going through `instance_variable_get(:@varid)`. No
+      # `_FillValue → UNDEF`, no scale/offset, no `_Unsigned` reinterpret,
+      # no CATime. Accepts the same indexing as {#[]}.
+      #
+      # @param argv [Array] CArray-style index arguments
+      # @return [CArray, Numeric] raw on-disk values
+      def raw(*argv)
+        @group.ensure_data_mode
+        refresh_shape
+        if scalar?
+          raise ArgumentError, "scalar variable #{@name.inspect} takes no index" unless argv.empty?
+          return nc_get_var(@ncid, @varid)[0]
+        end
+        return nc_get_var(@ncid, @varid) if argv.empty?
+        info = CArray.scan_index(@shape, argv)
+        case info.type
+        when CA_REG_METHOD_CALL
+          nc_get_var(@ncid, @varid)[*argv]
+        else
+          read_raw(info, argv)
+        end
+      end
+
+      # @!endgroup
+
+      # @!group Write
+
+      # Write (to a hyperslab of) this variable with codec encoding
+      # applied.
+      #
+      # @param argv [Array] index arguments followed by the value; the
+      #   final positional argument is the value to assign
+      # @return [Object] the assigned value
+      def []=(*argv)
+        value = argv.pop
+        @group.ensure_data_mode
+        refresh_shape
+        return write_scalar_var(argv, value) if scalar?
+
+        info, scan_shape = scan_index_for_write(argv)
+        case info.type
+        when CA_REG_ADDRESS
+          put_var1(addr_to_index(info.index[0], scan_shape), encoded_scalar(value))
+        when CA_REG_POINT
+          put_var1(info.index, encoded_scalar(value))
+        when CA_REG_FLATTEN
+          nc_put_var(@ncid, @varid, encoded_array(value))
+        when CA_REG_ALL
+          ca = encoded_array(value)
+          # Use ca.shape (not @shape) so a whole-variable write into an
+          # unlimited-dim variable can extend the unlimited axis; every
+          # fixed axis must match, or part of the variable would be left
+          # as it was.
+          check_whole_write_shape(ca.shape)
+          start = ca.shape.map { 0 }
+          nc_put_vara(@ncid, @varid, start, ca.shape, ca)
+        when CA_REG_BLOCK
+          start, count, stride, _, reversed = block_spec(info)
+          ca = reverse_axes(encoded_array(value, count), count, reversed)
+          if stride.all? { |s| s == 1 }
+            nc_put_vara(@ncid, @varid, start, count, ca)
+          else
+            nc_put_vars(@ncid, @varid, start, count, stride, ca)
+          end
+        else
+          raise ArgumentError, "unsupported index for writing #{@name}"
+        end
+      end
+
+      # @!endgroup
+
+      # Define a variable attribute, typed as {Group#def_var} types its
+      # attributes: a CArray keeps its own type, and a packed-space
+      # attribute (`_FillValue`, `valid_range`, …) given as a Ruby value
+      # takes the variable's on-disk type.
+      # @param name [#to_s]
+      # @param value [Object] Ruby value or CArray
+      # @return [Object] the value written
+      def def_att(name, value)
+        @group.__transaction__ do
+          put_var_att(@ncid, @varid, name.to_s, value, @nc_type)
+        end
+        @atts = read_attributes
+        value
+      end
+
+      # @return [String]
+      def inspect = "#<#{self.class} #{@name}#{shape.inspect}>"
+
+      # CArray methods that change their receiver without a `!` / `=` in
+      # the name. Delegated, they would change a throwaway copy.
+      # @api private
+      IN_PLACE_METHODS = %i[
+        fill unmask paste inherit_mask inherit_mask_replace attach
+        load_binary set_attr
+      ].freeze
+
+      # @api private
+      def respond_to_missing?(id, include_private = false)
+        (CArray.method_defined?(id) && !in_place?(id)) || super
+      end
+
+      # Delegate unknown methods to {#to_ca} — enables `var.mean`, `var.sum`, …
+      #
+      # A method that changes its receiver (`fill`, `seq!`, `mask=`, …) is
+      # refused: it would change the fresh array {#to_ca} returns, not the
+      # file. Write with `v[…] = …`, or call it on `v.to_ca` explicitly.
+      # @raise [NoMethodError] for an in-place CArray method
+      def method_missing(id, *argv, &block)
+        return super unless CArray.method_defined?(id)
+        if in_place?(id)
+          raise NoMethodError.new(
+            "#{self.class}##{id} would change a copy read from the file, not " \
+            "the variable; write with `v[...] = ...' (or call it on v.to_ca)",
+            id)
+        end
+        to_ca.__send__(id, *argv, &block)
+      end
+
+      # @api private
+      private def in_place?(id)
+        id.end_with?("!", "=") || IN_PLACE_METHODS.include?(id)
+      end
+
+      # ---- helpers -------------------------------------------------------
+
+      # The array read path goes straight through the codec; the scalar
+      # (var1) read path wraps the value in a 1-elem carrier, decodes it
+      # with the same codec pipeline, and unwraps. That way
+      # `_FillValue → UNDEF`, scale/offset, and CF time → `CATime::Element`
+      # all work identically for `v[i, j]` and for `v[:, :]`. The 1-cell
+      # allocation is cheap enough for interactive use and keeps the
+      # semantics uniform.
+      # @api private
+      private def decode(raw)
+        return codec.decode(self, raw) if raw.is_a?(CArray)
+        wrapped = CArray.new(data_type, [1]) { raw }
+        codec.decode(self, wrapped)[0]                 # CArray[0] / CATime[0]
+      end
+
+      # Encode a value to a CArray usable by `nc_put_var{,a,s,m}`.
+      # - CArray input: pass through the codec.
+      # - Numeric / UNDEF / Element: broadcast-fill this variable's shape,
+      #   run through the codec. This is how `nc["v"] = 42.0` fills the
+      #   whole variable with a scalar (matches CArray's `ca[] = 42.0`
+      #   idiom); a block write passes the block's `shape`, which may reach
+      #   past the end of an unlimited dim. Only used by array-shaped
+      #   writes (`CA_REG_ALL` / `_BLOCK` / `_FLATTEN`); scalar writes go
+      #   through {#encoded_scalar}.
+      # @api private
+      private def encoded_array(value, shape = @shape)
+        return codec.encode(self, value) if value.is_a?(CArray)
+        ca = broadcast_scalar_to_shape(value, shape)
+        codec.encode(self, ca)
+      end
+
+      # Encode a value to a plain Numeric usable by `nc_put_var1`. Accepts
+      # a Numeric, UNDEF, a `CATime[delta]::Element`, OR a 1-element /
+      # 0-D CArray (the point-write case where the user handed us an
+      # array).
+      #
+      # Numerics are wrapped in a type that preserves precision (int64 for
+      # Integer, float64 for Float / UNDEF) so scale/offset can operate on
+      # the true value; the array-write path never truncates for the same
+      # reason. A float value going to an integer variable is rounded at
+      # the end to match `encode_scale`'s CF §8.1 round rule.
+      # @api private
+      private def encoded_scalar(value)
+        wrapped = case value
+                  when CATime::Element
+                    CA_INT64([value.value]).time(unit: value.unit)
+                  when CATimedelta::Element
+                    CA_INT64([value.value]).timedelta(unit: value.unit)
+                  when CArray
+                    unless value.elements == 1
+                      raise ArgumentError,
+                            "point write to #{@name} needs a single value; " \
+                            "got a #{value.shape.inspect} array"
+                    end
+                    value                       # trust caller's dtype
+                  else
+                    w = CArray.new(preserve_scalar_type(value), [1])
+                    w[0] = value                # handles UNDEF, coerces numerics
+                    w
+                  end
+        encoded = codec.encode(self, wrapped)
+        # Consistency with the array-write path: for an integer target, a
+        # float-typed CArray gets rounded (encode_scale does this when
+        # scale/offset is present; do it unconditionally here so the point
+        # path also rounds when the user writes 3.7 to an :int var).
+        if encoded.is_a?(CArray) &&
+           INTEGER_TYPES.include?(data_type) &&
+           !INTEGER_TYPES.include?(encoded.data_type)
+          encoded = encoded.round
+        end
+        # If the codec did NOT encode the mask into a sentinel (Identity,
+        # or CF for a var without _FillValue), guard against silently
+        # writing CArray's internal UNDEF representation to disk.
+        if encoded.is_a?(CArray) && encoded.has_mask? && encoded.is_masked[0]
+          raise ArgumentError,
+                "cannot point-write UNDEF to #{@name}: the codec did not " \
+                "encode the mask.  Add a _FillValue attribute and open with " \
+                "the CF codec, or write a real value."
+        end
+        encoded[0]
+      end
+
+      # @api private
+      INTEGER_TYPES = %i[int8 uint8 int16 uint16 int32 uint32 int64 uint64].freeze
+
+      # Widest natural type for a Ruby scalar, so wrapping a Float in a
+      # 1-elem CArray doesn't truncate against an int variable's data_type.
+      # @api private
+      private def preserve_scalar_type(value)
+        case value
+        when Float   then :float64
+        when Integer then :int64
+        else              data_type   # UNDEF, nil, ...  (mask-capable of any type)
+        end
+      end
+
+      # Build a shape-matching CArray filled with `value`. CArray's
+      # arity-0 block form is a documented single-evaluation fill idiom.
+      # @api private
+      private def broadcast_scalar_to_shape(value, shape)
+        if shape.empty?                         # 0-D scalar variable
+          w = CArray.new(data_type, [1])
+          w[0] = value
+          w
+        else
+          CArray.new(data_type, shape) { value }
+        end
+      end
+
+      # 0-D scalar variables have shape [] — `CArray.scan_index` doesn't
+      # accept that, and `nc_get_var` / `nc_put_var` deal in a 1-element
+      # buffer. Reads / writes route through `nc_get_var` / `nc_put_var`
+      # and drop the extra dimension for the caller.
+      # @api private
+      private def read_scalar_var(argv)
+        unless argv.empty?
+          raise ArgumentError, "scalar variable #{@name.inspect} takes no index"
+        end
+        raw = nc_get_var(@ncid, @varid)         # 1-element CArray
+        decoded = codec.decode(self, raw)
+        decoded[0]                              # scalar Numeric / UNDEF / Element
+      end
+
+      # @api private
+      private def write_scalar_var(argv, value)
+        unless argv.empty?
+          raise ArgumentError, "scalar variable #{@name.inspect} takes no index"
+        end
+        ca = encoded_array(value)               # 1-element CArray of var type
+        nc_put_var(@ncid, @varid, ca)
+      end
+
+      # @api private
+      private def read_raw(info, argv)
+        case info.type
+        when CA_REG_ALL
+          nc_get_var(@ncid, @varid)
+        when CA_REG_FLATTEN
+          nc_get_var(@ncid, @varid)[nil]
+        when CA_REG_ADDRESS
+          get_var1(addr_to_index(info.index[0]))
+        when CA_REG_POINT
+          get_var1(info.index)
+        when CA_REG_BLOCK
+          start, count, stride, kept, reversed = block_spec(info)
+          out = if stride.all? { |s| s == 1 }
+                  nc_get_vara(@ncid, @varid, start, count)
+                else
+                  nc_get_vars(@ncid, @varid, start, count, stride)
+                end
+          out = reverse_axes(out, count, reversed)
+          # drop point-indexed axes (size 1) by reshaping to the kept shape
+          (kept.empty? || kept.length == count.length) ? out : out.reshape(*kept)
+        when CA_REG_SELECT, CA_REG_GRID
+          nc_get_var(@ncid, @varid)[*argv]
+        else
+          raise ArgumentError, "unsupported index for reading #{@name}"
+        end
+      end
+
+      # One element. An NC_CHAR element travels as a 1-char String in the
+      # netcdf gem's var1 calls; here it is the byte, as it is in an array
+      # read (NC_CHAR maps to :uint8).
+      # @api private
+      private def get_var1(index)
+        v = nc_get_var1(@ncid, @varid, index)
+        @nc_type == NC_CHAR && v.is_a?(String) ? (v.getbyte(0) || 0) : v
+      end
+
+      # @api private
+      private def put_var1(index, value)
+        value = Integer(value).chr if @nc_type == NC_CHAR && value.is_a?(Numeric)
+        nc_put_var1(@ncid, @varid, index, value)
+      end
+
+      # @api private
+      private def check_whole_write_shape(shape)
+        fits = shape.length == @shape.length &&
+               shape.each_index.all? { |i| @unlimited[i] || shape[i] == @shape[i] }
+        return if fits
+        raise ArgumentError,
+              "whole-variable write to #{@name} needs shape #{@shape.inspect}" \
+              "#{' (any length on an unlimited axis)' if @unlimited.any?}; " \
+              "got #{shape.inspect}"
+      end
+
+      # -> [start, count, stride, kept_shape, reversed]; kept_shape drops
+      # point axes. libnetcdf takes only a positive stride, so a reversed
+      # range (`(9..0).step(-1)`) becomes the same cells walked forward,
+      # with its axis flagged in `reversed` for {#reverse_axes}.
+      # @api private
+      private def block_spec(info)
+        start = []; count = []; stride = []; kept = []; reversed = []
+        info.index.each do |idx|
+          if idx.is_a?(Array)
+            first, n, step = idx
+            if step.negative?
+              first += (n - 1) * step
+              step   = -step
+            end
+            start << first; count << n; stride << step; kept << n
+            reversed << idx[2].negative?
+          else
+            start << idx; count << 1; stride << 1; reversed << false
+          end
+        end
+        [start, count, stride, kept, reversed]
+      end
+
+      # `ca` (of `count` elements) laid out as `count`, with the flagged
+      # axes reversed — the same both ways, so it serves read and write.
+      # @api private
+      private def reverse_axes(ca, count, reversed)
+        return ca unless reversed.any?
+        ca = ca.to_ca.reshape(*count) unless ca.shape == count
+        ca[*reversed.each_with_index.map { |r, i| r ? [count[i] - 1, count[i], -1] : nil }].to_ca
+      end
+
+      # @api private
+      private def addr_to_index(addr, shape = @shape)
+        index = Array.new(shape.length)
+        (shape.length - 1).downto(0) do |i|
+          index[i] = addr % shape[i]
+          addr /= shape[i]
+        end
+        index
+      end
+
+      # Refresh `@shape` from the file and return it — the unlimited dim
+      # can grow between reads/writes, so the length cached at open time
+      # goes stale. Public callers (like {#shape}) receive the returned
+      # array; internal callers rely on `@shape` being updated as a side
+      # effect.
+      # @api private
+      private def refresh_shape
+        @shape = @dimids.map { |d| nc_inq_dimlen(@ncid, d) }
+      end
+
+      # Resolve a write index -> [info, shape it was resolved against].
+      #
+      # Indices are resolved against the true `@shape` whenever they fit,
+      # so a negative index or an open range means what it means on read.
+      # Only an index past the end of an unlimited dimension (libnetcdf
+      # grows it on the actual `put_var*` call) falls back to a shape with
+      # that axis capped at a huge length. An index that resolves
+      # differently under two such caps depends on the axis length — a
+      # negative index or an open range reaching past the end — and has no
+      # meaning there, so it is refused instead of landing near 2**31.
+      # @api private
+      private def scan_index_for_write(argv)
+        [CArray.scan_index(@shape, argv), @shape]
+      rescue IndexError
+        raise unless @unlimited.any?
+        shape = shape_for_write(INT_MAX_SCAN)
+        info  = CArray.scan_index(shape, argv)
+        unless CArray.scan_index(shape_for_write(INT_MAX_SCAN - 1), argv) == info
+          raise IndexError,
+                "index #{argv.inspect} for writing #{@name} is out of range " \
+                "for shape #{@shape.inspect}; only a non-negative index can " \
+                "extend the unlimited dimension"
+        end
+        [info, shape]
+      end
+
+      # `@shape` with every unlimited axis replaced by `cap`.
+      # @api private
+      INT_MAX_SCAN = 2**31 - 1
+      private def shape_for_write(cap)
+        @unlimited.each_with_index.map { |u, i| u ? cap : @shape[i] }
+      end
+
+      # @api private
+      private def read_attributes
+        natts = nc_inq_varnatts(@ncid, @varid)
+        atts  = {}
+        natts.times do |i|
+          name = nc_inq_attname(@ncid, @varid, i)
+          atts[name] = nc_get_att(@ncid, @varid, name)
+        end
+        atts.freeze
+      end
+    end
+  end
+end

data/lib/carray/netcdf/version.rb

@@ -0,0 +1,5 @@
+module CA
+  module NetCDF
+    VERSION = "2.0.0"
+  end
+end