carray-netcdf 1.0.13 → 2.0.0

data/lib/carray/netcdf/codec.rb

@@ -0,0 +1,536 @@
+module CA
+  module NetCDF
+    # Value codec — the encode/decode half of a NetCDF "convention".
+    #
+    # ## Contract (var-local, value-only)
+    #
+    # A codec only transforms one variable's VALUES using that variable's
+    # own attributes. It does not model CF's coordinate systems, grid
+    # mappings, cell methods, `standard_name`, `coordinates`, or
+    # `ancillary_variables` — anything that names OTHER variables or lives
+    # outside the variable itself belongs in a higher-level convenience
+    # layer. A codec method takes `(var, ca)`; walking to `var.file` to
+    # reach other variables is a contract violation, not a feature.
+    #
+    # The transforms this codec performs (in CF §2.5 / §8.1 order):
+    #
+    #   1. `_Unsigned` bit-reinterpret (signed on-disk → unsigned user)
+    #   2. `_FillValue` / `missing_value` → mask
+    #   3. `valid_range` / `valid_min` / `valid_max` → mask (opt-in)
+    #   4. `scale_factor` / `add_offset` → unpack
+    #   5. `units: "<u> since <ref>"` (+ `calendar`) → CATime
+    #
+    # `Conventions` is used at {File.open} time to CHOOSE a codec
+    # (dispatch), not by a codec method to interpret values. Once
+    # selected, a codec instance stays var-local for the file's lifetime.
+    module Codec
+
+      @registry = []
+
+      class << self
+        # Register a codec (a class or a stateless module). Later
+        # registrations take precedence.
+        # @param codec [Class, Module]
+        # @return [Class, Module] `codec`
+        def register(codec)
+          @registry.unshift(codec)
+          codec
+        end
+
+        # Resolve `requested` into an object that answers to
+        # `#decode(var, ca)` / `#encode(var, ca)`. Accepts:
+        #
+        #   :auto                          # pick by Conventions attr (or a token of it), else Identity
+        #   :identity | false | nil        # Identity
+        #   :cf | :coards                  # the named codec with defaults
+        #   [:cf, mask_valid_range: ...]   # the named codec with options
+        #   Class                          # instantiated with defaults (Codec::CF)
+        #   Module                         # a stateless codec module (Identity)
+        #   codec instance                 # e.g. Codec::CF.new(...)
+        #
+        # @param file [File]
+        # @param requested [Symbol, Array, Module, Object]
+        # @return [Object] the resolved codec
+        # @raise [ArgumentError] on an unrecognized specifier
+        def resolve(file, requested = :auto)
+          case requested
+          when :auto
+            # The whole string first, then each of its tokens, so
+            # "ACDD-1.3, CF-1.8" still finds CF.
+            conv = file.att("Conventions")
+            hit  = conv && (@registry.find { |c| c.claims?(conv) } ||
+                            @registry.find { |c| file.conventions.any? { |t| c.claims?(t) } })
+            hit ? build(hit) : Identity
+          when :identity, false, nil
+            Identity
+          when Symbol
+            symbol_class(requested).new
+          when Array
+            name, opts = requested[0], (requested[1] || {})
+            symbol_class(name).new(**opts)
+          when Class
+            # A class is instantiated (Class < Module, so this comes
+            # first), unless it is itself the codec.
+            codec?(requested) ? requested : requested.new
+          when Module
+            requested        # a stateless codec module used verbatim
+          else
+            return requested if codec?(requested)
+            raise ArgumentError, "unknown codec: #{requested.inspect}"
+          end
+        end
+
+        private
+
+        def codec?(obj) = obj.respond_to?(:decode) && obj.respond_to?(:encode)
+
+        def symbol_class(sym)
+          case sym
+          when :cf     then CF
+          when :coards then COARDS
+          else raise ArgumentError, "unknown codec: #{sym.inspect}"
+          end
+        end
+
+        def build(codec) = codec.is_a?(Class) ? codec.new : codec
+      end
+
+      # No decoding — raw values pass through unchanged. The safe default
+      # so a file written by an unknown tool round-trips untouched.
+      module Identity
+        module_function
+        # @return [false]
+        def claims?(_conventions) = false
+        # @param _var [Var]
+        # @param ca [CArray]
+        # @return [CArray] `ca`
+        def decode(_var, ca)      = ca
+        # @param _var [Var]
+        # @param ca [CArray]
+        # @return [CArray] `ca`
+        def encode(_var, ca)      = ca
+      end
+
+      # CF / COARDS value transforms. See the module-level docstring for
+      # the pipeline order; the codec chapter of the user guide has the
+      # CF attribute matrix.
+      class CF
+        # @param conventions [String]
+        # @return [Boolean]
+        def self.claims?(conventions)
+          s = conventions.to_s
+          s.start_with?("CF-", "CF ") || s == "CF"
+        end
+
+        # @param mask_valid_range [Boolean] when true, mask cells that
+        #   fall outside `valid_range` / `valid_min` / `valid_max`
+        #   (packed-space judgement, before scale/offset).
+        def initialize(mask_valid_range: false)
+          @mask_valid_range = mask_valid_range
+          @warned = {}         # per-instance = per-File-open; not process-global
+        end
+
+        # @return [Boolean]
+        attr_reader :mask_valid_range
+
+        # Decode on-disk raw values into Ruby-side values.
+        # @param var [Var]
+        # @param ca [CArray] raw values
+        # @return [CArray, CATime] decoded values
+        def decode(var, ca)
+          a  = var.atts
+          ca = decode_unsigned(var, ca)                          # signed disk -> unsigned user
+          ca = decode_fill(var, a, ca)                           # (in unsigned/packed space)
+          ca = decode_valid(var, a, ca) if @mask_valid_range     # (same)
+          ca = decode_scale(a, ca)                               # packed -> physical
+          ca = decode_time(var, a, ca)  if time_units?(a["units"])
+          ca
+        end
+
+        # Encode Ruby-side values into on-disk raw values.
+        # @param var [Var]
+        # @param ca [CArray, CATime]
+        # @return [CArray] raw values ready for `nc_put_var*`
+        def encode(var, ca)
+          a  = var.atts
+          ca = encode_time(var, a, ca)  if time_units?(a["units"])
+          ca = encode_scale(var, a, ca)
+          ca = encode_fill(var, a, ca)
+          ca = encode_unsigned(var, ca)                          # unsigned user -> signed disk
+          ca
+        end
+
+        # ---- _Unsigned: bit-reinterpret signed <-> unsigned ---------------
+        #
+        # netCDF-3 has no unsigned integer types. Producers set
+        # `_Unsigned = "true"` on a byte/short/int/int64 variable to say
+        # "these bits are semantically unsigned; reinterpret." Same
+        # convention shows up on NetCDF-4 files from tools with an
+        # HDF-EOS heritage (satellite data, etc.). This runs FIRST
+        # because every subsequent transform (fill/valid comparison,
+        # scale/offset) needs the numerically-correct interpretation.
+
+        SIGNED_TO_UNSIGNED = {
+          int8: :uint8, int16: :uint16, int32: :uint32, int64: :uint64,
+        }.freeze
+        UNSIGNED_TO_SIGNED = SIGNED_TO_UNSIGNED.invert.freeze
+        UNSIGNED_MASK      = {
+          int8: 0xFF, int16: 0xFFFF, int32: 0xFFFFFFFF,
+          int64: 0xFFFFFFFFFFFFFFFF,
+        }.freeze
+
+        # @api private
+        private def unsigned?(var)
+          v = var.att("_Unsigned")
+          case v
+          when String  then v.downcase == "true"
+          when Integer then v != 0
+          when true    then true
+          else              false
+          end
+        end
+
+        # @api private
+        private def decode_unsigned(var, ca)
+          return ca unless unsigned?(var)
+          t = SIGNED_TO_UNSIGNED[var.data_type] or return ca
+          ca.refer(t)                              # zero-copy view
+        end
+
+        # @api private
+        private def encode_unsigned(var, ca)
+          return ca unless unsigned?(var)
+          t = var.data_type                        # signed on-disk type
+          return ca unless SIGNED_TO_UNSIGNED.key?(t)
+          return ca unless ca.is_a?(CArray)
+          # An already-signed value is written as is. Anything else is
+          # brought to the unsigned type of the same width (rounding a
+          # float, as a packed value is rounded), and `refer`
+          # bit-reinterprets that — `refer` only works between types of
+          # one byte size.
+          return ca if ca.data_type == t
+          u  = SIGNED_TO_UNSIGNED[t]
+          ca = ca.round if ca.float?
+          ca = ca.to_type(u) unless ca.data_type == u
+          ca.refer(t)
+        end
+
+        # If `_Unsigned` is set, the attribute is stored on disk in the
+        # underlying signed type — reinterpret to the semantic unsigned
+        # value so comparisons in unsigned space work directly.
+        # @api private
+        private def unsigned_reinterpret(var, val)
+          return val unless unsigned?(var)
+          mask = UNSIGNED_MASK[var.data_type] or return val
+          case val
+          when Integer then val & mask
+          when Array   then val.map { |x| x.is_a?(Integer) ? (x & mask) : x }
+          when CArray  then val & mask
+          else              val
+          end
+        end
+
+        # ---- fill / missing -> mask -----------------------------------------
+        # Only copy when at least one cell actually matches a fill sentinel;
+        # a large (100s of MB) variable with fill defined but no fill cells
+        # would otherwise double its resident size for no reason.
+        # @api private
+        private def decode_fill(var, atts, ca)
+          fills = fill_values(var, atts).select { |fv| fill_cells(ca, fv).any }
+          return ca if fills.empty?
+          ca = ca.copy
+          fills.each { |fv| ca[fill_cells(ca, fv)] = UNDEF }
+          ca
+        end
+
+        # Cells equal to the fill value. A NaN fill (common in float data)
+        # equals nothing, not even NaN, so it matches the NaN cells.
+        # @api private
+        private def fill_cells(ca, fv)
+          fv.is_a?(Float) && fv.nan? && ca.float? ? ca.is_nan : ca.eq(fv)
+        end
+
+        # ---- valid_range -> mask (opt-in) -----------------------------------
+        # Same copy-avoidance as decode_fill.
+        # @api private
+        private def decode_valid(var, atts, ca)
+          vmin, vmax = valid_bounds(var, atts)
+          return ca unless vmin || vmax
+          below = vmin && ca.lt(vmin).any
+          above = vmax && ca.gt(vmax).any
+          return ca unless below || above
+          ca = ca.copy
+          ca[:lt, vmin] = UNDEF if below
+          ca[:gt, vmax] = UNDEF if above
+          ca
+        end
+
+        # -> [min, max] with either bound possibly nil. `valid_range`
+        # wins over `valid_min` / `valid_max` per CF; `valid_range` must
+        # be length 2.
+        # @api private
+        private def valid_bounds(var, atts)
+          if (vr = atts["valid_range"])
+            pair = vr.is_a?(CArray) ? vr.to_a : Array(vr)
+            if pair.length == 2
+              return [unsigned_reinterpret(var, pair[0]),
+                      unsigned_reinterpret(var, pair[1])]
+            end
+          end
+          [unsigned_reinterpret(var, atts["valid_min"]),
+           unsigned_reinterpret(var, atts["valid_max"])]
+        end
+
+        # ---- scale_factor / add_offset --------------------------------------
+
+        # `unpacked = packed * scale_factor + add_offset` (CF §8.1)
+        # @api private
+        private def decode_scale(atts, ca)
+          sf = atts["scale_factor"]
+          ao = atts["add_offset"]
+          ca = ca * sf if sf
+          ca = ca + ao if ao
+          ca
+        end
+
+        INTEGER_TYPES = %i[int8 uint8 int16 uint16 int32 uint32 int64 uint64].freeze
+
+        # `packed = round((unpacked - add_offset) / scale_factor)`
+        # (A bare cast to an integer variable would truncate:
+        # `1.0/0.1 == 9.999… -> 9`.)
+        # @api private
+        private def encode_scale(var, atts, ca)
+          sf = atts["scale_factor"]
+          ao = atts["add_offset"]
+          return ca unless sf || ao
+          ca = ca - ao if ao
+          ca = ca / sf if sf
+          ca = ca.round if INTEGER_TYPES.include?(var.data_type)
+          ca
+        end
+
+        # ---- mask <-> fill --------------------------------------------------
+
+        # @api private
+        private def encode_fill(var, atts, ca)
+          return ca unless ca.is_a?(CArray) && ca.has_mask?
+          fv = fill_values(var, atts).first or
+            raise ArgumentError,
+                  "variable '#{var.name}' has masked values but no _FillValue " \
+                  "to encode them"
+          ca.strip_mask(fv)
+        end
+
+        # @api private
+        private def fill_values(var, atts)
+          fills = []
+          if atts.key?("_FillValue")
+            fills << unsigned_reinterpret(var, atts["_FillValue"])
+          end
+          if atts.key?("missing_value")
+            Array(atts["missing_value"]).each do |mv|
+              fills << unsigned_reinterpret(var, mv)
+            end
+          end
+          fills
+        end
+
+        # ---- CF time --------------------------------------------------------
+        #
+        # `units: "<interval> since <reference>"` maps directly to
+        # `CATime`'s `.time(unit: <resolution>, origin: <reference>)`
+        # (designed for exactly this "index relative to a base instant"
+        # pattern). `dt.timesteps(unit:, origin:)` is the inverse, used
+        # on encode.
+        #
+        # This gem does NOT do UDUNITS conversions of other units strings
+        # (`"m s-1"`, `"K"`, `"kg m-3"`, …). Those pass through as raw
+        # attribute values. Only the `"<u> since <ref>"` form is
+        # recognized.
+        #
+        # CATime is fixed to the proleptic Gregorian calendar in UTC, so:
+        #   - `standard` / `gregorian` / `proleptic_gregorian` /
+        #     (missing) -> CATime
+        #   - `noleap` / `365_day` / `all_leap` / `366_day` / `360_day` /
+        #     `julian` / `none` -> leave the raw values alone (and warn
+        #     once per calendar per File).
+
+        # @api private
+        private def time_units?(units)
+          units.is_a?(String) && units.include?(" since ")
+        end
+
+        UNIT_WORDS = {
+          "year"        => :Y, "years"       => :Y, "yr"    => :Y, "yrs" => :Y, "y" => :Y,
+          "month"       => :M, "months"      => :M, "mon"   => :M, "mons" => :M,
+          "day"         => :D, "days"        => :D, "d"     => :D,
+          "hour"        => :h, "hours"       => :h, "hr"    => :h, "hrs" => :h, "h" => :h,
+          "minute"      => :m, "minutes"     => :m, "min"   => :m, "mins" => :m,
+          "second"      => :s, "seconds"     => :s, "sec"   => :s, "secs" => :s, "s" => :s,
+          "millisecond" => :ms, "milliseconds" => :ms, "msec" => :ms, "msecs" => :ms, "ms" => :ms,
+          "microsecond" => :us, "microseconds" => :us, "usec" => :us, "usecs" => :us, "us" => :us,
+          "nanosecond"  => :ns, "nanoseconds"  => :ns, "nsec" => :ns, "nsecs" => :ns, "ns" => :ns,
+        }.freeze
+
+        SUPPORTED_CALENDARS = %w[standard gregorian proleptic_gregorian].freeze
+
+        # -> [count, base_symbol, reference_string] or nil if unparseable.
+        # @api private
+        private def parse_time_units(units)
+          interval, ref = units.split(/\s+since\s+/, 2)
+          return nil unless ref
+          m = interval.strip.match(/\A(\d+)?\s*([A-Za-z]+)\z/)
+          return nil unless m
+          count = (m[1] || "1").to_i
+          base  = UNIT_WORDS[m[2].downcase] or return nil
+          [count, base, ref.strip]
+        end
+
+        # See docs/codec.md §"N-of-a-unit intervals" for why we fold
+        # count into the raw values instead of storing at `count × base`
+        # resolution.
+        # @api private
+        private def decode_time(var, atts, ca)
+          units    = atts["units"]
+          calendar = atts["calendar"]
+          unless supported_calendar?(calendar)
+            warn_once(:calendar, calendar,
+                      "variable #{var.name.inspect} uses calendar " \
+                      "#{calendar.inspect} which is not supported; " \
+                      "raw values returned")
+            return ca
+          end
+          parsed = parse_time_units(units)
+          unless parsed
+            warn_once(:units, units,
+                      "variable #{var.name.inspect} has units " \
+                      "#{units.inspect} which the CF time parser does not " \
+                      "understand; raw values returned")
+            return ca
+          end
+          count, base, ref = parsed
+          vals = count == 1 ? ca : ca * count
+          if vals.float?
+            unit, ticks = time_ticks(vals, base)
+            return ticks.time(unit: unit, origin: ref)
+          end
+          ints = vals.data_type == :int64 ? vals : vals.int64
+          ints.time(unit: base, origin: ref)
+        end
+
+        # Length of each fixed-length unit, in ns, finest last. `:Y` / `:M`
+        # have no fixed length, so a fraction of one can't be carried to a
+        # finer unit.
+        NS_PER_UNIT = {
+          D: 86_400_000_000_000, h: 3_600_000_000_000, m: 60_000_000_000,
+          s: 1_000_000_000, ms: 1_000_000, us: 1_000, ns: 1,
+        }.freeze
+
+        # Float time values in `base` units -> [unit, int64 ticks]. Values
+        # with a fraction (`0.5 days since …`) are carried to the first
+        # finer unit that holds them whole, so the fraction is not rounded
+        # away; the finest unit that still fits int64 is rounded to when
+        # none does.
+        # @api private
+        private def time_ticks(vals, base)
+          eps   = vals.data_type == :float32 ? 1.2e-7 : 2.3e-16
+          limit = vals.abs.max
+          limit = 0.0 unless limit.is_a?(Numeric)          # all masked
+          whole = ->(x, f) { d = (x - x.round).abs.max
+                             !d.is_a?(Numeric) || d <= [1e-6, 4 * limit * f * eps].max }
+          return [base, vals.round.int64] if whole.(vals, 1) || !NS_PER_UNIT.key?(base)
+          best = [base, vals]
+          NS_PER_UNIT.each_key.drop_while { |u| u != base }.drop(1).each do |u|
+            f = NS_PER_UNIT[base] / NS_PER_UNIT[u]
+            break if limit * f >= 2**62
+            x = vals * f
+            return [u, x.round.int64] if whole.(x, f)
+            best = [u, x]
+          end
+          [best[0], best[1].round.int64]
+        end
+
+        # @api private
+        private def encode_time(var, atts, ca)
+          return ca unless ca.is_a?(CATime)
+          units    = atts["units"]
+          calendar = atts["calendar"]
+          unless supported_calendar?(calendar)
+            raise ArgumentError,
+                  "cannot encode CATime into variable #{var.name.inspect}: " \
+                  "calendar #{calendar.inspect} is not supported"
+          end
+          parsed = parse_time_units(units) or
+            raise ArgumentError,
+                  "cannot encode CATime into variable #{var.name.inspect}: " \
+                  "units #{units.inspect} is not a CF time-since spec"
+          count, base, ref = parsed
+          fine = ca.unit.base
+          if finer_unit?(fine, base)
+            # The instants are finer than the units' interval: work in the
+            # finer unit so a fraction of an interval isn't truncated.
+            f     = NS_PER_UNIT[base] / NS_PER_UNIT[fine] * count
+            ticks = ca.timesteps(unit: fine, origin: ref)
+            return ticks.float64 / f unless INTEGER_TYPES.include?(var.data_type)
+            unless (ticks % f).eq(0).all
+              raise ArgumentError,
+                    "cannot encode CATime into integer variable #{var.name.inspect}: " \
+                    "some instants fall between the steps of units #{units.inspect}"
+            end
+            return ticks / f
+          end
+          ticks = ca.timesteps(unit: base, origin: ref)
+          return ticks if count == 1
+          unless (ticks % count).eq(0).all
+            raise ArgumentError,
+                  "cannot encode CATime into variable #{var.name.inspect}: " \
+                  "some instants do not land on the #{count}-#{base} grid " \
+                  "implied by units #{units.inspect}"
+          end
+          ticks / count
+        end
+
+        # @api private
+        private def finer_unit?(unit, base)
+          NS_PER_UNIT.key?(unit) && NS_PER_UNIT.key?(base) &&
+            NS_PER_UNIT[unit] < NS_PER_UNIT[base]
+        end
+
+        # @api private
+        private def supported_calendar?(cal)
+          cal.nil? || SUPPORTED_CALENDARS.include?(cal.to_s)
+        end
+
+        # Per-instance cache (a new CF instance is built per
+        # {File.open}, so this is effectively per-File — not
+        # process-global).
+        # @api private
+        private def warn_once(kind, key, message)
+          k = [kind, key]
+          return if @warned[k]
+          @warned[k] = true
+          warn "carray-netcdf CF codec: #{message}"
+        end
+      end
+
+      # COARDS (1995) is CF's predecessor. Every adherent COARDS file is
+      # also an adherent CF file — set inclusion, not just similarity —
+      # so we inherit every transform and only override the class-level
+      # dispatch name. Callers that need to distinguish (COARDS pins the
+      # axis order T,Z,Y,X; CF does not) should check `is_a?(COARDS)`
+      # BEFORE `is_a?(CF)`.
+      class COARDS < CF
+        # @param conventions [String]
+        # @return [Boolean]
+        def self.claims?(conventions)
+          conventions.to_s.start_with?("COARDS")
+        end
+      end
+
+      register(Identity)
+      register(CF)
+      register(COARDS)   # registered last -> checked first, so "COARDS/CF-*"
+                         # goes to COARDS rather than CF.
+    end
+  end
+end

data/lib/carray/netcdf/dim.rb

@@ -0,0 +1,67 @@
+module CA
+  module NetCDF
+    # A NetCDF dimension. Bridges to its coordinate variable (a variable of
+    # the same name in the same group) when one exists.
+    class Dim
+
+      include CA::NetCDF
+
+      # @param group [Group] containing group
+      # @param dimid [Integer] NetCDF dimension id within `group`
+      # @param unlimited [Boolean, nil] known unlimited-ness (from
+      #   {Group#def_dim}); nil to inquire the file
+      def initialize(group, dimid, unlimited: nil)
+        @group  = group
+        @ncid   = group.ncid
+        @dimid  = dimid
+        @name   = nc_inq_dimname(@ncid, @dimid)
+        # Whether a dim is unlimited is a define-time property and doesn't
+        # change over the file's lifetime — cache it once. Asked of the
+        # dim's own group, which is where netcdf-4 keeps it.
+        # nc_inq_unlimdim names only the group's first unlimited dim; a
+        # further one is known here only when this gem defined it.
+        @unlimited = if unlimited.nil?
+                       @dimid == (nc_inq_unlimdim(@ncid) rescue -1)
+                     else
+                       unlimited
+                     end
+      end
+
+      # @!attribute [r] name
+      #   @return [String] dimension name
+      # @!attribute [r] group
+      #   @return [Group] containing group
+      attr_reader :name, :group
+
+      # Current length. Always freshly inquired from the file: an unlimited
+      # dim grows as writes come in, so a value cached at open time goes
+      # stale.
+      # @return [Integer]
+      def length = nc_inq_dimlen(@ncid, @dimid)
+
+      # @return [Boolean] true if this is the unlimited dimension
+      def unlimited? = @unlimited
+
+      # @return [Integer] raw NetCDF dimension id
+      def to_i = @dimid
+
+      # The coordinate variable of the same name in the same group.
+      # @return [Var, nil]
+      def var = @group.var(@name)
+
+      # Coordinate values (via the coordinate variable).
+      #
+      # `writable:` is passed through to {Var#to_ca}, which refuses it — the
+      # values are read into a fresh array either way.
+      #
+      # @param writable [Boolean] forwarded to the coordinate variable.
+      # @return [CArray, nil]
+      def to_ca(writable: false) = var&.to_ca(writable: writable)
+
+      # @return [String]
+      def inspect
+        "#<#{self.class} #{@name}=#{length}#{@unlimited ? '(unlimited)' : ''}>"
+      end
+    end
+  end
+end