carray-netcdf 2.0.0 → 2.1.0

data/lib/carray/netcdf/codec.rb

@@ -1,3 +1,5 @@
+require "date"
+
 module CA
   module NetCDF
     # Value codec — the encode/decode half of a NetCDF "convention".
@@ -57,10 +59,15 @@ module CA
           when :auto
             # The whole string first, then each of its tokens, so
             # "ACDD-1.3, CF-1.8" still finds CF.
+            # A non-String attribute (an NC_STRING array, a number) is
+            # tried only through its String tokens.
             conv = file.att("Conventions")
-            hit  = conv && (@registry.find { |c| c.claims?(conv) } ||
-                            @registry.find { |c| file.conventions.any? { |t| c.claims?(t) } })
-            hit ? build(hit) : Identity
+            conv = nil unless conv.is_a?(String)
+            hit  = (conv && @registry.find { |c| c.claims?(conv) }) ||
+                   @registry.find { |c| file.conventions.any? { |t| c.claims?(t) } }
+            return build(hit) if hit
+            warn_cf_without_conventions(file) if conv.nil? && file.conventions.empty?
+            Identity
           when :identity, false, nil
             Identity
           when Symbol
@@ -82,6 +89,23 @@ module CA
 
         private
 
+        # Attributes that make a variable look CF-encoded.
+        CF_LOOKING_ATTS = %w[_FillValue missing_value scale_factor add_offset].freeze
+
+        # xarray and netCDF4-python write CF attributes but no Conventions,
+        # so such a file reads as stored. Say so once, and how to decode it.
+        def warn_cf_without_conventions(file)
+          v = file.vars.each_value.find do |var|
+            atts = var.atts
+            CF_LOOKING_ATTS.any? { |k| atts.key?(k) } ||
+              (atts["units"].is_a?(String) && atts["units"].include?(" since "))
+          end
+          return unless v
+          warn "carray-netcdf: #{file.filename} has CF attributes (on #{v.name.inspect}) " \
+               "but no Conventions attribute naming CF or COARDS, so values are read " \
+               "as stored; open it with codec: :cf to decode them"
+        end
+
         def codec?(obj) = obj.respond_to?(:decode) && obj.respond_to?(:encode)
 
         def symbol_class(sym)
@@ -142,7 +166,7 @@ module CA
           ca = decode_unsigned(var, ca)                          # signed disk -> unsigned user
           ca = decode_fill(var, a, ca)                           # (in unsigned/packed space)
           ca = decode_valid(var, a, ca) if @mask_valid_range     # (same)
-          ca = decode_scale(a, ca)                               # packed -> physical
+          ca = decode_scale(var, a, ca)                          # packed -> physical
           ca = decode_time(var, a, ca)  if time_units?(a["units"])
           ca
         end
@@ -154,6 +178,7 @@ module CA
         def encode(var, ca)
           a  = var.atts
           ca = encode_time(var, a, ca)  if time_units?(a["units"])
+          ca = mask_nan(var, ca)
           ca = encode_scale(var, a, ca)
           ca = encode_fill(var, a, ca)
           ca = encode_unsigned(var, ca)                          # unsigned user -> signed disk
@@ -285,15 +310,36 @@ module CA
 
         # `unpacked = packed * scale_factor + add_offset` (CF §8.1)
         # @api private
-        private def decode_scale(atts, ca)
+        private def decode_scale(var, atts, ca)
           sf = atts["scale_factor"]
           ao = atts["add_offset"]
+          return ca unless sf || ao
+          # CF: the unpacked type is the attributes'. float32 attributes
+          # unpack in float32, as xarray and netCDF4-python do.
+          if !ca.float? || ca.data_type == :float32
+            names = %w[scale_factor add_offset].select { |k| atts.key?(k) }
+            if names.all? { |k| var.respond_to?(:att_type) && var.att_type(k) == ::NetCDF::NC_FLOAT }
+              return unpack(ca.to_type(:float32), sf, ao)
+            end
+          end
+          # Integer attributes keep integer arithmetic in the packed type,
+          # which overflows (a short times 200); widen to int64 first.
+          if INTEGER_TYPES.include?(ca.data_type) && (sf.is_a?(Integer) || ao.is_a?(Integer))
+            ca = ca.to_type(:int64)
+          end
+          unpack(ca, sf, ao)
+        end
+
+        # @api private
+        private def unpack(ca, sf, ao)
           ca = ca * sf if sf
           ca = ca + ao if ao
           ca
         end
 
         INTEGER_TYPES = %i[int8 uint8 int16 uint16 int32 uint32 int64 uint64].freeze
+        INT64_MIN = -2**63
+        INT64_MAX = 2**63 - 1
 
         # `packed = round((unpacked - add_offset) / scale_factor)`
         # (A bare cast to an integer variable would truncate:
@@ -303,18 +349,41 @@ module CA
           sf = atts["scale_factor"]
           ao = atts["add_offset"]
           return ca unless sf || ao
+          # Integer values would be divided as integers (truncating) and
+          # could overflow their type; work in float64 and round below.
+          ca = ca.to_type(:float64) if INTEGER_TYPES.include?(ca.data_type)
           ca = ca - ao if ao
           ca = ca / sf if sf
           ca = ca.round if INTEGER_TYPES.include?(var.data_type)
           ca
         end
 
+        # NaN is how float data marks a missing value (xarray writes it so).
+        # An integer variable cannot hold it — cast, it would become a
+        # valid-looking 0 — so it is masked here and written as the fill.
+        # @api private
+        private def mask_nan(var, ca)
+          return ca unless ca.is_a?(CArray) && ca.float? && INTEGER_TYPES.include?(var.data_type)
+          nan = ca.is_nan
+          return ca unless nan.any
+          ca = ca.copy
+          ca[nan] = UNDEF
+          ca
+        end
+
         # ---- mask <-> fill --------------------------------------------------
 
         # @api private
         private def encode_fill(var, atts, ca)
           return ca unless ca.is_a?(CArray) && ca.has_mask?
-          fv = fill_values(var, atts).first or
+          # Values still in the signed on-disk type take the stored fill;
+          # unsigned ones take it reinterpreted, as fill_values gives it.
+          fv = if unsigned?(var) && ca.data_type == var.data_type
+                 atts["_FillValue"] || Array(atts["missing_value"]).first
+               else
+                 fill_values(var, atts).first
+               end
+          fv or
             raise ArgumentError,
                   "variable '#{var.name}' has masked values but no _FillValue " \
                   "to encode them"
@@ -383,10 +452,55 @@ module CA
           return nil unless m
           count = (m[1] || "1").to_i
           base  = UNIT_WORDS[m[2].downcase] or return nil
-          [count, base, ref.strip]
+          # UDUNITS writes short years ("days since 1-1-1"); CATime would
+          # read a one- or two-digit year as 20xx / 19xx, so pad it.
+          ref = ref.strip.sub(/\A(-?)(\d{1,3})(?=-)/) { "#{$1}#{$2.rjust(4, '0')}" }
+          [count, base, ref]
+        end
+
+        # The unit to work in for a reference time: `base` when the
+        # reference falls on its grid, else the first finer fixed-length
+        # unit it falls on (`"days since 2000-01-01 12:00"` works in hours).
+        # nil when there is none — a `:Y` / `:M` base whose reference is not
+        # the head of a year / month. Cached per instance.
+        # @api private
+        private def origin_unit(base, ref)
+          @origin_units ||= {}
+          key = [base, ref]
+          return @origin_units[key] if @origin_units.key?(key)
+          units = NS_PER_UNIT.key?(base) ? NS_PER_UNIT.keys.drop_while { |u| u != base } : [base]
+          @origin_units[key] = units.find { |u| origin_on_grid?(u, ref) }
         end
 
-        # See docs/codec.md §"N-of-a-unit intervals" for why we fold
+        # The first day of the Gregorian calendar; under `standard` /
+        # `gregorian` (CF's mixed calendar) a date before it is Julian.
+        GREGORIAN_REFORM = Date.new(1582, 10, 15, Date::GREGORIAN)
+
+        # A reference time before the reform, in the mixed calendar, is a
+        # Julian date. CATime is proleptic Gregorian, so the same day is
+        # relabelled (`1-1-1` is `0000-12-30`); every instant from the reform
+        # on then comes out as cftime / UDUNITS give it. -> [ref, shifted?]
+        # @api private
+        private def mixed_calendar_reference(ref, calendar)
+          return [ref, false] unless calendar.nil? || %w[standard gregorian].include?(calendar.to_s.strip.downcase)
+          m = ref.match(/\A(-?\d+)-(\d{1,2})-(\d{1,2})(.*)\z/m) or return [ref, false]
+          y, mo, d = m[1].to_i, m[2].to_i, m[3].to_i
+          return [ref, false] unless Date.valid_date?(y, mo, d, Date::JULIAN)
+          julian = Date.new(y, mo, d, Date::JULIAN)
+          return [ref, false] unless julian < GREGORIAN_REFORM
+          g = julian.gregorian
+          return [format("%04d-%02d-%02d%s", g.year, g.month, g.day, m[4]), true]
+        end
+
+        # @api private
+        private def origin_on_grid?(unit, ref)
+          CA_INT64([0]).time(unit: unit, origin: ref)
+          return true
+        rescue ArgumentError
+          return false
+        end
+
+        # See docs/04_codec.md §"N-of-a-unit intervals" for why we fold
         # count into the raw values instead of storing at `count × base`
         # resolution.
         # @api private
@@ -409,15 +523,61 @@ module CA
             return ca
           end
           count, base, ref = parsed
-          vals = count == 1 ? ca : ca * count
-          if vals.float?
-            unit, ticks = time_ticks(vals, base)
-            return ticks.time(unit: unit, origin: ref)
+          ref, shifted = mixed_calendar_reference(ref, calendar)
+          unit = origin_unit(base, ref)
+          unless unit
+            warn_once(:origin, units,
+                      "variable #{var.name.inspect} has units #{units.inspect} " \
+                      "whose reference time is not on the #{base} grid; " \
+                      "raw values returned")
+            return ca
+          end
+          # Values in `count × base` steps, counted in `unit`.
+          f = count * (NS_PER_UNIT.key?(base) ? NS_PER_UNIT[base] / NS_PER_UNIT[unit] : 1)
+          if ca.float?
+            vals = f == 1 ? ca : ca * f
+            tick_unit, ticks = time_ticks(vals, unit)
+            return check_reform(var, ticks.time(unit: tick_unit, origin: ref), shifted)
           end
-          ints = vals.data_type == :int64 ? vals : vals.int64
-          ints.time(unit: base, origin: ref)
+          ints = ca.data_type == :int64 ? ca : ca.int64
+          # xarray writes NaT as the smallest int64, with no _FillValue.
+          if ca.data_type == :int64 && ! atts.key?("_FillValue")
+            nat = ints.eq(INT64_MIN)
+            if nat.any
+              ints = ints.copy
+              ints[nat] = UNDEF
+            end
+          end
+          unless f == 1
+            # Counted in a finer unit, a value may no longer fit int64.
+            limit = ints.abs.max
+            if limit.is_a?(Integer) && limit > INT64_MAX / f
+              warn_once(:overflow, units,
+                        "variable #{var.name.inspect} has values too large to count " \
+                        "in #{unit} for units #{units.inspect}; raw values returned")
+              return ca
+            end
+            ints = ints * f
+          end
+          check_reform(var, ints.time(unit: unit, origin: ref), shifted)
+        end
+
+        # With a pre-reform reference, an instant before the reform is a
+        # Julian date that CATime shows as proleptic Gregorian. Say so.
+        # @api private
+        private def check_reform(var, t, shifted)
+          if shifted && (t < reform_instant).any
+            warn_once(:reform, var.name,
+                      "variable #{var.name.inspect} has times before 1582-10-15 " \
+                      "in the mixed Julian/Gregorian calendar; they are shown as " \
+                      "proleptic Gregorian, not as the Julian dates the file means")
+          end
+          t
         end
 
+        # @api private
+        private def reform_instant = (@reform_instant ||= CArray.time(["1582-10-15"], unit: :D)[0])
+
         # Length of each fixed-length unit, in ns, finest last. `:Y` / `:M`
         # have no fixed length, so a fraction of one can't be carried to a
         # finer unit.
@@ -436,8 +596,11 @@ module CA
           eps   = vals.data_type == :float32 ? 1.2e-7 : 2.3e-16
           limit = vals.abs.max
           limit = 0.0 unless limit.is_a?(Numeric)          # all masked
+          # Whole within the rounding error of the values themselves; an
+          # absolute floor would round away real fractions (50 ms of a day).
+          scale = [limit, 1.0].max
           whole = ->(x, f) { d = (x - x.round).abs.max
-                             !d.is_a?(Numeric) || d <= [1e-6, 4 * limit * f * eps].max }
+                             !d.is_a?(Numeric) || d <= 4 * scale * f * eps }
           return [base, vals.round.int64] if whole.(vals, 1) || !NS_PER_UNIT.key?(base)
           best = [base, vals]
           NS_PER_UNIT.each_key.drop_while { |u| u != base }.drop(1).each do |u|
@@ -465,21 +628,52 @@ module CA
                   "cannot encode CATime into variable #{var.name.inspect}: " \
                   "units #{units.inspect} is not a CF time-since spec"
           count, base, ref = parsed
+          ref, shifted = mixed_calendar_reference(ref, calendar)
+          if shifted && (ca < reform_instant).any
+            raise ArgumentError,
+                  "cannot encode CATime into variable #{var.name.inspect}: instants " \
+                  "before 1582-10-15 would be Julian dates in its mixed calendar; " \
+                  "use calendar: \"proleptic_gregorian\""
+          end
+          unit = origin_unit(base, ref) or
+            raise ArgumentError,
+                  "cannot encode CATime into variable #{var.name.inspect}: " \
+                  "the reference time of units #{units.inspect} is not on the #{base} grid"
+          return encode_calendar_time(var, ca, count, base, ref, units) unless NS_PER_UNIT.key?(base)
+          # A month or year instant is a real instant on a finer grid.
+          ca   = ca.to_unit(unit) unless NS_PER_UNIT.key?(ca.unit.base)
           fine = ca.unit.base
-          if finer_unit?(fine, base)
-            # The instants are finer than the units' interval: work in the
-            # finer unit so a fraction of an interval isn't truncated.
-            f     = NS_PER_UNIT[base] / NS_PER_UNIT[fine] * count
-            ticks = ca.timesteps(unit: fine, origin: ref)
-            return ticks.float64 / f unless INTEGER_TYPES.include?(var.data_type)
-            unless (ticks % f).eq(0).all
-              raise ArgumentError,
-                    "cannot encode CATime into integer variable #{var.name.inspect}: " \
-                    "some instants fall between the steps of units #{units.inspect}"
-            end
-            return ticks / f
+          work = finer_unit?(fine, unit) ? fine : unit
+          # One `count × base` step, in `work` ticks.
+          f     = NS_PER_UNIT[base] / NS_PER_UNIT[work] * count
+          ticks = ca.timesteps(unit: work, origin: ref)
+          return (f == 1 ? ticks : ticks.float64 / f) unless INTEGER_TYPES.include?(var.data_type)
+          return ticks if f == 1
+          unless (ticks % f).eq(0).all
+            raise ArgumentError,
+                  "cannot encode CATime into integer variable #{var.name.inspect}: " \
+                  "some instants fall between the steps of units #{units.inspect}"
+          end
+          ticks / f
+        end
+
+        # Encode into `months since` / `years since` units. A month or year
+        # has no fixed length, so an instant between two steps can't be
+        # written, even into a float variable, and raises.
+        # @api private
+        private def encode_calendar_time(var, ca, count, base, ref, units)
+          warn_once(:calendar_units, units,
+                    "variable #{var.name.inspect} is written in #{units.inspect} as " \
+                    "calendar months / years; xarray and cftime refuse such units under " \
+                    "this calendar (UDUNITS takes a month as 30.436875 days), so other " \
+                    "tools may not read it")
+          on_grid = ca.to_unit(base).to_unit(ca.unit.base)
+          unless on_grid.ticks.eq(ca.ticks).all
+            raise ArgumentError,
+                  "cannot encode CATime into variable #{var.name.inspect}: " \
+                  "some instants fall between the steps of units #{units.inspect}"
           end
-          ticks = ca.timesteps(unit: base, origin: ref)
+          ticks = ca.to_unit(base).timesteps(unit: base, origin: ref)
           return ticks if count == 1
           unless (ticks % count).eq(0).all
             raise ArgumentError,
@@ -498,7 +692,7 @@ module CA
 
         # @api private
         private def supported_calendar?(cal)
-          cal.nil? || SUPPORTED_CALENDARS.include?(cal.to_s)
+          cal.nil? || SUPPORTED_CALENDARS.include?(cal.to_s.strip.downcase)
         end
 
         # Per-instance cache (a new CF instance is built per

data/lib/carray/netcdf/dim.rb

@@ -5,6 +5,9 @@ module CA
     class Dim
 
       include CA::NetCDF
+      # The low-level calls are this class's helpers, not its interface.
+      private :nc_get_var, :nc_get_vara, :nc_get_vars, :nc_get_varm,
+              :nc_put_var, :nc_put_vara, :nc_put_vars, :nc_put_varm
 
       # @param group [Group] containing group
       # @param dimid [Integer] NetCDF dimension id within `group`
@@ -17,11 +20,10 @@ module CA
         @name   = nc_inq_dimname(@ncid, @dimid)
         # Whether a dim is unlimited is a define-time property and doesn't
         # change over the file's lifetime — cache it once. Asked of the
-        # dim's own group, which is where netcdf-4 keeps it.
-        # nc_inq_unlimdim names only the group's first unlimited dim; a
-        # further one is known here only when this gem defined it.
+        # dim's own group, which is where netcdf-4 keeps it; a NetCDF-4
+        # group can have several.
         @unlimited = if unlimited.nil?
-                       @dimid == (nc_inq_unlimdim(@ncid) rescue -1)
+                       (nc_inq_unlimdims(@ncid) rescue []).include?(@dimid)
                      else
                        unlimited
                      end
@@ -37,7 +39,7 @@ module CA
       # dim grows as writes come in, so a value cached at open time goes
       # stale.
       # @return [Integer]
-      def length = nc_inq_dimlen(@ncid, @dimid)
+      def length = (@group.file.ensure_open; nc_inq_dimlen(@ncid, @dimid))
 
       # @return [Boolean] true if this is the unlimited dimension
       def unlimited? = @unlimited
@@ -60,6 +62,7 @@ module CA
 
       # @return [String]
       def inspect
+        return "#<#{self.class} #{@name} (closed)>" if @group.file.closed?
         "#<#{self.class} #{@name}=#{length}#{@unlimited ? '(unlimited)' : ''}>"
       end
     end