carray-netcdf 2.0.0 → 2.1.0

data/lib/carray/netcdf.rb

@@ -3,7 +3,7 @@ module CA
   #
   #   CA::NetCDF   ← this module
   #
-  # Layer 1 (this module's functions): a CArray-flavored view of the netcdf
+  # The low-level API (this module's functions): a CArray-flavored view of the netcdf
   # gem's `nc_*` API. `extend self` exposes every method both as a module
   # function and as an instance method, so it is usable two ways:
   #
@@ -20,15 +20,15 @@ module CA
   # speak CArray; scalar (`var1`), definition, inquiry, attribute and group
   # calls fall through to {::NetCDF} verbatim.
   #
-  # Layer 1 is raw values only: no convention decoding, no mask handling.
+  # The low-level API is raw values only: no convention decoding, no mask handling.
   # Reads return unmasked CArrays; writing a masked CArray raises. The mask
-  # / CF story lives in layer 2 (the {File} / {Var} / {Codec} classes).
+  # / CF story lives in the high-level API (the {File} / {Var} / {Codec} classes).
   module NetCDF
 
     include ::NetCDF
     extend self
 
-    # @!group Layer 1 — array reads
+    # @!group Low-level API — array reads
 
     # Read an entire variable.
     #
@@ -82,17 +82,29 @@ module CA
     # @param buffer [CArray, MemoryView, nil]
     # @return [CArray]
     def nc_get_varm(ncid, varid, start, count, stride, imap, buffer = nil)
+      # A buffer this function allocates is laid out row-major over
+      # `count`; any other imap needs a buffer laid out for it.
+      if buffer.nil? && imap != row_major_imap(count)
+        raise ArgumentError,
+              "nc_get_varm with imap #{imap.inspect} needs a buffer; without one, " \
+              "imap must be row-major over count (#{row_major_imap(count).inspect})"
+      end
       ::NetCDF.nc_get_varm(ncid, varid, start, count, stride, imap,
                            buffer || new_buffer(ncid, varid, count))
     end
 
+    # @api private
+    private def row_major_imap(count)
+      count.each_index.map { |i| count[(i + 1)..].inject(1, :*) }
+    end
+
     # @!endgroup
 
-    # @!group Layer 1 — array writes
+    # @!group Low-level API — array writes
     #
     # Accept a CArray as the buffer. netcdf requires a C-contiguous buffer,
     # so non-entity views are materialized; masked input is rejected (fill
-    # encoding is layer 2's job).
+    # encoding is the high-level API's job).
 
     # Write an entire variable.
     #
@@ -141,6 +153,11 @@ module CA
         _, _, base, _, klass = nc_inq_user_type(ncid, xtype)
         xtype = base if klass == NC_ENUM
       end
+      if xtype == NC_STRING
+        raise NotImplementedError,
+              "no CArray type for an NC_STRING variable; read it with " \
+              "NetCDF.nc_get_var, which returns Strings"
+      end
       data_type = NC_TO_CA[xtype] or
         raise NotImplementedError, "no CArray type to read nc_type #{xtype} into; pass a buffer"
       CArray.new(data_type, shape.empty? ? [1] : shape)
@@ -148,14 +165,18 @@ module CA
 
     # A C-contiguous, unmasked CArray ready to hand to libnetcdf:
     # materialize non-entity views, refuse masked input (encoding a fill
-    # value is the convention layer's decision, not layer 1's).
+    # value is the codec's decision, not the low-level API's).
     # @api private
     private def writable_buffer(ca)
       return ca unless ca.is_a?(CArray)
+      # A mask with nothing masked is only carried along: drop it, so the
+      # values go out as they are.
+      return ca.strip_mask(0) if ca.has_mask? && ! ca.any_masked?
       if ca.has_mask?
         raise ArgumentError,
-              "cannot write a masked CArray at layer 1; strip the mask " \
-              "(e.g. #strip_mask(fill)) in the convention layer first"
+              "cannot write a masked CArray without a codec to encode the mask; " \
+              "strip the mask first (e.g. #strip_mask(fill)), or write through " \
+              "a codec such as CF (a Conventions attribute naming CF selects it)"
       end
       ca.entity? ? ca : ca.copy
     end
@@ -192,7 +213,8 @@ module CA
     # @param type [Symbol, String, Integer]
     # @return [Integer] an `NC_*` type constant
     # @raise [ArgumentError] on an unrecognized type name
-    def nc_type_for(type)
+    # @api private
+    private def nc_type_for(type)
       return type if type.is_a?(Integer)
       TYPE_TO_NC[type.to_s.to_sym] or
         raise ArgumentError, "unknown NetCDF variable type: #{type.inspect}"
@@ -224,7 +246,8 @@ module CA
     # @param value [Object] Ruby value or CArray
     # @param nc_type [Integer] target `NC_*` type
     # @return [void]
-    def put_att_typed(ncid, varid, name, value, nc_type)
+    # @api private
+    private def put_att_typed(ncid, varid, name, value, nc_type)
       suffix = PUT_ATT_SUFFIX[nc_type]
       return nc_put_att(ncid, varid, name, value) unless suffix
       fn = "nc_put_att_#{suffix}"
@@ -250,12 +273,69 @@ module CA
     # @api private
     private def put_att_takes_xtype?(fn) = PUT_ATT_XTYPE[fn]
 
+    # Marks an attribute {#readable_att} could not read.
+    # @api private
+    UNREADABLE_ATT = Object.new.freeze
+
+    # An attribute's value, or {UNREADABLE_ATT} with a warning when the
+    # netcdf gem cannot read its type (an enum-typed `_FillValue`, as
+    # netCDF4-python writes, is one): one such attribute should not stop
+    # the whole file from opening.
+    # @api private
+    private def readable_att(ncid, varid, name, owner)
+      value = nc_get_att(ncid, varid, name)
+      # A C writer's text often ends in a NUL; ncdump and xarray drop it.
+      # Done on the bytes: the text need not be valid UTF-8.
+      return value unless value.is_a?(String) && value.end_with?("\0")
+      String.new(value.b.sub(/\0+\z/n, ""), encoding: value.encoding)
+    rescue NotImplementedError => e
+      warn "carray-netcdf: attribute #{name.inspect} of #{owner} skipped: #{e.message}"
+      UNREADABLE_ATT
+    end
+
+    # An attribute value frozen through, so a caller changing what `atts`
+    # returned cannot change what the codec reads next.
+    # @api private
+    private def deep_freeze(value)
+      value.each { |x| x.freeze } if value.is_a?(Array)
+      value.freeze
+    end
+
+    # Whether an `_Unsigned` attribute value says "true".
+    # @api private
+    private def unsigned_flag?(value) = value.to_s.strip.downcase == "true"
+
+    # Signed bits per `NC_*` type, for {#signed_wrap}.
+    # @api private
+    SIGNED_BITS = { NC_BYTE => 8, NC_SHORT => 16, NC_INT => 32, NC_INT64 => 64 }.freeze
+
+    # An `_Unsigned` variable stores unsigned values in a signed type, and
+    # its fill / range attributes are read in unsigned space; one given in
+    # unsigned space (255 for a byte) is written as the signed value with
+    # the same bits (-1).
+    # @api private
+    private def signed_wrap(value, var_type)
+      bits = SIGNED_BITS[var_type] or return value
+      wrap = ->(x) { x.is_a?(Integer) && x >= 2**(bits - 1) && x < 2**bits ? x - 2**bits : x }
+      value.is_a?(Array) ? value.map(&wrap) : wrap.(value)
+    end
+
+    # A CArray attribute value as the flat Array the netcdf gem takes. An
+    # attribute is one-dimensional, and has no masked elements.
+    # @api private
+    private def attribute_values(name, ca)
+      if ca.any_masked?
+        raise ArgumentError, "attribute #{name.inspect} has masked elements"
+      end
+      ca.to_a.flatten
+    end
+
     # Write a variable attribute the way {Group#def_var} does:
     #
-    # - a CArray keeps its own type (`_FillValue: CA_SHORT([-1])` on a
-    #   float variable is a short attribute);
-    # - a Ruby value for a {PACKED_SPACE_ATTS} name takes the variable's
-    #   on-disk type `var_type`, which libnetcdf requires of `_FillValue`;
+    # - a value for a {PACKED_SPACE_ATTS} name, a CArray included, takes
+    #   the variable's on-disk type `var_type`, which libnetcdf requires of
+    #   `_FillValue` and CF asks of the others;
+    # - another CArray keeps its own type;
     # - anything else is left to the generic `nc_put_att` inference.
     #
     # @param ncid [Integer]
@@ -265,9 +345,11 @@ module CA
     # @param var_type [Integer] the variable's `NC_*` type
     # @return [void]
     # @api private
-    def put_var_att(ncid, varid, name, value, var_type)
+    private def put_var_att(ncid, varid, name, value, var_type, unsigned: false)
+      value = attribute_values(name, value) if value.is_a?(CArray) && PACKED_SPACE_ATTS.include?(name)
+      value = signed_wrap(value, var_type) if unsigned && PACKED_SPACE_ATTS.include?(name)
       if value.is_a?(CArray)
-        put_att_typed(ncid, varid, name, value.to_a, TYPE_TO_NC[value.data_type])
+        put_att_typed(ncid, varid, name, attribute_values(name, value), TYPE_TO_NC[value.data_type])
       elsif PACKED_SPACE_ATTS.include?(name)
         put_att_typed(ncid, varid, name, value, var_type)
       else

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: carray-netcdf
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - himotoyoshi
@@ -29,20 +29,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.2.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.2.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
 description: |
   A NetCDF interface for CArray, built on the netcdf gem.  Data is
   exchanged as CArray at every level, through Ruby's MemoryView protocol,
@@ -74,7 +74,7 @@ files:
 - docs/03_writing.md
 - docs/04_codec.md
 - docs/05_groups.md
-- docs/06_layer1.md
+- docs/06_low_level_api.md
 - docs/07_cookbook.md
 - docs/08_cheatsheet.md
 - docs/09_cli.md
@@ -97,7 +97,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/docs/06_layer1.md

@@ -1,160 +0,0 @@
-# Layer 1 — raw `nc_*` API
-
-Everyday code uses layer 2 (`File`, `Var`, ...). Layer 1 is the CArray-flavored view of the netcdf gem's `nc_*` functions — same names, same argument order, but numeric buffers are CArrays.
-
-Use it when:
-
-- You want to open by ncid or work outside the `File` object model.
-- You want to stream a very large variable into a preallocated buffer (`nc_get_vara` with an explicit buffer = zero allocation per read).
-- You're bridging to code that already speaks the C `nc_*` idiom.
-- You want to skip the codec and every layer-2 semantics for a specific operation.
-
-## The module
-
-`CA::NetCDF` itself IS layer 1. It `include`s `::NetCDF` (the netcdf gem's module) so all `nc_*` functions and `NC_*` constants pass through, and overrides only the array I/O to speak CArray.
-
-```ruby
-require "carray-netcdf"
-NC = CA::NetCDF
-```
-
-Two idioms:
-
-```ruby
-# (a) call style — module functions
-NC.nc_get_vara(ncid, varid, [0, 0], [3, 4])   # => CArray
-NC::NC_DOUBLE                                  # constants inherited from ::NetCDF
-
-# (b) mixin style — `nc_*` become bare methods in your class
-class Reader
-  include CA::NetCDF
-  def read_all(ncid, varid) = nc_get_var(ncid, varid)   # bare, returns CArray
-  def define(ncid, name)    = nc_def_dim(ncid, name, 100)   # passes through
-end
-```
-
-## What layer 1 overrides vs passes through
-
-Overridden (numeric array I/O):
-
-- `nc_get_var`, `nc_get_vara`, `nc_get_vars`, `nc_get_varm`
-- `nc_put_var`, `nc_put_vara`, `nc_put_vars`, `nc_put_varm`
-
-Everything else — `nc_open`, `nc_create`, `nc_def_dim`, `nc_def_var`, `nc_put_att`, `nc_get_att`, `nc_inq_*`, `nc_def_grp`, `nc_close`, `NC_DOUBLE`, `NC_CLOBBER`, ... — comes straight from the netcdf gem, which this layer includes without modifying.
-
-## `nc_get_var*` — buffer-less returns a CArray
-
-Without a buffer argument, layer 1 allocates a CArray of the requested shape, typed from the variable's on-disk type (`NC_CHAR` as `:uint8`, an enum as its base type), and reads into it:
-
-```ruby
-ncid = ::NetCDF.nc_open("data.nc", ::NetCDF::NC_NOWRITE)
-vid  = ::NetCDF.nc_inq_varid(ncid, "temp")
-
-ca = NC.nc_get_var(ncid, vid)                          # whole variable  -> CArray
-ca = NC.nc_get_vara(ncid, vid, [0, 0], [3, 4])         # hyperslab       -> CArray
-ca = NC.nc_get_vars(ncid, vid, [0, 0], [3, 4], [1, 2]) # strided         -> CArray
-ca = NC.nc_get_varm(ncid, vid, [0, 0], [3, 4], [1, 1], [4, 1])  # mapped -> CArray
-
-::NetCDF.nc_close(ncid)
-```
-
-No codec runs — this is raw, no `_FillValue → UNDEF`, no scale/offset, no CATime. Use layer 2 for those.
-
-A type with no CArray counterpart — opaque, `NC_STRING` — raises `NotImplementedError`; read an opaque variable into a buffer of your own.
-
-## `nc_get_var*` — with a buffer, reads into it
-
-Pass an existing CArray and layer 1 asks netcdf to read into that buffer. Perfect for streaming a slab at a time with **zero allocation per read**:
-
-```ruby
-ncid = ::NetCDF.nc_open("big.nc", ::NetCDF::NC_NOWRITE)
-vid  = ::NetCDF.nc_inq_varid(ncid, "temp")
-
-buf = CArray.float32(73, 144)   # one time slice
-
-365.times do |day|
-  NC.nc_get_vara(ncid, vid, [day, 0, 0], [1, 73, 144], buf)
-  process(buf)                  # buf now holds day `day`
-end
-
-::NetCDF.nc_close(ncid)
-```
-
-The buffer must be C-contiguous (a plain entity CArray is; a view is not — the gem will refuse a non-contiguous buffer). Type conversion is handled by libnetcdf — you can read a short variable into a float64 buffer and libnetcdf converts.
-
-## `nc_put_var*` — CArray as buffer
-
-Symmetric. A contiguous CArray goes through as-is:
-
-```ruby
-NC.nc_put_vara(ncid, vid, [0, 0], [3, 4], CArray.float32(3, 4).seq!)
-```
-
-A non-contiguous CArray (a view / slice / reference) is materialized to a contiguous entity before the write:
-
-```ruby
-big = CArray.float32(100, 4).seq!
-NC.nc_put_vara(ncid, vid, [0, 0], [3, 4], big[10..12, nil])
-# big[10..12, nil] is a CABlock view; layer 1 does .copy on your behalf
-# and passes the resulting entity to nc_put_vara.
-```
-
-A **masked** CArray is *refused*. Layer 1 does no fill encoding — that's layer 2's job (via the codec). Strip the mask first:
-
-```ruby
-ca = CA_SHORT([1, 2, 3])
-ca[1] = UNDEF
-NC.nc_put_vara(ncid, vid, [0], [3], ca)          # ArgumentError
-NC.nc_put_vara(ncid, vid, [0], [3], ca.strip_mask(-32768))    # OK, fill baked in
-```
-
-## `nc_get_var1` / `nc_put_var1` — scalar
-
-These are pure passthrough (the netcdf gem handles them directly), included in layer 1 so the mixin style has them:
-
-```ruby
-value = NC.nc_get_var1(ncid, vid, [0, 0])         # Numeric
-NC.nc_put_var1(ncid, vid, [0, 0], 273.15)         # writes one cell
-```
-
-Layer 2's scalar decode/encode wraps these calls in a 1-cell CArray to run through the codec — see [Codecs](04_codec.md).
-
-## Definition, inquiry, attribute — all passthrough
-
-The netcdf gem covers everything else. In layer-1 code, use them bare (via mixin) or fully qualified:
-
-```ruby
-# create + define
-ncid = NC.nc_create("out.nc", NC::NC_CLOBBER | NC::NC_NETCDF4)
-tid  = NC.nc_def_dim(ncid, "time", 24)
-vid  = NC.nc_def_var(ncid, "temp", NC::NC_FLOAT, [tid])
-NC.nc_put_att(ncid, vid, "units", "K")
-NC.nc_enddef(ncid)
-
-# write and close
-NC.nc_put_var(ncid, vid, CArray.float32(24).seq!)
-NC.nc_close(ncid)
-
-# inquire
-ncid = NC.nc_open("out.nc", NC::NC_NOWRITE)
-NC.nc_inq_ndims(ncid)                              # => 1
-NC.nc_inq_nvars(ncid)                              # => 1
-NC.nc_inq_varname(ncid, 0)                         # => "temp"
-NC.nc_inq_vardimid(ncid, 0)                        # => [0]
-NC.nc_inq_dimlen(ncid, 0)                          # => 24
-NC.nc_close(ncid)
-```
-
-Any of these can also be reached as `::NetCDF.nc_*` (unchanged), or through layer 2's higher-level surface.
-
-## Coexisting with `::NetCDF` and future `Numo::NetCDF`
-
-Layer 1 does **not** modify `::NetCDF`. It only adds behavior in the `CA::NetCDF` namespace. So an app that mixes carray-netcdf with a hypothetical `Numo::NetCDF` gem (which would follow the same pattern in its own namespace) can call:
-
-```ruby
-CA::NetCDF.nc_get_vara(ncid, vid, s, c)   # -> CArray
-Numo::NetCDF.nc_get_vara(ncid, vid, s, c)     # -> Numo::NArray  (hypothetical)
-::NetCDF.nc_get_vara(ncid, vid, s, c, buf)    # -> buf, any MemoryView producer (untouched)
-```
-
-with no clashes. This coexistence is exactly why layer 1 avoids monkey-patching `::NetCDF`.