carray 3.0.4 → 3.0.6

data/lib/carray/serialize.rb

@@ -141,7 +141,15 @@ class CArray::Serializer   # :nodoc:
             "keep its values with Marshal.dump(ca.to_type(:object))"
     end
 
-    file_endian = endian || CArray.endian
+    file_endian = case endian
+                  when nil, :native        then CArray.endian
+                  when :big, CA_BIG_ENDIAN then CA_BIG_ENDIAN
+                  when :little, CA_LITTLE_ENDIAN then CA_LITTLE_ENDIAN
+                  else
+                    raise ArgumentError,
+                          "endian: must be :big, :little, :native, " \
+                          "CA_BIG_ENDIAN or CA_LITTLE_ENDIAN (got #{endian.inspect})"
+                  end
     swap        = (file_endian != CArray.endian)
 
     elements     = ca.elements
@@ -383,6 +391,7 @@ class CArray::Serializer   # :nodoc:
       "members"      => members,
     }
     schema["name"] = klass.name if klass.name
+    schema["order_by"] = klass.order_by.map(&:to_s) if klass.order_by
     schema
   end
 
@@ -439,7 +448,15 @@ class CArray::Serializer   # :nodoc:
     members      = schema["members"]
     record_bytes = schema["record_bytes"]
     builder      = (schema["kind"] == "union") ? CArray.method(:union) : CArray.method(:struct)
-    builder.call(:pack => 1, :size => record_bytes) do
+    order_by     = schema["order_by"]
+    names        = members.map { |m| m["name"].to_s }
+    if order_by && !(order_by.is_a?(Array) && order_by.all? { |n| names.include?(n.to_s) })
+      raise ArgumentError, "corrupt CArray binary data (order_by #{order_by.inspect} " \
+            "names a member the data_class does not have)"
+    end
+    opts = { :pack => 1, :size => record_bytes }
+    opts[:order_by] = order_by.map(&:to_sym) if order_by
+    builder.call(**opts) do
       members.each do |m|
         sym = PEP_TO_SYMBOL[m["type"]]
         unless sym
@@ -668,11 +685,11 @@ class CArray
 
 end
 
-# Marshal of a view and a scalar: the {CArray#marshal_dump} payload, loaded
-# back by the class method rather than into an empty instance.  A view comes
-# back as a plain array (a copy owns its memory, as #copy gives); a scalar
-# comes back a scalar.  A wrap is refused below.
-[CAView, CAWrap, CScalar].each do |klass|
+# Marshal of a view, a mask and a scalar: the {CArray#marshal_dump} payload,
+# loaded back by the class method rather than into an empty instance.  A
+# view, a wrap and a mask come back as a plain array (a copy owns its
+# memory, as #copy gives); a scalar comes back a scalar.
+[CAView, CAWrap, CScalar, CArrayMask, CAObjectMask].each do |klass|
   klass.class_eval do
     # A Face other than a record is refused: the payload would carry its
     # storage without what makes it the Face (a unit, labels), and load
@@ -686,28 +703,45 @@ end
               "dump its values with to_type(:object)" +
               (storage ? ", or its storage, which ##{storage} gives" : "")
       end
+      return dump_payload unless data_type == CA_OBJECT
+      # An object view dumps its cells by a nested Marshal.dump, which does
+      # not know the objects the outer dump has written.  A cell holding the
+      # view again (directly or through other views) would dump it without
+      # end, so dumping the same view from inside its own dump is refused,
+      # as reading such a cell is.
+      refuse_reentry("Marshal.dump") { dump_payload }
+    end
+
+    private
+
+    # Marshal writes the instance variables of an object that has _dump
+    # after the String it returns, unless the String has instance
+    # variables of its own.  A view keeps its parent in one, so a view of
+    # a large array would carry the whole parent, and a parent holding the
+    # view in a cell would be an object cycle.  The String is given one of
+    # its own, which _load ignores.
+    def dump_payload
       Marshal.dump(CArray.instance_method(:marshal_dump).bind_call(self))
+             .tap { |s| s.instance_variable_set(:@carray_payload, true) }
     end
   end
 end
 
-class CAView
+[CAView, CArrayMask, CAObjectMask].each do |klass|
   # @return [CArray] the plain array a {#_dump} string describes.
-  def self._load (str)
-    CArray.send(:array_from_marshal_payload, Marshal.load(str))
+  # Defined with def, not define_singleton_method: a block's method keeps
+  # the block's self and cannot be called from another Ractor.
+  klass.singleton_class.class_eval do
+    def _load (str)
+      CArray.send(:array_from_marshal_payload, Marshal.load(str))
+    end
   end
 end
 
 class CAWrap
-  # A wrap borrows memory, which Marshal cannot carry to another process;
-  # Ruby would also try to write the objects that keep the borrowed memory
-  # alive.  Its copy owns its memory and marshals.
-  def _dump (_level)
-    raise TypeError,
-          "Marshal cannot carry a #{self.class}: it borrows its memory; " \
-          "dump .copy instead"
-  end
-
+  # A wrap borrows its memory, which Marshal cannot carry to another
+  # process: {#_dump} above writes its values, and the load gives a plain
+  # array that owns a copy of them, as it does for a view of the wrap.
   def self._load (str)
     CArray.send(:array_from_marshal_payload, Marshal.load(str))
   end

data/lib/carray/stack.rb

@@ -69,27 +69,28 @@ class CArray
   # No data is copied; reads gather from parents on demand and writes flow
   # back to them (chain composability preserved).
   #
-  # Pieces must agree on `ndim`, `data_type`, byte width, and every axis
-  # length except `axis` (the "meld axis").  Mismatched `data_type` raises:
-  # cast the pieces yourself (`.to_type(:float64)`) or use
-  # {CArray.concatenate} (eager, auto-casts).
+  # Pieces must agree on `ndim` and every axis length except `axis` (the
+  # "meld axis").  Pieces of different data types are brought to a common
+  # one as {CArray.stack} brings them (`promote_list`), through read-only
+  # views of the pieces that need it.
   #
   # "meld" = melt + weld — pieces dissolve their boundaries along the named
   # axis and are regarded as one.
   #
-  # @overload meld(*arrays, axis: 0)
-  # @overload meld(list, axis: 0)
+  # @overload meld(*arrays, axis: 0, data_type: nil)
+  # @overload meld(list, axis: 0, data_type: nil)
   #   Convenience form: a single Array argument is treated as the list.
   #   @param arrays [Array<CArray>] pieces to weld.  A single Array
   #     argument is accepted for compatibility with older callers.
   #   @param axis [Integer, nil] existing axis to extend (normalises
   #     negative values against the reference ndim); `nil` is the same
   #     as `0`.
+  #   @param data_type [Symbol, Integer, nil] result `data_type`;
+  #     inferred when `nil`.
   #   @return [CAMeld] view over the welded pieces.
   #   @raise [ArgumentError] when the list is empty, ndim mismatch,
-  #     data_type mismatch, or non-axis dim mismatch across pieces
-  #     (surfaced by CAMeld.new).
-  def self.meld (*arrays, axis: 0)
+  #     or non-axis dim mismatch across pieces (surfaced by CAMeld.new).
+  def self.meld (*arrays, axis: 0, data_type: nil)
     if arrays.length == 1 && arrays[0].is_a?(Array)
       arrays = arrays[0]
     end
@@ -100,15 +101,19 @@ class CArray
       raise ArgumentError, "meld: entries must be CArray (got #{first.class})"
     end
     axis_norm = CArray.normalize_axis(axis, first.ndim, "meld")
+    arrays = CArray.promote_list(arrays, data_type: data_type)
     # Flatten nested CAMeld inputs that share our meld axis: they already
     # describe a segment sequence, so absorbing their parents keeps chain
     # depth at 1 (avoids 2-level xfer_all / reduce chains through the
     # intermediate CAMeld).  A CAMeld with a different meld_axis is left
-    # intact — its segment structure is orthogonal.
-    if arrays.any? { |a| a.is_a?(CAMeld) && a.meld_axis == axis_norm }
-      arrays = arrays.flat_map { |a|
-        a.is_a?(CAMeld) && a.meld_axis == axis_norm ? a.parents : [a]
-      }
+    # intact — its segment structure is orthogonal.  So is a read-only one
+    # (frozen or flagged): writes through it are refused, and absorbing it
+    # would drop that.  The mask of a meld is a CAMeld with no parents of
+    # its own; it is passed on, to be refused as a part.
+    absorbed = ->(a) { a.is_a?(CAMeld) && a.meld_axis == axis_norm &&
+                       !a.read_only? && !a.mask_array? }
+    if arrays.any?(&absorbed)
+      arrays = arrays.flat_map { |a| absorbed.(a) ? a.parents : [a] }
     end
     CAMeld.new(arrays, axis: axis_norm)
   end
@@ -119,14 +124,15 @@ class CArray
   # Instance form of {CArray.meld}; non-destructive, see the class method
   # for full semantics.
   #
-  # @overload meld(*others, axis: 0)
+  # @overload meld(*others, axis: 0, data_type: nil)
   #   @param others [Array<CArray>] additional pieces.
   #   @param axis [Integer] existing axis to extend.
+  #   @param data_type [Symbol, Integer, nil] result `data_type`.
   #   @return [CAMeld]
   #   @raise [ArgumentError] when no `others` are given.
-  def meld (*others, axis: 0)
+  def meld (*others, axis: 0, data_type: nil)
     raise ArgumentError, "meld: at least one other array required" if others.empty?
-    CArray.meld(self, *others, axis: axis)
+    CArray.meld(self, *others, axis: axis, data_type: data_type)
   end
 
   # Arrange `list` of uniform-shape pieces in a `tdim`-shape grid that
@@ -288,10 +294,19 @@ class CAStack
   #   preserving `self.k_axis`. Distinct from `CArray#stack`, which
   #   always treats `self` as a single parent of a new K-stack.
   #   @param others [Array<CArray>] extra parents to append.
+  #   A read-only stack hands on read-only views of its parents, so
+  #   writes into its cells stay refused.
   #   @return [CAStack]
   #   @raise [ArgumentError] when no `others` are given.
   def append (*others)
     raise ArgumentError, "append: at least one parent required" if others.empty?
-    CArray.stack(self.parents + others, axis: self.k_axis)
+    if mask_array?
+      raise ArgumentError, "append: a mask array can have no mask and can not be a part"
+    end
+    mine = self.parents
+    if read_only?
+      mine = mine.map { |p| p[].tap(&:set_read_only_flag) }
+    end
+    CArray.stack(mine + others, axis: self.k_axis)
   end
 end

data/lib/carray/string.rb

@@ -57,6 +57,14 @@ class CArray
       else
         values = Array.new(n) { |i| block.call(i) }
       end
+    elsif arg.is_a?(Integer)
+      raise ArgumentError,
+            "CArray.string: a count is given with a block that makes each " \
+            "String (CArray.string(n) { |i| ... })"
+    elsif arg.is_a?(String) || ! arg.respond_to?(:to_a)
+      raise TypeError,
+            "CArray.string: expected an Array of Strings, an object array or " \
+            "a String array (got #{arg.class})"
     else
       values = arg.to_a
     end

data/lib/carray/string_operation_extension.rb

@@ -31,11 +31,14 @@ class CArray
     # (integer / boolean results).  Shape and mask are carried.
     def string_map (data_type = nil) # :nodoc:
       out = data_type ? template(data_type) : CArray.object(*shape)
+      # A typed result has no nil: a cell the String method answers nil for
+      # (index finds nothing) is UNDEF.
+      cell = data_type ? ->(s) { v = yield(s); v.nil? ? UNDEF : v } : ->(s) { yield(s) }
       if has_mask?
         m = is_masked
-        elements.times { |i| out[i] = (m[i] ? UNDEF : yield(self[i])) }
+        elements.times { |i| out[i] = (m[i] ? UNDEF : cell.(self[i])) }
       else
-        elements.times { |i| out[i] = yield(self[i]) }
+        elements.times { |i| out[i] = cell.(self[i]) }
       end
       data_type ? out : CAString.wrap(out)
     end
@@ -124,10 +127,12 @@ class CArray
     # @return [CArray] `:int`, shape and mask of `self` preserved.
     def bytesize ()    ; string_map(:int) { |s| s.bytesize } ; end
     # Per-cell `String#index`.
-    # @return [CArray] `:int`, shape and mask of `self` preserved.
+    # @return [CArray] `:int`, shape and mask of `self` preserved; UNDEF
+    #   where the cell does not hold the substring.
     def str_index (*a) ; string_map(:int) { |s| s.index(*a) } ; end
     # Per-cell `String#rindex`.
-    # @return [CArray] `:int`, shape and mask of `self` preserved.
+    # @return [CArray] `:int`, shape and mask of `self` preserved; UNDEF
+    #   where the cell does not hold the substring.
     def rindex (*a)    ; string_map(:int) { |s| s.rindex(*a) } ; end
 
     # ---- numeric parse (Integer / Float result) --------------------------
@@ -191,6 +196,9 @@ class CArray
       unless [:error, :silent].include?(truncate)
         raise ArgumentError, "truncate: must be :error or :silent (got #{truncate.inspect})"
       end
+      unless bytes.nil? || (bytes.is_a?(Integer) && bytes >= 1)
+        raise ArgumentError, "to_fixlen_string: invalid bytes: #{bytes.inspect} (a cell needs bytes: of 1 or more)"
+      end
       return self if is_a?(CAFixlenString) && (bytes.nil? || bytes == self.bytes)
 
       if has_mask?
@@ -200,7 +208,7 @@ class CArray
         values = Array.new(elements) { |i| self[i].to_s }
       end
       width = bytes || values.compact.map(&:bytesize).max || 1
-      width = 1 if width < 1
+      width = 1 if width < 1   # every string empty: the narrowest cell
       if truncate == :error
         values.each_with_index do |s, i|
           next if s.nil?

data/lib/carray/struct_builder.rb

@@ -224,11 +224,13 @@ class CAStruct::Builder  # :nodoc:
     end
     vtype = {1 => :uint8, 2 => :uint16, 4 => :uint32, 8 => :uint64}[view_bytes]
     range = bit_in_byte..(bit_in_byte + bits - 1)
+    # The word is read little-endian on every host: bits are numbered
+    # byte by byte from the record's start, as the layout places them.
     reader = ->(data) {
-      data.field(start_byte, vtype).bitfield(range)[0]
+      data.field(start_byte, vtype).endian(:little).bitfield(range)[0]
     }
     writer = ->(data, val) {
-      data.field(start_byte, vtype).bitfield(range)[0] = val
+      data.field(start_byte, vtype).endian(:little).bitfield(range)[0] = val
     }
     [reader, writer].freeze
   end
@@ -241,7 +243,7 @@ class CAStruct::Builder  # :nodoc:
   def self.build_nested_struct_dispatcher (offset, nested_class)
     reader = ->(data) {
       element = nested_class.allocate
-      element.instance_variable_set(:@data, data.field(offset, nested_class).parent)
+      element.instance_variable_set(:@data, data.field(offset, nested_class).send(:face_storage))
       element.freeze if data.frozen?
       element
     }