carray 3.0.6 → 3.0.7

data/lib/carray/methods/repeat.rb

@@ -59,7 +59,17 @@ class CArray
         end
         CArray.int64(width).fill(count)
       else
-        given = count.to_ca.int64
+        unless count.respond_to?(:to_ca)
+          raise TypeError, "repeat: count must be an Integer or a list of them (got #{count.class})"
+        end
+        given = count.to_ca
+        # A list arrives as an object array; a Float in it is no count, as a
+        # float array is not.
+        unless given.integer? || (given.data_type == CA_OBJECT &&
+                                  given.to_a.flatten.all? { |c| c.is_a?(Integer) || c.equal?(UNDEF) })
+          raise TypeError, "repeat: counts must be Integers"
+        end
+        given = given.int64
         unless given.elements == width
           raise ArgumentError,
                 "repeat: #{given.elements} counts for #{width} " \

data/lib/carray/record_order.rb

@@ -52,20 +52,20 @@ class CARecord
     define_method(op, &Ractor.make_shareable(proc { |*args, **kw| record_rank.public_send(op, *args, **kw) }))
   end
 
-  # @overload min(axis: nil, keep_axis: false)
+  # @overload min(axis: nil, keep_axis: false, min_count: nil, fill_value: nil)
   #   Returns the smallest record by the declared order, skipping masked
   #   records; UNDEF when every record is masked.  With `axis:` a
   #   {CARecord} of the smallest along that axis.
   #   @return [CAStruct, CARecord]
-  def min (axis: nil, keep_axis: false)
-    record_extreme(:min_index, axis, keep_axis)
+  def min (axis: nil, keep_axis: false, min_count: nil, fill_value: nil)
+    record_extreme_kept(:min_index, axis, keep_axis, min_count, fill_value)
   end
 
-  # @overload max(axis: nil, keep_axis: false)
+  # @overload max(axis: nil, keep_axis: false, min_count: nil, fill_value: nil)
   #   The largest record; see {#min}.
   #   @return [CAStruct, CARecord]
-  def max (axis: nil, keep_axis: false)
-    record_extreme(:max_index, axis, keep_axis)
+  def max (axis: nil, keep_axis: false, min_count: nil, fill_value: nil)
+    record_extreme_kept(:max_index, axis, keep_axis, min_count, fill_value)
   end
 
   # @overload minmax(axis: nil, keep_axis: false)
@@ -162,6 +162,28 @@ class CARecord
   end
 
   # The record at each min_index / max_index, gathered from self.
+  # An extreme under the reduction keywords every extreme takes: UNDEF where
+  # fewer than `min_count` cells are present, then `fill_value` for UNDEF.
+  def record_extreme_kept (index_op, axis, keep_axis, min_count, fill_value)
+    out  = record_extreme(index_op, axis, keep_axis)
+    name = index_op.to_s.sub("_index", "").to_sym
+    need = CArray.__present_cells_needed_per_result_cell__(name, min_count, 0)
+    if need > 0
+      if axis.nil?
+        out = UNDEF if count_not_masked < need
+      else
+        counts = count_not_masked(axis: axis, keep_axis: keep_axis)
+        if counts.is_a?(CArray)
+          few = counts.lt(need)
+          out[few] = UNDEF if few.any
+        elsif counts < need
+          out = UNDEF
+        end
+      end
+    end
+    CArray.__fill_undefined_result_cells__(out, fill_value)
+  end
+
   def record_extreme (index_op, axis, keep_axis)
     rank = record_rank
     if axis.nil?
@@ -175,6 +197,13 @@ class CARecord
       return record_extreme_over(index_op, axes, keep_axis)
     end
     axis = normalize_axis(axis, name)
+    if ndim == 1
+      r = record_extreme(index_op, nil, false)
+      return r unless keep_axis
+      out = flatten[0..0].copy
+      out[0] = r
+      return out
+    end
     idx   = rank.public_send(index_op, axis: axis)
     empty = idx.is_masked
     picked_shape = shape.dup

data/lib/carray/runtime.rb

@@ -172,5 +172,15 @@ class CArray
   end
   private_class_method :define_named_methods
 
+  # CArray.promote_list, its refusal named after the verb that gave it the
+  # list (stack, meld, concatenate, ...), not after promote_list.
+  def self.promote_list_for (verb, list, data_type: nil)
+    CArray.promote_list(list, data_type: data_type)
+  rescue ArgumentError, CArray::DataTypeError => e
+    raise unless e.message.start_with?("promote_list: ")
+    raise e.class, "#{verb}: #{e.message.delete_prefix("promote_list: ")}"
+  end
+  private_class_method :promote_list_for
+
 end
 

data/lib/carray/segment_iterator.rb

@@ -125,8 +125,7 @@ class CASegmentIterator < CAIterator
     # CABlock, whose own #count is the block geometry accessor, so dispatch
     # CArray#count explicitly. (Not fused: a value-equality reduceat would have
     # to reproduce core's cross-type / out-of-range equality exactly.)
-    cnt = CArray.instance_method(:count)
-    per_segment(CA_INT64) { |s| cnt.bind_call(s, *args) }
+    per_segment(CA_INT64) { |s| count_cells(s, *args) }
   end
 
   # @overload count_masked
@@ -233,8 +232,10 @@ class CASegmentIterator < CAIterator
   #   as five length-k float64 CArrays (matching `CArray#quantile`): the
   #   percentiles at 0 / 25 / 50 / 75 / 100. Empty / all-masked segments are
   #   MASKED. For a single fraction q in 0..1 use `percentile(q * 100)`.
+  #   @param axis [Integer, nil] as for {#percentile}.
   #   @return [Array<CArray>]
-  def quantile
+  def quantile (axis: nil)
+    return CArray.__name_errors_after_method__(:quantile) { [0, 25, 50, 75, 100].map { |p| percentile(p, axis: axis) } } if axis
     unless kernel_reads?
       return CArray.__name_errors_after_method__(:quantile) { [0, 25, 50, 75, 100].map { |p| percentile(p) } }
     end
@@ -297,7 +298,8 @@ class CASegmentIterator < CAIterator
   #   iff every present value is truthy (empty segment -> true, vacuously).
   #   The value data type must be boolean, as for `CArray#all`.
   #   @return [CArray]
-  def all
+  def all (axis: nil)
+    return axis_by_core(axis, :all) if axis
     aa = all_any
     aa ? aa[:all] : per_segment(CA_BOOLEAN) { |s| s.all }
   end
@@ -307,7 +309,8 @@ class CASegmentIterator < CAIterator
   #   iff some present value is truthy (empty segment -> false). The value
   #   data type must be boolean, as for `CArray#any`.
   #   @return [CArray]
-  def any
+  def any (axis: nil)
+    return axis_by_core(axis, :any) if axis
     aa = all_any
     aa ? aa[:any] : per_segment(CA_BOOLEAN) { |s| s.any }
   end
@@ -731,7 +734,9 @@ class CASegmentIterator < CAIterator
 
   def scan_answered (op)
     # An empty Face has no cell to ask the core about, and nothing to scan.
-    return blank_output(@src_shape) if @value.face? && @value.elements.zero?
+    if @value.face? && @value.elements.zero?
+      return op == :cumcount ? CArray.int64(*@src_shape) : blank_output(@src_shape)
+    end
     answer = core_scan_answer(op)
     unless kernel_scans?(op, answer)
       return scatter_segments(->(shape) { blank_like(answer, shape) }) { |s| s.public_send(op) }
@@ -757,7 +762,7 @@ class CASegmentIterator < CAIterator
   # (the core counts a boolean's running sum in uint64).
   def kernel_scans? (op, answer)
     return false unless CArray.__fast_paths_enabled__(:segment_scan_kernel)
-    return false if @value.face? || answer.face? || @value.complex?
+    return false if @value.face? || answer.face? || @value.complex? || @value.data_type == CA_FIXLEN
     kernel_type =
       case op
       when :cumcount         then CA_INT64
@@ -798,10 +803,10 @@ class CASegmentIterator < CAIterator
   end
 
   # A blank array of +shape+ of the same kind as +model+: its data type, or its
-  # Face. A read-only Face cannot be written cell by cell, so its values are
-  # held as the surface objects they are.
+  # Face. A Face that holds no new value cannot be written cell by cell, so its
+  # values are held as the surface objects they are.
   def blank_like (model, shape)
-    if model.face? && model.read_only?
+    if model.face? && !model.__send__(:holds_new_values?)
       CArray.new(CA_OBJECT, shape)
     else
       model.__send__(:blank_like, shape)
@@ -1042,10 +1047,10 @@ class CASegmentIterator < CAIterator
 
   # Whether an output can be built by lifting one. A Face is filled by writing
   # surface values into storage, so this needs a Face that can be written
-  # into; a read-only one -- a const string's records index a shared pool, a
+  # into; another -- a const string's records index a shared pool, a
   # categorical's codes index a vocabulary -- has no blank form to fill.
   def face_output?
-    @value.face? && ! @value.read_only?
+    @value.face? && @value.__send__(:holds_new_values?)
   end
 
   # Build a length-k typed output by folding each segment's members with the
@@ -1056,8 +1061,6 @@ class CASegmentIterator < CAIterator
   # for free (an all-masked segment reduces like an empty one; identity-bearing
   # reductions return their identity, ratios return UNDEF).
   def per_segment (kind)
-    # a data type the values' own Face answers in names that Face
-    kind = @value if ! kind.is_a?(CArray) && @value.face? && kind == @value.data_type
     data_type = kind.is_a?(CArray) ? kind.data_type : kind
     out = if kind.nil?
             CArray.new(CA_OBJECT, [@k])
@@ -1066,13 +1069,13 @@ class CASegmentIterator < CAIterator
                     elsif data_type == @value.data_type then @value.bytes
                     else 0 end
             CArray.new(data_type, [@k], bytes: bytes)
-          elsif ! kind.read_only?
+          elsif kind.__send__(:holds_new_values?)
             # the core answered in a Face, so the output is one too:
             # CATime#min hands back a CATime::Element, which only a CATime
             # has anywhere to put
             kind.__send__(:blank_like, [@k])
           else
-            # a read-only Face cannot be filled, so its answers are collected
+            # a Face that holds no new value cannot be filled, so its answers are collected
             # as the surface objects they already are
             CArray.new(CA_OBJECT, [@k])
           end

data/lib/carray/serialize.rb

@@ -662,6 +662,12 @@ class CArray
     case tag
     when "object"
       shape, values, mask, attrs = rest
+      # The shape is only a claim until the cells are seen to fill it, and
+      # a damaged one can ask for more memory than the machine has.
+      unless marshal_cells_fill_shape?(values, shape) &&
+             (mask.nil? || marshal_cells_fill_shape?(mask, shape))
+        raise ArgumentError, "corrupt CArray Marshal payload (the cells do not fill shape #{shape.inspect})"
+      end
       ca = CArray.object(*shape)
       ca[] = values
       if mask
@@ -683,6 +689,22 @@ class CArray
   end
   private_class_method :array_from_marshal_payload
 
+  # Whether `cells`, nested as #to_a nests them, has the extent of every axis
+  # of `shape`.  A cell may itself hold an Array, so only shape.size levels
+  # are counted.
+  def self.marshal_cells_fill_shape? (cells, shape)
+    return false unless shape.is_a?(Array) && !shape.empty? &&
+                        shape.all? { |extent| extent.is_a?(Integer) && extent >= 0 }
+    level = [cells]
+    shape.each do |extent|
+      return false unless level.all? { |row| row.is_a?(Array) && row.size == extent }
+      level = level.flat_map { |row| row }
+      return true if level.empty?
+    end
+    true
+  end
+  private_class_method :marshal_cells_fill_shape?
+
 end
 
 # Marshal of a view, a mask and a scalar: the {CArray#marshal_dump} payload,

data/lib/carray/slab_iterator.rb

@@ -204,7 +204,7 @@ class CASlabIterator < CAIterator
     return count_not_masked(**kw) if args.empty?
     # CABlock / CAWindow shadow #count with a geometry accessor, and @reference
     # may be such a view, so dispatch CArray#count explicitly for count(v).
-    CArray.instance_method(:count).bind_call(@reference, *args, axis: @slab_axes,
+    count_cells(@reference, *args, axis: @slab_axes,
                                              **CAIterator.reduce_keywords(:count, kw))
   end
 

data/lib/carray/stack.rb

@@ -60,7 +60,7 @@ class CArray
   def self.stack (list, axis: 0, data_type: nil)
     raise ArgumentError, "stack: list must not be empty" if list.empty?
     axis = 0 if axis.nil?
-    list = CArray.promote_list(list, data_type: data_type)
+    list = promote_list_for("stack", list, data_type: data_type)
     axis = CArray.normalize_axis(axis, list[0].ndim + 1, "stack")
     CAStack.new(list, axis: axis)                # CAStack.new does Face lift internally
   end
@@ -101,7 +101,7 @@ class CArray
       raise ArgumentError, "meld: entries must be CArray (got #{first.class})"
     end
     axis_norm = CArray.normalize_axis(axis, first.ndim, "meld")
-    arrays = CArray.promote_list(arrays, data_type: data_type)
+    arrays = promote_list_for("meld", arrays, data_type: data_type)
     # Flatten nested CAMeld inputs that share our meld axis: they already
     # describe a segment sequence, so absorbing their parents keeps chain
     # depth at 1 (avoids 2-level xfer_all / reduce chains through the
@@ -183,7 +183,7 @@ class CArray
             "montage: tdim product (#{expected}) must equal list size (#{list.size})"
     end
 
-    list = CArray.promote_list(list, data_type: data_type)
+    list = promote_list_for("montage", list, data_type: data_type)
     parent_shape = list[0].shape
     ntile = tdim.size
     nparent = parent_shape.size

data/lib/carray/string.rb

@@ -115,22 +115,22 @@ class CAString
   def mod (other) ; as_text(super) ; end
   # The later String of each cell and `other`, cell by cell.
   # @return [CAString]
-  def pmax (other) ; as_text(super) ; end
+  def pmax (other) ; refuse_unordered_operands("pmax", other) ; as_text(super) ; end
   # The earlier String of each cell and `other`, cell by cell.
   # @return [CAString]
-  def pmin (other) ; as_text(super) ; end
+  def pmin (other) ; refuse_unordered_operands("pmin", other) ; as_text(super) ; end
   # Another name of {#pmax} (a String has no NaN to let win).
   # @return [CAString]
-  def maximum (other) ; as_text(super) ; end
+  def maximum (other) ; refuse_unordered_operands("maximum", other) ; as_text(super) ; end
   # Another name of {#pmin}.
   # @return [CAString]
-  def minimum (other) ; as_text(super) ; end
+  def minimum (other) ; refuse_unordered_operands("minimum", other) ; as_text(super) ; end
   # Another name of {#pmax}.
   # @return [CAString]
-  def fmax (other) ; as_text(super) ; end
+  def fmax (other) ; refuse_unordered_operands("fmax", other) ; as_text(super) ; end
   # Another name of {#pmin}.
   # @return [CAString]
-  def fmin (other) ; as_text(super) ; end
+  def fmin (other) ; refuse_unordered_operands("fmin", other) ; as_text(super) ; end
 
   # The String a blank cell holds: the cell a fresh {#template} gives and
   # the one {#shift} fills by default.  Frozen, because every blank cell

data/lib/carray/string_operation_extension.rb

@@ -44,6 +44,25 @@ class CArray
     end
     private :string_map
 
+    # An ordering (pmax and its kin, clip) with a value that is not a String
+    # is refused as String#< refuses it.  nil (no bound, for clip), UNDEF,
+    # an Array and a CArray are left to the operation.
+    def refuse_unordered_operands (verb, *operands)
+      operands.each do |v|
+        next if v.nil? || v.is_a?(String) || v.is_a?(Array) || v.is_a?(CArray) || UNDEF.equal?(v)
+        name = v.is_a?(Symbol) ? v.inspect : v.class
+        raise ArgumentError, "#{verb}: comparison of String with #{name} failed"
+      end
+    end
+    private :refuse_unordered_operands
+
+    # Clips the strings in byte order (see {CArray#clip}); a bound that is
+    # not a String is refused.
+    def clip (min, max = nil, *rest, **kw)
+      refuse_unordered_operands("clip", min, max)
+      super
+    end
+
     # ---- transforms (String result -> CAString) --------------------------
 
     # Per-cell `String#upcase`.
@@ -93,7 +112,7 @@ class CArray
     def encode (*a)    ; string_map { |s| s.encode(*a) } ; end
     # Per-cell `String#force_encoding`.
     # @return [CAString] shape and mask of `self` preserved.
-    def force_encoding (e) ; string_map { |s| s.force_encoding(e) } ; end
+    def force_encoding (e) ; string_map { |s| s.dup.force_encoding(e) } ; end
     # Per-cell `String#scrub`.
     # @return [CAString] shape and mask of `self` preserved.
     def scrub ()       ; string_map { |s| s.scrub } ; end
@@ -221,14 +240,16 @@ class CArray
       end
       entity = CArray.new(CA_FIXLEN, shape, :bytes => width)
       values.each_with_index { |s, i| entity[i] = s.nil? ? UNDEF : s }
-      CAFixlenString.wrap(entity)
+      text = respond_to?(:encoding) ? encoding : CAFixlenString.__send__(:encoding_of_strings, values)
+      CAFixlenString.wrap(entity, encoding: text)
     end
 
     # Self (zero-copy) when already a {CAConstString} of the same encoding;
     # otherwise materialises a compact column.
-    # @param encoding [Encoding] column encoding.
+    # @param encoding [Encoding] column encoding; defaults to the encoding
+    #   the array carries, and to UTF-8 for one that carries none.
     # @return [CAConstString]
-    def to_const_string (encoding: Encoding::UTF_8)
+    def to_const_string (encoding: respond_to?(:encoding) ? self.encoding : Encoding::UTF_8)
       if is_a?(CAConstString) && encoding == self.encoding
         return self
       end

data/lib/carray/struct_builder.rb

@@ -699,7 +699,10 @@ class CAStruct::Builder  # :nodoc:
   #   Declares a bitfield member. Bits accumulate within the
   #   current byte; when a byte member follows, the cursor rounds
   #   up to the next byte boundary. Only supported in `:struct`
-  #   (not `:union`) and always requires a name.
+  #   (not `:union`) and always requires a name.  A member of more
+  #   than one bit keeps the low `bits` bits of an Integer written to
+  #   it (300 into 3 bits is 4), through an element or the member's
+  #   column alike; a one-bit member is boolean.
   #   @param name [Symbol, String] member name.
   #   @param bits [Integer] bit width in `1..64`.
   #   @return [void]