carray 3.0.2 → 3.0.3

data/lib/carray/methods/mode.rb

@@ -65,12 +65,25 @@ class CArray
   #   Only meaningful for discrete / binned data: raw float is almost all
   #   unique, so every value is modal and the per-axis Array grows to the
   #   fiber length. Bin first. Same NaN / mask semantics as {#is_mode}.
+  #   A complex array raises {CArray::DataTypeError}, since the modes
+  #   are listed in ascending order; {#is_mode} marks its modal cells.
   #
   #   @param axis [Integer, nil] axis to take the mode along; `nil` uses
   #     the whole array.
   #   @return [CArray, Array<CArray>] 1-D CArray (flat) or an Array of
   #     reduced CArrays, one per mode rank (per-axis).
   def mode (axis: nil)
+    # An error from the methods mode is made of (sort, ...) names mode.
+    CArray.__answer_as__(:mode) { mode_answered(axis) }
+  end
+
+  private def mode_answered (axis)
+    if complex?
+      raise CArray::DataTypeError,
+            "mode: the modes are listed in ascending order, and a " \
+            "#{data_type_name} value has no order (complex numbers are not " \
+            "ordered); is_mode marks the cells that hold a mode"
+    end
     return __mode_flat if axis.nil?
     k = normalize_axis(axis, "mode")
 
@@ -120,7 +133,11 @@ class CArray
   def __mode_flat
     values, counts = value_counts
     return values if values.elements == 0
-    values[counts.eq(counts.max)].sort
+    modes = values[counts.eq(counts.max)]
+    # Records are sorted only by the members their struct declares in
+    # order_by:; without one the modes stay in first-appearance order.
+    return modes if is_a?(CARecord) && data_class.order_by.nil?
+    modes.sort
   end
 
 end

data/lib/carray/methods/repeat.rb

@@ -76,26 +76,7 @@ class CArray
       end
 
     source = along ? self : (ndim > 1 ? flatten : self)
-    source[*gather_args(repeat_index(counts), along)]
-  end
-
-  #  The source index of every position in the result: element i appears
-  #  at the `counts[i]` positions starting at the exclusive prefix sum.
-  #
-  #  Marking those starts with +1 and running a cumulative sum turns the
-  #  marks back into indices, and does the right thing with a zero
-  #  count for free -- two elements then share a start, that position
-  #  carries 2, and the sum steps over the one that was asked for zero
-  #  times.
-  private def repeat_index (counts)
-    total = counts.sum.to_i
-    return CArray.int64(0) if total.zero?
-    starts = counts.cumsum.int64 - counts
-    marks  = CArray.int64(total)
-    #  A trailing zero count starts one past the end; it marks nothing,
-    #  which is what a zero count means.
-    marks.scatter_add!(starts[starts.lt(total)], 1)
-    marks.cumsum.int64 - 1
+    source[*gather_args(CArray.segment_index(lengths: counts), along)]
   end
 
   #  One index argument per axis: the gathered index on the axis being

data/lib/carray/methods/resize.rb

@@ -4,8 +4,8 @@ class CArray
   #   Returns `self` resized to `newdim`. The original data is
   #   placed at offset 0 (positive size) or right-aligned (negative
   #   size); the new area outside the original region is filled
-  #   with `fill_value`, or -- for fixlen storage -- left as zero
-  #   bytes. Pass `UNDEF` to mask the new area.
+  #   with `fill_value`, or left as zero storage when it is not given
+  #   (0 for a numeric array). Pass `UNDEF` to mask the new area.
   #
   #   Each entry of `newdim` is: `nil` to keep the current
   #   `shape[i]`, a positive Integer for a new size with the
@@ -13,15 +13,19 @@ class CArray
   #   `d` for a new size `|d|` with the original right-aligned.
   #
   #   Works on Face arrays (CARecord / CATime / ...) and
-  #   plain fixlen: the resize is done on the storage layout
-  #   (preserving bytes) and re-wrapped as the same Face.
+  #   plain fixlen: the result is the same Face, and `fill_value` is a
+  #   value of it (a Time for a time array).
   #
   #   @param newdim [Array<Integer, nil>] new shape spec, one entry
   #     per axis.
-  #   @param fill_value [Object] value for the extended area.
+  #   @param fill_value [Object] value for the extended area (zero
+  #     storage when not given).
   #   @return [CArray] resized copy.
-  def resize (*newdim, fill_value: 0)
-    raise "ndim mismatch" if newdim.size != ndim
+  NO_FILL_VALUE = ::Object.new.freeze
+  private_constant :NO_FILL_VALUE
+
+  def resize (*newdim, fill_value: NO_FILL_VALUE)
+    raise ArgumentError, "ndim mismatch" if newdim.size != ndim
     offset = Array.new(ndim, 0)
     newdim = newdim.each_with_index.map do |d, i|
       case d
@@ -32,24 +36,18 @@ class CArray
         offset[i] = size - shape[i] if d < 0
         size
       else
-        raise "invalid dimension size"
+        raise ArgumentError, "invalid dimension size"
       end
     end
-    face_parent = self.face? ? self : nil
-    src = self
-    src = src.parent while src.face?
-    dt    = src.data_type
-    bytes = (dt == :fixlen) ? src.bytes : nil
-    out = CArray.new(dt, newdim, bytes: bytes)
-    # Fill the new area: numeric storage takes fill_value as-is; fixlen
-    # storage cannot hold a numeric 0, so leave zero bytes and honor only
-    # UNDEF (mask) or an explicit String fill.
-    if dt != :fixlen || fill_value.equal?(UNDEF) || fill_value.is_a?(String)
-      out[] = fill_value
-    end
-    out.mask.paste(offset, src.false) if out.has_mask?
-    out.paste(offset, src)
-    out = out.face_lift(face_parent) if face_parent
+    # The original cells are copied as storage, so a read-only Face (a
+    # const string, a categorical) resizes too.  The new area starts as zero
+    # storage; a given fill_value is written as a value of the array, which a
+    # read-only Face refuses.
+    out = __blank_like__(newdim)
+    out_storage = out.__send__(:__face_storage__)
+    out[] = fill_value unless fill_value.equal?(NO_FILL_VALUE)
+    out_storage.mask.paste(offset, self.false) if out_storage.has_mask?
+    out_storage.paste(offset, __face_storage__)
     out
   end
 

data/lib/carray/methods/snap.rb

@@ -66,8 +66,11 @@ class CArray
   #   the sample, `:ceil` at or above), and delegates to
   #   `locate_nearest_addr(direction:)`.
   #
-  #   `list` must be a 1-D ascending numeric CArray (or convertible via
-  #   `CArray.wrap_readonly`). Out-of-range and NaN handling follows the
+  #   `list` must be a 1-D ascending numeric CArray, or a Ruby Array /
+  #   Range.  An Array grid takes `self`'s data_type, except that a Float
+  #   in it makes an integer array's grid float64 (so `0.5` is not
+  #   truncated); an Integer that does not fit `self`'s type raises
+  #   `RangeError`.  Out-of-range and NaN handling follows the
   #   same pattern as `bin_to`, but with an additional `:clamp`
   #   sentinel that snaps out-of-range cells to the nearest list end.
   #
@@ -95,19 +98,21 @@ class CArray
   #     shape as `self`.
   #   @raise [ArgumentError] when `list` is not 1-D or is empty, or
   #     `direction` is not one of the accepted symbols.
+  #   @raise [RangeError] when an Integer in an Array `list` does not
+  #     fit `self`'s data_type.
   #   @example
   #     temp.snap_to([270.0, 280.0, 290.0, 300.0])                   # clamp OOB
   #     temp.snap_to(grid, lfill: nil, ufill: nil)                   # mask OOB
   #     rain.snap_to([0.0, 1.0, 5.0, 20.0], direction: :floor)       # list value at or below
   def snap_to(list, lfill: :clamp, ufill: :clamp, direction: :round)
-    ref = list.is_a?(CArray) ? list : CArray.wrap_readonly(list, self.data_type)
+    ref = list.is_a?(CArray) ? list : snap_list(list)
     raise ArgumentError, "snap_to: list must be 1-D" unless ref.ndim == 1
     n = ref.elements
     raise ArgumentError, "snap_to: list must have at least one value" if n < 1
 
     if n == 1
       # Degenerate: every finite cell snaps to the only value.
-      out = CArray.new(ref.data_type, shape).fill(ref[0])
+      out = CArray.new(ref.data_type, shape, bytes: ref.bytes).fill(ref[0])
       out.mask = self.mask.to_ca if self.has_mask?
       if self.float?
         inv = self.is_invalid
@@ -118,17 +123,16 @@ class CArray
       return out
     end
 
-    # `locate_nearest_addr` (via `linear_section`) accepts a 1-D `val` only;
-    # flatten multi-D input and reshape the result back to preserve the
-    # element-wise semantic on any shape.
+    # Multi-D input is snapped as its flattened cells and given its shape
+    # back.
     if ndim > 1
       return reshape(-1).snap_to(ref, lfill: lfill, ufill: ufill,
                                  direction: direction).reshape(*shape)
     end
 
-    # locate_nearest_addr returns int64 indices; OOB (below / above / NaN)
-    # cells come back masked. We split OOB into below / above with
-    # explicit comparisons so the two sides can be filled independently.
+    # locate_nearest_addr returns int64 indices; whether a cell outside the
+    # list gets one depends on the direction, so the two sides are split with
+    # explicit comparisons and filled here independently of it.
     idx = self.locate_nearest_addr(ref, direction: direction)
     out = ref.project(idx)
 
@@ -137,13 +141,13 @@ class CArray
 
     case lfill
     when :clamp then out[below] = ref[0]
-    when nil    then # leave masked (locate_nearest_addr already masked OOB)
+    when nil    then out[below] = UNDEF
     else             out[below] = lfill
     end
 
     case ufill
     when :clamp then out[above] = ref[-1]
-    when nil    then # leave masked
+    when nil    then out[above] = UNDEF
     else             out[above] = ufill
     end
 
@@ -158,4 +162,22 @@ class CArray
     out
   end
 
+  private
+
+  # A Ruby Array grid in the type a search query would get: a Float in it
+  # makes an integer array's grid float64 (so 0.5 is not truncated to 0),
+  # otherwise the grid takes self's type, and an Integer that does not fit
+  # it is refused rather than wrapped.
+  def snap_list(list)
+    vals = list.to_a
+    type = data_type
+    type = CA_FLOAT64 if (integer? || data_type == CA_BOOLEAN) && vals.any? { |v| v.is_a?(Float) }
+    ref = CArray.wrap_readonly(vals, type)
+    if ref.integer?
+      bad = vals.zip(ref.to_a).find { |v, w| v != w }
+      raise RangeError, "snap_to: #{bad[0].inspect} is out of range for #{ref.data_type_name}" if bad
+    end
+    ref
+  end
+
 end

data/lib/carray/methods/unique.rb

@@ -26,10 +26,13 @@ class CArray
   #   For `CA_OBJECT` / `CA_FIXLEN` distinctness follows Ruby
   #   `eql?` / `hash`; note that Ruby does not collapse distinct NaN
   #   objects, so a `CA_OBJECT` array of Float NaN is not collapsed
-  #   (unlike the numeric path).
+  #   (unlike the numeric path). Complex values follow the float rules
+  #   in each component; the values come back with -0.0 written as 0.0
+  #   and every NaN as one NaN.
   #
   #   @param sort [Boolean] when true, return the distinct values
-  #     sorted ascending instead of in first-appearance order.
+  #     sorted ascending instead of in first-appearance order. A complex
+  #     array has no order and raises {CArray::DataTypeError}.
   #   A time array (`CATime` / `CATimedelta`) answers with its own type on
   #   its own unit: the distinct values are values, so the array comes back
   #   as itself rather than as raw storage ticks.
@@ -80,6 +83,12 @@ class CArray
     # collapsed and -0.0 / +0.0 normalized; object keys on rb_hash + rb_eql and
     # fixlen on a byte-hash + memcmp, both reproducing Ruby Hash distinctness.
     # Masked cells are skipped in the kernel.
+    if sort && complex?
+      raise CArray::DataTypeError,
+            "unique(sort: true): a #{data_type_name} value has no order " \
+            "(complex numbers are not ordered); unique without sort: gives " \
+            "the values in first-appearance order"
+    end
     levels = __unique_flat__
     sort ? levels.sort : levels
   end

data/lib/carray/record_order.rb

@@ -0,0 +1,208 @@
+# ----------------------------------------------------------------------------
+#
+#  carray/record_order.rb
+#
+#  The order of a CARecord: the members its struct names in `order_by:`,
+#  compared in that order.
+#
+#  A record's storage is its bytes, and comparing bytes is no order of the
+#  values in them (a little-endian float, a signed integer).  So the
+#  ordering members do not run on the storage: they run on one int64 array,
+#  each record's rank among the others by the declared members, and the
+#  values they return are gathered from the record array by the positions
+#  that array gives.  A struct that declares no order has none, and these
+#  raise.
+#
+# ----------------------------------------------------------------------------
+
+class CARecord
+
+  # @overload sort(axis: nil, kind: :quick, masked_position: :last)
+  #   Returns a sorted {CARecord} view, ordered by the struct's
+  #   `order_by:` members.  With no `axis:` the array is flattened first.
+  #   @return [CARecord]
+  #   @raise [ArgumentError] when the struct declares no `order_by:`.
+  def sort (*args, **kw)
+    addr = record_rank.sort_addr(*args, **kw)
+    kw[:axis].nil? ? self[addr.flatten] : self[addr]
+  end
+
+  # @overload sort_copy(axis: nil, kind: :quick, masked_position: :last)
+  #   @return [CARecord] an owned sorted copy.
+  def sort_copy (*args, **kw)
+    sort(*args, **kw).copy
+  end
+
+  # @!method sort_addr(axis: nil, kind: :quick, masked_position: :last)
+  #   @return [CArray] the view-flat addresses of a sort by the declared order.
+  # @!method sort_index(axis: nil, kind: :quick, masked_position: :last)
+  #   @return [CArray] the per-fiber indices of a sort by the declared order.
+  # @!method rank_index(axis: nil)
+  #   @return [CArray] each record's rank by the declared order.
+  # @!method order(axis: nil, descending: false, method: :ordinal, kind: :quick)
+  #   @return [CArray] each record's rank by the declared order.
+  # @!method min_index(axis: nil)
+  #   @return [Integer, CArray] where the smallest record sits.
+  # @!method max_index(axis: nil)
+  #   @return [Integer, CArray] where the largest record sits.
+  # @!method partition_index(kth, axis: nil)
+  #   @return [CArray] the indices that partition about the `kth` record.
+  [:sort_addr, :sort_index, :rank_index, :order,
+   :min_index, :max_index, :partition_index].each do |op|
+    define_method(op) { |*args, **kw| record_rank.public_send(op, *args, **kw) }
+  end
+
+  # @overload min(axis: nil, keep_axis: false)
+  #   Returns the smallest record by the declared order, skipping masked
+  #   records; UNDEF when every record is masked.  With `axis:` a
+  #   {CARecord} of the smallest along that axis.
+  #   @return [CAStruct, CARecord]
+  def min (axis: nil, keep_axis: false)
+    record_extreme(:min_index, axis, keep_axis)
+  end
+
+  # @overload max(axis: nil, keep_axis: false)
+  #   The largest record; see {#min}.
+  #   @return [CAStruct, CARecord]
+  def max (axis: nil, keep_axis: false)
+    record_extreme(:max_index, axis, keep_axis)
+  end
+
+  # @overload minmax(axis: nil, keep_axis: false)
+  #   @return [Array] `[min, max]`.
+  def minmax (axis: nil, keep_axis: false)
+    [min(axis: axis, keep_axis: keep_axis), max(axis: axis, keep_axis: keep_axis)]
+  end
+
+  # @overload partition_copy(kth, axis: nil)
+  #   @return [CARecord] partitioned about the `kth` record in the declared
+  #     order.  With no `axis:` the array is flattened first.
+  def partition_copy (kth, axis: nil)
+    if axis.nil?
+      flat = flatten
+      return flat[flat.send(:record_rank).partition_index(kth)].copy
+    end
+    take_along_axis(record_rank.partition_index(kth, axis: axis), axis: axis).copy
+  end
+
+  # @!method lt(other)
+  #   Compares by the declared order.  `other` is a record of the same
+  #   struct, or a {CARecord} of them.
+  #   @return [CArray] boolean.
+  # @!method le(other)
+  # @!method gt(other)
+  # @!method ge(other)
+  [:lt, :le, :gt, :ge].each do |op|
+    define_method(op) do |other|
+      mine, theirs = record_rank_with(other)
+      mine.public_send(op, theirs)
+    end
+  end
+
+  alias <  lt
+  alias <= le
+  alias >  gt
+  alias >= ge
+
+  private
+
+  # The struct's order_by: members, or a refusal naming where to declare it.
+  def record_order_by
+    data_class.order_by or
+      raise ArgumentError,
+            "#{data_class.inspect} declares no order; pass order_by: to " \
+            "CArray.struct to name the members records are ordered by"
+  end
+
+  # Each record's dense rank by the declared members, shaped like self.
+  # Each member is ranked on its own (a numeric member by value, NaN last;
+  # a fixlen member by its bytes), and the ranks are folded in member order.
+  # Ranking again after each fold keeps every value below the record count.
+  def record_rank
+    record_rank_of(self).reshape(*shape)
+  end
+
+  def record_rank_of (records)
+    rank = nil
+    record_order_by.each do |key|
+      r = records[key].flatten.order(method: :dense)
+      if rank.nil?
+        rank = r
+      else
+        top  = r.max
+        span = top.equal?(UNDEF) ? 1 : top + 1
+        rank = (rank * span + r).order(method: :dense)
+      end
+    end
+    rank
+  end
+
+  # Ranks of self and `other` in one ordering, so they can be compared.
+  def record_rank_with (other)
+    case other
+    when CARecord
+      unless other.data_class.equal?(data_class)
+        raise ArgumentError,
+              "can not compare a record of #{data_class.inspect} " \
+              "with one of #{other.data_class.inspect}"
+      end
+      theirs = other.flatten
+    when data_class
+      theirs = CARecord.new(data_class, 1)
+      theirs[0] = other
+    else
+      raise ArgumentError,
+            "can not compare a record of #{data_class.inspect} with #{other.class}"
+    end
+    n    = elements
+    both = record_rank_of(CArray.concatenate([flatten, theirs]))
+    mine = both[0...n].reshape(*shape)
+    rest = both[n..-1]
+    [mine, other.is_a?(CARecord) ? rest.reshape(*other.shape) : rest[0]]
+  end
+
+  # The record at each min_index / max_index, gathered from self.
+  def record_extreme (index_op, axis, keep_axis)
+    rank = record_rank
+    if axis.nil?
+      i = rank.flatten.public_send(index_op)
+      return i.equal?(UNDEF) ? UNDEF : flatten[i]
+    end
+    name = index_op.to_s.sub("_index", "")
+    if axis.is_a?(Array)
+      axes = axis.map { |k| normalize_axis(k, name) }.uniq
+      return record_extreme(index_op, axes[0], keep_axis) if axes.size == 1
+      return record_extreme_over(index_op, axes, keep_axis)
+    end
+    axis = normalize_axis(axis, name)
+    idx   = rank.public_send(index_op, axis: axis)
+    empty = idx.is_masked
+    picked_shape = shape.dup
+    picked_shape[axis] = 1
+    out = take_along_axis(idx.strip_mask(0).reshape(*picked_shape), axis: axis).copy
+    if empty.any
+      out[empty.reshape(*picked_shape)] = UNDEF
+    end
+    return out if keep_axis
+    out.reshape(*(shape[0...axis] + shape[axis + 1..-1]))
+  end
+
+  # Over several axes: the reduced axes are moved last and merged into one.
+  def record_extreme_over (index_op, axes, keep_axis)
+    keep = (0...ndim).to_a - axes
+    kept_shape = shape.values_at(*keep)
+    if keep.empty?
+      r = record_extreme(index_op, nil, false)
+      return r unless keep_axis
+      out = flatten[0..0].copy
+      out[0] = r
+      return out.reshape(*([1] * ndim))
+    end
+    merged = transpose(*(keep + axes))
+               .reshape(*kept_shape, shape.values_at(*axes).inject(:*))
+    out = merged.send(:record_extreme, index_op, keep.size, false)
+    return out unless keep_axis
+    out.reshape(*shape.each_with_index.map { |n, k| axes.include?(k) ? 1 : n })
+  end
+
+end

data/lib/carray/runtime.rb

@@ -5,6 +5,16 @@
 
 class CArray
 
+  # A fast path answers by a different algorithm what a general path also
+  # answers.  A development build defines this in C, false when run with
+  # CARRAY_GENERAL_PATH=1 so the test suite takes the general paths (see
+  # ext/carray_internal.h); a release build has no switch.
+  unless respond_to?(:__fast_paths_enabled__)
+    def self.__fast_paths_enabled__ # :nodoc:
+      true
+    end
+  end
+
   # String fallback for the global cast functions CA_INT32(data) etc.  The C
   # dispatcher (ext/carray_cast.c) handles every other input class in C and
   # delegates only String here, because the whitespace / `,` / `;` /
@@ -65,23 +75,16 @@ class CArray
     num = prod.sum(**nopt)
     den = wv.sum(axis: axis, keep_axis: keep_axis)
     result =
-      if axis
+      if den.is_a?(CArray)               # axis given, or keep_axis
         den = den.copy
         den[den.eq(0)] = UNDEF          # zero / empty denominator -> undetermined
         num / den                        # UNDEF propagates cell-wise
-      elsif den.equal?(UNDEF) || den == 0
-        UNDEF
+      elsif num.equal?(UNDEF) || den.equal?(UNDEF) || den == 0
+        UNDEF                            # min_count unmet leaves num UNDEF
       else
         num / den
       end
-    fill_value = opts[:fill_value]
-    return result if fill_value.nil?
-    if result.is_a?(CArray)
-      result[result.is_masked] = fill_value if result.has_mask?
-      result
-    else
-      result.equal?(UNDEF) ? fill_value : result
-    end
+    CArray.__reduce_fill__(result, opts[:fill_value])
   end
   private :__wmean_object__