carray 3.0.1 → 3.0.3

data/lib/carray/methods/snap.rb

@@ -66,8 +66,11 @@ class CArray
   #   the sample, `:ceil` at or above), and delegates to
   #   `locate_nearest_addr(direction:)`.
   #
-  #   `list` must be a 1-D ascending numeric CArray (or convertible via
-  #   `CArray.wrap_readonly`). Out-of-range and NaN handling follows the
+  #   `list` must be a 1-D ascending numeric CArray, or a Ruby Array /
+  #   Range.  An Array grid takes `self`'s data_type, except that a Float
+  #   in it makes an integer array's grid float64 (so `0.5` is not
+  #   truncated); an Integer that does not fit `self`'s type raises
+  #   `RangeError`.  Out-of-range and NaN handling follows the
   #   same pattern as `bin_to`, but with an additional `:clamp`
   #   sentinel that snaps out-of-range cells to the nearest list end.
   #
@@ -95,19 +98,21 @@ class CArray
   #     shape as `self`.
   #   @raise [ArgumentError] when `list` is not 1-D or is empty, or
   #     `direction` is not one of the accepted symbols.
+  #   @raise [RangeError] when an Integer in an Array `list` does not
+  #     fit `self`'s data_type.
   #   @example
   #     temp.snap_to([270.0, 280.0, 290.0, 300.0])                   # clamp OOB
   #     temp.snap_to(grid, lfill: nil, ufill: nil)                   # mask OOB
   #     rain.snap_to([0.0, 1.0, 5.0, 20.0], direction: :floor)       # list value at or below
   def snap_to(list, lfill: :clamp, ufill: :clamp, direction: :round)
-    ref = list.is_a?(CArray) ? list : CArray.wrap_readonly(list, self.data_type)
+    ref = list.is_a?(CArray) ? list : snap_list(list)
     raise ArgumentError, "snap_to: list must be 1-D" unless ref.ndim == 1
     n = ref.elements
     raise ArgumentError, "snap_to: list must have at least one value" if n < 1
 
     if n == 1
       # Degenerate: every finite cell snaps to the only value.
-      out = CArray.new(ref.data_type, shape).fill(ref[0])
+      out = CArray.new(ref.data_type, shape, bytes: ref.bytes).fill(ref[0])
       out.mask = self.mask.to_ca if self.has_mask?
       if self.float?
         inv = self.is_invalid
@@ -118,17 +123,16 @@ class CArray
       return out
     end
 
-    # `locate_nearest_addr` (via `linear_section`) accepts a 1-D `val` only;
-    # flatten multi-D input and reshape the result back to preserve the
-    # element-wise semantic on any shape.
+    # Multi-D input is snapped as its flattened cells and given its shape
+    # back.
     if ndim > 1
       return reshape(-1).snap_to(ref, lfill: lfill, ufill: ufill,
                                  direction: direction).reshape(*shape)
     end
 
-    # locate_nearest_addr returns int64 indices; OOB (below / above / NaN)
-    # cells come back masked. We split OOB into below / above with
-    # explicit comparisons so the two sides can be filled independently.
+    # locate_nearest_addr returns int64 indices; whether a cell outside the
+    # list gets one depends on the direction, so the two sides are split with
+    # explicit comparisons and filled here independently of it.
     idx = self.locate_nearest_addr(ref, direction: direction)
     out = ref.project(idx)
 
@@ -137,13 +141,13 @@ class CArray
 
     case lfill
     when :clamp then out[below] = ref[0]
-    when nil    then # leave masked (locate_nearest_addr already masked OOB)
+    when nil    then out[below] = UNDEF
     else             out[below] = lfill
     end
 
     case ufill
     when :clamp then out[above] = ref[-1]
-    when nil    then # leave masked
+    when nil    then out[above] = UNDEF
     else             out[above] = ufill
     end
 
@@ -158,4 +162,22 @@ class CArray
     out
   end
 
+  private
+
+  # A Ruby Array grid in the type a search query would get: a Float in it
+  # makes an integer array's grid float64 (so 0.5 is not truncated to 0),
+  # otherwise the grid takes self's type, and an Integer that does not fit
+  # it is refused rather than wrapped.
+  def snap_list(list)
+    vals = list.to_a
+    type = data_type
+    type = CA_FLOAT64 if (integer? || data_type == CA_BOOLEAN) && vals.any? { |v| v.is_a?(Float) }
+    ref = CArray.wrap_readonly(vals, type)
+    if ref.integer?
+      bad = vals.zip(ref.to_a).find { |v, w| v != w }
+      raise RangeError, "snap_to: #{bad[0].inspect} is out of range for #{ref.data_type_name}" if bad
+    end
+    ref
+  end
+
 end

data/lib/carray/methods/unique.rb

@@ -1,3 +1,5 @@
+require "carray/methods/discovery_along"
+
 class CArray
 
   # @overload unique(sort: false)
@@ -24,22 +26,69 @@ class CArray
   #   For `CA_OBJECT` / `CA_FIXLEN` distinctness follows Ruby
   #   `eql?` / `hash`; note that Ruby does not collapse distinct NaN
   #   objects, so a `CA_OBJECT` array of Float NaN is not collapsed
-  #   (unlike the numeric path).
+  #   (unlike the numeric path). Complex values follow the float rules
+  #   in each component; the values come back with -0.0 written as 0.0
+  #   and every NaN as one NaN.
   #
   #   @param sort [Boolean] when true, return the distinct values
-  #     sorted ascending instead of in first-appearance order.
+  #     sorted ascending instead of in first-appearance order. A complex
+  #     array has no order and raises {CArray::DataTypeError}.
   #   A time array (`CATime` / `CATimedelta`) answers with its own type on
   #   its own unit: the distinct values are values, so the array comes back
   #   as itself rather than as raw storage ticks.
   #
   #   @return [CArray] 1-D CArray of the distinct values, same data type
   #     as `self`.
-  def unique (sort: false)
+  #
+  # @overload unique(along: k)
+  #   Returns the distinct **sub-arrays** of `self`, comparing whole
+  #   sub-arrays rather than cells: `along: k` names the axis whose
+  #   index enumerates them, so `z.unique(along: 0)` gives the distinct
+  #   rows of a 2-D array, in first-appearance order. This is
+  #   `np.unique(z, axis=k)`.
+  #
+  #   Note the contrast with `axis:` on {#nunique} and
+  #   {#mask_duplicates}, which names the axis a *fiber runs along* and
+  #   asks about the values inside each fiber. The two cannot be given
+  #   together.
+  #
+  #   Distinctness is the family's, widened from a cell to a sub-array:
+  #   two sub-arrays are the same when every cell is, with all NaN one
+  #   value and -0.0 == +0.0. A sub-array holding a masked cell does not
+  #   participate and never appears in the result.
+  #
+  #   The result is a **view** of `self` -- the surviving sub-arrays,
+  #   not copies of them -- so writing through it reaches `self`. Take a
+  #   `copy` when that is not wanted. `sort:` is not available here:
+  #   sub-arrays have no order to sort by. `object` arrays are refused,
+  #   because their cells hold Ruby references.
+  #
+  #   @param along [Integer] axis whose index enumerates the sub-arrays.
+  #   @return [CArray] the distinct sub-arrays, same shape as `self`
+  #     except along `along`.
+  def unique (sort: false, along: nil)
+    if along
+      if sort
+        raise ArgumentError,
+              "unique: sort: is not available with along: -- sub-arrays have " \
+              "no order to sort by; the result is in first-appearance order"
+      end
+      keys = fibers_as_cells(along, "unique")
+      args = [nil] * ndim
+      args[normalize_axis(along, "unique")] = keys.mask_duplicates.is_not_masked
+      return self[*args]
+    end
     # Single-pass seen-set hash (C __unique_flat__), one lane per data type family:
     # integer widens to a 64-bit key; float uses the bitwise key with all-NaN
     # collapsed and -0.0 / +0.0 normalized; object keys on rb_hash + rb_eql and
     # fixlen on a byte-hash + memcmp, both reproducing Ruby Hash distinctness.
     # Masked cells are skipped in the kernel.
+    if sort && complex?
+      raise CArray::DataTypeError,
+            "unique(sort: true): a #{data_type_name} value has no order " \
+            "(complex numbers are not ordered); unique without sort: gives " \
+            "the values in first-appearance order"
+    end
     levels = __unique_flat__
     sort ? levels.sort : levels
   end

data/lib/carray/record_order.rb

@@ -0,0 +1,208 @@
+# ----------------------------------------------------------------------------
+#
+#  carray/record_order.rb
+#
+#  The order of a CARecord: the members its struct names in `order_by:`,
+#  compared in that order.
+#
+#  A record's storage is its bytes, and comparing bytes is no order of the
+#  values in them (a little-endian float, a signed integer).  So the
+#  ordering members do not run on the storage: they run on one int64 array,
+#  each record's rank among the others by the declared members, and the
+#  values they return are gathered from the record array by the positions
+#  that array gives.  A struct that declares no order has none, and these
+#  raise.
+#
+# ----------------------------------------------------------------------------
+
+class CARecord
+
+  # @overload sort(axis: nil, kind: :quick, masked_position: :last)
+  #   Returns a sorted {CARecord} view, ordered by the struct's
+  #   `order_by:` members.  With no `axis:` the array is flattened first.
+  #   @return [CARecord]
+  #   @raise [ArgumentError] when the struct declares no `order_by:`.
+  def sort (*args, **kw)
+    addr = record_rank.sort_addr(*args, **kw)
+    kw[:axis].nil? ? self[addr.flatten] : self[addr]
+  end
+
+  # @overload sort_copy(axis: nil, kind: :quick, masked_position: :last)
+  #   @return [CARecord] an owned sorted copy.
+  def sort_copy (*args, **kw)
+    sort(*args, **kw).copy
+  end
+
+  # @!method sort_addr(axis: nil, kind: :quick, masked_position: :last)
+  #   @return [CArray] the view-flat addresses of a sort by the declared order.
+  # @!method sort_index(axis: nil, kind: :quick, masked_position: :last)
+  #   @return [CArray] the per-fiber indices of a sort by the declared order.
+  # @!method rank_index(axis: nil)
+  #   @return [CArray] each record's rank by the declared order.
+  # @!method order(axis: nil, descending: false, method: :ordinal, kind: :quick)
+  #   @return [CArray] each record's rank by the declared order.
+  # @!method min_index(axis: nil)
+  #   @return [Integer, CArray] where the smallest record sits.
+  # @!method max_index(axis: nil)
+  #   @return [Integer, CArray] where the largest record sits.
+  # @!method partition_index(kth, axis: nil)
+  #   @return [CArray] the indices that partition about the `kth` record.
+  [:sort_addr, :sort_index, :rank_index, :order,
+   :min_index, :max_index, :partition_index].each do |op|
+    define_method(op) { |*args, **kw| record_rank.public_send(op, *args, **kw) }
+  end
+
+  # @overload min(axis: nil, keep_axis: false)
+  #   Returns the smallest record by the declared order, skipping masked
+  #   records; UNDEF when every record is masked.  With `axis:` a
+  #   {CARecord} of the smallest along that axis.
+  #   @return [CAStruct, CARecord]
+  def min (axis: nil, keep_axis: false)
+    record_extreme(:min_index, axis, keep_axis)
+  end
+
+  # @overload max(axis: nil, keep_axis: false)
+  #   The largest record; see {#min}.
+  #   @return [CAStruct, CARecord]
+  def max (axis: nil, keep_axis: false)
+    record_extreme(:max_index, axis, keep_axis)
+  end
+
+  # @overload minmax(axis: nil, keep_axis: false)
+  #   @return [Array] `[min, max]`.
+  def minmax (axis: nil, keep_axis: false)
+    [min(axis: axis, keep_axis: keep_axis), max(axis: axis, keep_axis: keep_axis)]
+  end
+
+  # @overload partition_copy(kth, axis: nil)
+  #   @return [CARecord] partitioned about the `kth` record in the declared
+  #     order.  With no `axis:` the array is flattened first.
+  def partition_copy (kth, axis: nil)
+    if axis.nil?
+      flat = flatten
+      return flat[flat.send(:record_rank).partition_index(kth)].copy
+    end
+    take_along_axis(record_rank.partition_index(kth, axis: axis), axis: axis).copy
+  end
+
+  # @!method lt(other)
+  #   Compares by the declared order.  `other` is a record of the same
+  #   struct, or a {CARecord} of them.
+  #   @return [CArray] boolean.
+  # @!method le(other)
+  # @!method gt(other)
+  # @!method ge(other)
+  [:lt, :le, :gt, :ge].each do |op|
+    define_method(op) do |other|
+      mine, theirs = record_rank_with(other)
+      mine.public_send(op, theirs)
+    end
+  end
+
+  alias <  lt
+  alias <= le
+  alias >  gt
+  alias >= ge
+
+  private
+
+  # The struct's order_by: members, or a refusal naming where to declare it.
+  def record_order_by
+    data_class.order_by or
+      raise ArgumentError,
+            "#{data_class.inspect} declares no order; pass order_by: to " \
+            "CArray.struct to name the members records are ordered by"
+  end
+
+  # Each record's dense rank by the declared members, shaped like self.
+  # Each member is ranked on its own (a numeric member by value, NaN last;
+  # a fixlen member by its bytes), and the ranks are folded in member order.
+  # Ranking again after each fold keeps every value below the record count.
+  def record_rank
+    record_rank_of(self).reshape(*shape)
+  end
+
+  def record_rank_of (records)
+    rank = nil
+    record_order_by.each do |key|
+      r = records[key].flatten.order(method: :dense)
+      if rank.nil?
+        rank = r
+      else
+        top  = r.max
+        span = top.equal?(UNDEF) ? 1 : top + 1
+        rank = (rank * span + r).order(method: :dense)
+      end
+    end
+    rank
+  end
+
+  # Ranks of self and `other` in one ordering, so they can be compared.
+  def record_rank_with (other)
+    case other
+    when CARecord
+      unless other.data_class.equal?(data_class)
+        raise ArgumentError,
+              "can not compare a record of #{data_class.inspect} " \
+              "with one of #{other.data_class.inspect}"
+      end
+      theirs = other.flatten
+    when data_class
+      theirs = CARecord.new(data_class, 1)
+      theirs[0] = other
+    else
+      raise ArgumentError,
+            "can not compare a record of #{data_class.inspect} with #{other.class}"
+    end
+    n    = elements
+    both = record_rank_of(CArray.concatenate([flatten, theirs]))
+    mine = both[0...n].reshape(*shape)
+    rest = both[n..-1]
+    [mine, other.is_a?(CARecord) ? rest.reshape(*other.shape) : rest[0]]
+  end
+
+  # The record at each min_index / max_index, gathered from self.
+  def record_extreme (index_op, axis, keep_axis)
+    rank = record_rank
+    if axis.nil?
+      i = rank.flatten.public_send(index_op)
+      return i.equal?(UNDEF) ? UNDEF : flatten[i]
+    end
+    name = index_op.to_s.sub("_index", "")
+    if axis.is_a?(Array)
+      axes = axis.map { |k| normalize_axis(k, name) }.uniq
+      return record_extreme(index_op, axes[0], keep_axis) if axes.size == 1
+      return record_extreme_over(index_op, axes, keep_axis)
+    end
+    axis = normalize_axis(axis, name)
+    idx   = rank.public_send(index_op, axis: axis)
+    empty = idx.is_masked
+    picked_shape = shape.dup
+    picked_shape[axis] = 1
+    out = take_along_axis(idx.strip_mask(0).reshape(*picked_shape), axis: axis).copy
+    if empty.any
+      out[empty.reshape(*picked_shape)] = UNDEF
+    end
+    return out if keep_axis
+    out.reshape(*(shape[0...axis] + shape[axis + 1..-1]))
+  end
+
+  # Over several axes: the reduced axes are moved last and merged into one.
+  def record_extreme_over (index_op, axes, keep_axis)
+    keep = (0...ndim).to_a - axes
+    kept_shape = shape.values_at(*keep)
+    if keep.empty?
+      r = record_extreme(index_op, nil, false)
+      return r unless keep_axis
+      out = flatten[0..0].copy
+      out[0] = r
+      return out.reshape(*([1] * ndim))
+    end
+    merged = transpose(*(keep + axes))
+               .reshape(*kept_shape, shape.values_at(*axes).inject(:*))
+    out = merged.send(:record_extreme, index_op, keep.size, false)
+    return out unless keep_axis
+    out.reshape(*shape.each_with_index.map { |n, k| axes.include?(k) ? 1 : n })
+  end
+
+end

data/lib/carray/rng.rb

@@ -0,0 +1,86 @@
+class CArray
+
+  # A random number generator with its own state.
+  #
+  # `Rng` rather than `Random`: a `CArray::Random` would shadow `::Random`
+  # for every bare `Random` written inside `class CArray`, and the two mean
+  # different generators -- `random!(rng: Random.new(4))` draws through
+  # Ruby's MT19937 and `random!(rng: CArray::Rng.new(seed: 4))` through the
+  # one below.  Two names that different should not look alike.
+  #
+  # The class itself is defined in ext/carray_random.c, which is where the
+  # state is advanced.  What is added here is the generator's source text,
+  # so that another gem can run the same code rather than a second
+  # implementation of it.
+  class Rng
+
+    # The generators this carray knows, and how many int64 cells each one
+    # keeps its state in.
+    STATE_CELLS = {
+      :xoshiro256pp => 4,
+    }.freeze
+
+    GENERATORS = STATE_CELLS.keys.freeze
+
+    # Where a generator's C lives.  Shipped: the gemspec takes ext/*.h,
+    # and `ext` is a require path, so this resolves in a checkout and in
+    # an installed gem alike -- lib/ and ext/ are siblings in both.
+    SOURCE_FILES = {
+      :xoshiro256pp => File.expand_path("../../ext/ca_rng_xoshiro256pp.h",
+                                        __dir__),
+    }.freeze
+
+    # The C each generator is, as text.
+    #
+    #   CArray::Rng::SOURCE[:xoshiro256pp]   #=> "/* ---- ... */\n..."
+    #
+    # This is the same text the extension compiled -- the file below is
+    # `#include`d by ext/carray_random.c -- which is the point of handing
+    # it out.  A caller that pastes it into a translation unit of its own
+    # gets a generator that continues a sequence this one started, because
+    # it is running the code this one ran and not a copy of it.
+    #
+    # carray-jit is the caller this exists for: a kernel's `random(rng:)`
+    # pastes the text beside the helpers its own compiler emits, so a
+    # kernel draws from the CArray::Rng it was handed.  The text needs
+    # nothing but <stdint.h> and defines only `static inline` functions
+    # under a `ca_` prefix.
+    #
+    # It is read once, when this file is first required.
+    SOURCE = SOURCE_FILES.transform_values { |path|
+      File.read(path, :encoding => "UTF-8")
+    }.freeze
+
+    # What every generator's text needs beside it, and what none of them
+    # owns: turning uniforms into a normal is the same arithmetic whichever
+    # generator the uniforms came from.
+    #
+    # Handed out apart from SOURCE because whoever pastes it pastes it once,
+    # however many generators are drawing.  Folded into each generator's own
+    # text instead, two generators in one translation unit would define it
+    # twice.
+    COMMON_SOURCE = File.read(
+      File.expand_path("../../ext/ca_rng_normal.h", __dir__), :encoding => "UTF-8"
+    ).freeze
+
+    # The functions in SOURCE a caller may reach, by generator and by what
+    # the draw answers: `:random` a double in [0.0, 1.0), `:randomn` a
+    # standard normal, `:bits` the raw word a draw came from.  Each takes the
+    # state and advances it -- `:randomn` by two draws, the other two by one.
+    #
+    # Named here rather than worked out by whoever pastes the text: which
+    # symbols are the entry points is a fact about the generator, and the
+    # generator is CArray's.  These are the three `#random`, `#randomn`
+    # and `#bits` call, so a caller pasting the text draws the sequence
+    # this one draws.
+    DRAW_FUNCTIONS = {
+      :xoshiro256pp => {
+        :random  => "ca_xoshiro256pp_next_real",
+        :randomn => "ca_xoshiro256pp_next_normal",
+        :bits    => "ca_xoshiro256pp_next",
+      }.freeze,
+    }.freeze
+
+  end
+
+end

data/lib/carray/runtime.rb

@@ -5,6 +5,16 @@
 
 class CArray
 
+  # A fast path answers by a different algorithm what a general path also
+  # answers.  A development build defines this in C, false when run with
+  # CARRAY_GENERAL_PATH=1 so the test suite takes the general paths (see
+  # ext/carray_internal.h); a release build has no switch.
+  unless respond_to?(:__fast_paths_enabled__)
+    def self.__fast_paths_enabled__ # :nodoc:
+      true
+    end
+  end
+
   # String fallback for the global cast functions CA_INT32(data) etc.  The C
   # dispatcher (ext/carray_cast.c) handles every other input class in C and
   # delegates only String here, because the whitespace / `,` / `;` /
@@ -65,23 +75,16 @@ class CArray
     num = prod.sum(**nopt)
     den = wv.sum(axis: axis, keep_axis: keep_axis)
     result =
-      if axis
+      if den.is_a?(CArray)               # axis given, or keep_axis
         den = den.copy
         den[den.eq(0)] = UNDEF          # zero / empty denominator -> undetermined
         num / den                        # UNDEF propagates cell-wise
-      elsif den.equal?(UNDEF) || den == 0
-        UNDEF
+      elsif num.equal?(UNDEF) || den.equal?(UNDEF) || den == 0
+        UNDEF                            # min_count unmet leaves num UNDEF
       else
         num / den
       end
-    fill_value = opts[:fill_value]
-    return result if fill_value.nil?
-    if result.is_a?(CArray)
-      result[result.is_masked] = fill_value if result.has_mask?
-      result
-    else
-      result.equal?(UNDEF) ? fill_value : result
-    end
+    CArray.__reduce_fill__(result, opts[:fill_value])
   end
   private :__wmean_object__