carray 3.0.3 → 3.0.4

data/lib/carray/fusion.rb

@@ -66,6 +66,7 @@ class CArray
     Plan  = Struct.new(:nodes, :leaves, :data_type, :dim, :masked, :signature)
 
     LAZY_CLASSES = [CAMonOp, CABinOp, CATriOp, CAMonCmp, CABinCmp, CALazyMarker]
+    Ractor.make_shareable(LAZY_CLASSES)
 
     # A lazy node names its operation by an id and the kernels name it by a
     # symbol.  These are the same operations, spelled the way each side
@@ -76,7 +77,7 @@ class CArray
       :&  => :bit_and_i,  :|  => :bit_or_i,   :^  => :bit_xor_i,
       :<< => :bit_lshift, :>> => :bit_rshift,
     }.freeze
-    TRIOP_NAMES = { :__clip_ki__ => :clip }.freeze
+    TRIOP_NAMES = { :clamp_between => :clip }.freeze
 
     MONOP_BY_ID = CArray::LAZY_MONOP_OP_IDS.invert.freeze
     # A Float or Complex array to an Integer power is its own node, ipow,
@@ -118,7 +119,7 @@ class CArray
     # Returns the array, or nil where nothing computed it.
     def self.evaluate (view)
       plan = worth_asking(view) or return nil
-      out = CArray.__alloc_uninit__(view.data_type, view.dim)
+      out = CArray.__alloc_uninit__(view.data_type, view.shape)
       ask(plan, out) ? out : nil
     end
 
@@ -131,30 +132,46 @@ class CArray
     end
 
     # The plan to hand over, or nil.  A marker over an array, or anything
-    # else with nothing to compute, is not worth handing over.
+    # else with nothing to compute, is not worth handing over.  Nothing here
+    # is the evaluator's doing, so nothing here retires it: an array that
+    # raises when it is looked at (a CAObject whose backing is gone) raises
+    # again in the walk, where the caller sees it.
     def self.worth_asking (view)
       return nil unless askable?(view)
       plan = plan(view) or return nil
       plan.nodes.any? { |n| n.is_a?(Op) } ? plan : nil
-    rescue StandardError => error
-      retire(error)
+    rescue StandardError
       nil
     end
 
     def self.ask (plan, out)
       evaluator = CArray.expression_evaluator or return false
       return false if overlaps?(plan, out)
-      # The walk leaves the destination masked where the expression is, so
-      # a destination that arrives masked loses its mask to an expression
-      # that has none.
+      return false unless ready_mask(plan, out)
+      begin
+        evaluator.call(plan, out) ? true : false
+      rescue AddressBasis::Failure
+        # Raised by CArray lending the arrays or taking them back: a refusal,
+        # or an array that cannot be read.  The walk meets the same array.
+        false
+      rescue StandardError => error
+        retire(error)
+        false
+      end
+    end
+
+    # The walk leaves the destination masked where the expression is, so a
+    # destination that arrives masked loses its mask to an expression that
+    # has none.  A destination that cannot hold a mask (a value array) is
+    # left to the walk, which says so.
+    def self.ready_mask (plan, out)
       if plan.masked
         out.mask = 0 unless out.has_mask?
       elsif out.has_mask?
         out.mask = 0
       end
-      evaluator.call(plan, out) ? true : false
-    rescue StandardError => error
-      retire(error)
+      true
+    rescue StandardError
       false
     end
 
@@ -171,13 +188,15 @@ class CArray
         if leaf.equal?(out)
           shifted.include?(index)
         else
-          storages(leaf).any? { |r| mine.any? { |m| m.equal?(r) } }
+          storages(leaf).any? { |r| mine.any? { |m| m.send(:storage_overlaps?, r) } }
         end
       end
     end
 
     # The arrays that hold an array's cells: its root, or each parent's
-    # where a view stands over several.
+    # where a view stands over several.  Two of them share storage when
+    # they are one array, or entities whose memory overlaps (an array and a
+    # wrap_memory_view of it).
     def self.storages (array, found = [])
       root = array.root_array
       if ! root.entity? && root.respond_to?(:parents)
@@ -213,7 +232,7 @@ class CArray
       w.visit(view)
       masked = w.leaves.any? { |a| a.has_mask? } ||
                w.nodes.any? { |n| n.is_a?(Shifted) && n.bounds.include?(:mask) }
-      Plan.new(w.nodes, w.leaves, view.data_type, view.dim, masked, w.signature)
+      Plan.new(w.nodes, w.leaves, view.data_type, view.shape, masked, w.signature)
     end
 
     def self.lazy? (x)
@@ -350,8 +369,8 @@ class CArray
       # expression, say -- has no array to read and is taken as a whole.
       def shifted (n)
         array = n.parent
-        bounds = n.__axis_bounds__
-        if Fusion.lazy?(array) || array.dim != n.dim ||
+        bounds = n.__boundary_rule_of_each_axis__
+        if Fusion.lazy?(array) || array.shape != n.shape ||
            ! bounds.all? { |b| b == :fill || b == :mask }
           return leaf(n)
         end
@@ -366,8 +385,13 @@ class CArray
                          offset, bounds, fill)
       end
 
+      # A leaf the address bases would refuse to lend cannot be read by an
+      # evaluator, so the expression is not described.
       def leaf_index (array)
         @leaf_index.fetch(array.object_id) do
+          if (reason = AddressBasis.classify(array)[:refusal])
+            raise Refused, "a leaf: #{reason}"
+          end
           @leaves << array
           @leaf_index[array.object_id] = @leaves.size - 1
         end

data/lib/carray/histogram.rb

@@ -321,6 +321,7 @@ class CArray
 
       if weights
         raise ArgumentError, "weights given but accumulator is unweighted" unless @weighted
+        # coerce: the accumulator's type, declared by weights_data_type:
         weights = CArray.wrap_readonly(weights, @counts_data_type)
         expected_w_shape = chunk.shape.dup
         expected_w_shape.delete_at(channel_ax)
@@ -447,7 +448,7 @@ class CArray
   #   @param weights [CArray, nil] optional per-sample weights.
   #   @return [Histogram] 1-D accumulator (`m == 1`).
   def histogram1d (edges:, axis: -1, include_max: false, weights: nil)
-    ax = normalize_axis(axis, "histogram1d")
+    ax = normalize_axis(axis.nil? ? -1 : axis, "histogram1d")
 
     new_shape = shape + [1]
     arr_with_channel = reshape(*new_shape)
@@ -491,6 +492,10 @@ class CArray
   def histogram (edges:, axis: [-2, -1], include_max: false, weights: nil)
     raise ArgumentError, "edges must be an Array of edges arrays" unless edges.is_a?(Array)
     arr = self
+    axis = [-2, -1] if axis.nil?
+    unless axis.is_a?(Array) && axis.size == 2
+      raise ArgumentError, "histogram: axis: is a [sample, channel] pair (got #{axis.inspect})"
+    end
     sample_ax  = normalize_axis(axis[0], "histogram sample axis")
     channel_ax = normalize_axis(axis[1], "histogram channel axis")
     fiber_shape = arr.shape.dup

data/lib/carray/inspect.rb

@@ -34,16 +34,39 @@ class CArray::Inspector  # :nodoc:
   
   private
 
-  # The name before the shape: the class and the storage type, or what a
-  # Face says instead through a private inspect_label (CATime gives its
-  # tick, which the storage type does not tell).
+  # The name before the shape: the class and the storage type, or for a
+  # Face its own name (see .face_label).
   def get_label
-    if @carray.respond_to?(:inspect_label, true)
-      return @carray.send(:inspect_label)
-    end
+    return CArray::Inspector.face_label(@carray) if @carray.face?
     return format("%s.%s", get_class_name, get_type_name)
   end
 
+  public
+
+  # The name a Face goes by where a person reads it -- the head of inspect,
+  # a frame's column list.  A Face's data_type is the surface the kernels
+  # dispatch on (fixlen for a time, so arithmetic on it raises), which says
+  # nothing to a reader, so it is named by its class: what a private
+  # inspect_label gives (CATime adds its tick), or the class, with the
+  # surface type only for a Face whose surface is numeric.
+  # The type of an array as a person reads it in a message or a table: a
+  # Face by its name (face_label), any other array by its data type name.
+  # Every message and listing that names a type goes through here, so a
+  # Face is never named by its storage (:fixlen for a time).
+  def self.type_label (ca)
+    ca.face? ? face_label(ca) : CArray.data_type_name(ca.data_type).to_s
+  end
+
+  def self.face_label (ca)
+    return ca.send(:inspect_label) if ca.respond_to?(:inspect_label, true)
+    klass = ca.class
+    klass = klass.superclass while klass.name.nil?
+    return klass.name if [CA_FIXLEN, CA_OBJECT].include?(ca.data_type)
+    "#{klass.name}.#{CArray.data_type_name(ca.data_type)}"
+  end
+
+  private
+
   def get_class_name
     return @carray.class.to_s
   end
@@ -277,10 +300,10 @@ class CArray
     case data_type 
     when CA_FIXLEN
       output << sprintf("CArray.%s(%s, :bytes=>%i)", 
-                        data_type_name, dim.inspect[1..-2], bytes)
+                        data_type_name, shape.inspect[1..-2], bytes)
     else
       output << sprintf("CArray.%s(%s)", 
-                        data_type_name, dim.inspect[1..-2])
+                        data_type_name, shape.inspect[1..-2])
     end
     return output
   end

data/lib/carray/iterator.rb

@@ -32,6 +32,45 @@ class CAIterator
   # The keywords a member passes on to a core reduction over its own axes.
   # The member sets `axis:`, so one from the caller is refused; the rest go
   # to the core, which checks them.
+  # Whether a fill method carries values backward: :forward / :ffill carry
+  # them forward, :backward / :bfill back. A member fills within its own
+  # pieces by holding the nearest present value; :linear, which needs a
+  # coordinate, is the core's alone.
+  def self.hold_backward? (method) # :nodoc:
+    case method
+    when :forward, :ffill   then false
+    when :backward, :bfill  then true
+    else
+      raise ArgumentError,
+            "strip_mask: method must be :forward, :ffill, :backward or :bfill " \
+            "within a piece (got #{method.inspect})"
+    end
+  end
+
+  # A one-cell (1, 1) array of the kind `value` holds, for asking the core
+  # what a reduction or scan of it answers in: its data type, cell width and,
+  # for a Face, its Face.  A member that answers by a route of its own
+  # (a fused kernel, a fold by offset) takes its output type from here, so
+  # the route and the core cannot disagree.  A Face is a view of the values
+  # themselves rather than a blank -- a Face cannot be allocated from its
+  # surface data type alone, and a zeroed record is not a valid cell for every
+  # one (a const string's record indexes a shared pool).  A Face with no cell
+  # is asked with the blank cell its resize gives, so a reduction over no
+  # groups answers in the same kind as over some; this is nil only when the
+  # Face has no blank cell.
+  def self.core_probe (value) # :nodoc:
+    unless value.face?
+      return CArray.new(value.data_type, [1, 1], bytes: value.bytes)
+    end
+    flat = value.reshape(value.elements)
+    return flat[[0]].reshape(1, 1) if value.elements > 0
+    begin
+      flat.resize(1).reshape(1, 1)
+    rescue StandardError
+      nil
+    end
+  end
+
   def self.reduce_keywords (op, kw) # :nodoc:
     if kw.key?(:axis)
       raise ArgumentError, "#{op}: unknown keyword: :axis (the iterator sets the axes)"
@@ -127,6 +166,18 @@ class CAIterator
   #   Returns the five-number summary of each piece,
   #   `[min, Q1, median, Q3, max]`.
   #   @return [Array<CArray>] five arrays, one per position.
+  # @!method first
+  #   Returns the first present value of each piece, in the piece's own
+  #   order; a piece with no present value is UNDEF.
+  #   @return [CArray] one value per piece, in the source's data type.
+  # @!method last
+  #   Returns the last present value of each piece; a piece with no present
+  #   value is UNDEF.
+  #   @return [CArray] one value per piece, in the source's data type.
+  # @!method nunique
+  #   Returns the number of distinct present values in each piece, as
+  #   `CArray#nunique` counts them; a piece with none counts 0.
+  #   @return [CArray] `:int64`, one count per piece.
   # @!method count(value = nil)
   #   Returns a count per piece: with no argument the cells that are not
   #   masked, with `UNDEF` the masked cells, and with any other value the
@@ -161,14 +212,16 @@ class CAIterator
     :min_index, :max_index, :min_addr, :max_addr,                     # position
     :wsum, :wmean,                                                    # weighted
     :median, :percentile, :quantile,                                 # tier 3
+    :first, :last, :nunique,                                          # position / distinct
     :count, :count_not_masked, :count_masked, :elements,             # count family
     :each, :reduce,                                                   # generic iterate
   ].freeze
 
   REQUIRED_SURFACE.each do |name|
-    define_method(name) do |*, **, &_blk|
+    # define_method: a stub that names itself in its own message
+    define_method(name, &Ractor.make_shareable(proc do |*, **, &_blk|
       raise NotImplementedError, "#{self.class} must implement ##{name}"
-    end
+    end))
   end
 
   # Recommended (should) surface -- template methods.  These are well-defined
@@ -176,7 +229,8 @@ class CAIterator
   # required: `map` is a per-piece element-wise transform scattered back to the
   # source, `sort_addr` is a per-piece sort returning source flat addresses, and
   # the segment scans (cumsum / cumprod / cummax / cummin / cumcount) write a
-  # per-cell running statistic of each piece.  A member implements each when it
+  # per-cell running statistic of each piece, and `lag` the value some steps
+  # earlier in the same piece.  A member implements each when it
   # is well-defined and overrides it to raise with a reason when it is not.  The
   # scans belong here for the same reason as map: a running per-cell value is
   # single-valued only when each cell belongs to exactly one piece, so the
@@ -221,11 +275,35 @@ class CAIterator
   #   masked within each piece.
   #   @return [CArray] shaped like the source.
   #   @raise [NotImplementedError] for a member whose pieces overlap.
-  [:map, :sort_addr,
-   :cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |name|
-    define_method(name) do |*, **, &_blk|
+  # @!method lag(n = 1)
+  #   Returns a source-shaped array whose cell holds the value `n` steps
+  #   earlier within the same piece, as `CArray#lag` does along an axis; a
+  #   cell with no such value in its piece is UNDEF, so a lag never reaches
+  #   across into another piece.
+  #   @param n [Integer] steps back; a negative `n` looks ahead.
+  #   @return [CArray] shaped like the source.
+  #   @raise [NotImplementedError] for a member whose pieces overlap, where a
+  #     cell has no single position within a piece.
+  # @!method strip_mask(method:)
+  #   Returns a source-shaped array with each masked cell filled from its own
+  #   piece, as `CArray#strip_mask(method:)` fills: `:forward` (`:ffill`)
+  #   carries the last present value forward, `:backward` (`:bfill`) the next
+  #   one back, never across into another piece.
+  #   @param method [Symbol] `:forward`, `:ffill`, `:backward` or `:bfill`.
+  #   @return [CArray] shaped like the source.
+  #   @raise [NotImplementedError] for a member whose pieces overlap.
+  # @!method order(descending: false, method: :ordinal, kind: :quick)
+  #   Returns a source-shaped int64 array of each cell's rank among the cells
+  #   of its piece, as `CArray#order` ranks them (`0` the smallest).
+  #   @return [CArray] shaped like the source.
+  #   @raise [NotImplementedError] for a member whose pieces overlap, where a
+  #     cell has no single rank.
+  [:map, :sort_addr, :order, :strip_mask,
+   :cumsum, :cumprod, :cummax, :cummin, :cumcount, :lag].each do |name|
+    # define_method: a stub that names itself in its own message
+    define_method(name, &Ractor.make_shareable(proc do |*, **, &_blk|
       raise NotImplementedError, "#{self.class} does not provide ##{name} (optional)"
-    end
+    end))
   end
 
 end