carray 3.0.1 → 3.0.2

data/lib/carray/frame/io.rb

@@ -11,12 +11,27 @@ class CAFrame
   # later. Broken cells fail to_type and become UNDEF automatically
   # (parse-mask, §6-2).
   #
+  # +source+ is a path, or an open IO -- anything answering +gets+, which a
+  # StringIO is. So CSV that is already in memory does not have to go to a
+  # temporary file first:
+  #
+  #   CAFrame.from_csv("obs.csv")                    # a path
+  #   CAFrame.from_csv(StringIO.new(body))           # text already in hand
+  #   File.open("obs.csv") { |io| CAFrame.from_csv(io) }
+  #
+  # A String is always read as a path, never as CSV text: guessing between the
+  # two by looking for a newline is the kind of guess that is right until it is
+  # not, and StringIO says which one you meant in six characters. An IO is read
+  # from where it is and left open -- the caller opened it and closes it.
+  #
   # Parsing uses the built-in fast tokenizer (CSVParser). Options:
   #   sep:      field separator (default ",")
   #   quote:    quote character (default '"')
   #   strip:    trim spaces from unquoted fields (default false, RFC spacing)
-  #   +encoding+: IO open-mode encoding (default "bom|utf-8", strips a BOM)
-  #   parser:   a callable path -> [headers, rows] to inject another parser
+  #   +encoding+: open-mode encoding for a path (default "bom|utf-8", strips a
+  #             BOM). It has nothing to open when +source+ is an IO, so there
+  #             the IO's own encoding governs and a BOM is the caller's.
+  #   parser:   a callable source -> [headers, rows] to inject another parser
   #             (e.g. the stdlib +csv+, or a typed-table source); when given,
   #             sep/quote/strip/encoding and any block are that parser's concern.
   #
@@ -28,25 +43,25 @@ class CAFrame
   #     skip 2; header; skip 1; body
   #   end
   #
+  # A missing field -- an unquoted empty one, or a cell a short row never
+  # reached -- is UNDEF in the frame, whether or not the column is cast by
+  # +types:+. A quoted empty field ("") is the empty string, which is a value.
+  # So the mask +to_csv+ writes comes back as a mask.
+  #
   # Columns are handed to the frame as CABlock views over one backing object
   # array (§3.6 view-by-default); casting a column materializes it, and +copy+
   # gives an independent frame.
-  def self.from_csv(path, types: nil,
+  def self.from_csv(source, types: nil,
                     sep: ",", quote: '"', strip: false,
                     encoding: "bom|utf-8", parser: nil, &block)
     names, rows =
       if parser
-        parser.call(path)
+        parser.call(source)
+      elsif source.respond_to?(:gets)
+        read_csv(source, sep: sep, quote: quote, strip: strip, &block)
       else
-        File.open(path, "r:#{encoding}") do |io|
-          reader = CSVReader.new(io, sep: sep, quote: quote, strip: strip)
-          if block
-            block.arity == 1 ? block.call(reader) : reader.instance_exec(&block)
-          else
-            reader.header
-            reader.body
-          end
-          reader.result
+        File.open(source, "r:#{encoding}") do |io|
+          read_csv(io, sep: sep, quote: quote, strip: strip, &block)
         end
       end
 
@@ -55,6 +70,22 @@ class CAFrame
     frame
   end
 
+  # Drive the reading-control DSL over one open IO and hand back
+  # [names, rows]. Shared by the path and the IO source, so the two cannot
+  # come to read a file differently.
+  def self.read_csv (io, sep:, quote:, strip:, &block)
+    reader = CSVReader.new(io, sep: sep, quote: quote, strip: strip)
+    if block
+      block.arity == 1 ? block.call(reader) : reader.instance_exec(&block)
+    else
+      reader.header
+      reader.body
+    end
+    reader.result
+  end
+
+  private_class_method :read_csv
+
   # Build a frame from parsed [names, rows]. When names is nil (headerless and
   # no column_names) positional names "c0".."cN" are generated from the widest
   # row. Rows are squared off to the column count (short rows padded with nil,
@@ -159,7 +190,9 @@ class CAFrame
     parts = @columns.map { |k, v| "#{k}:#{v.data_type}#{v.ndim > 1 ? v.shape[1..].inspect : ''}" }
     idx = @index ? " index=#{@axis_name.inspect}" : ""
     head = "#<CAFrame nrow=#{@nrow} vars=[#{parts.join(', ')}]#{idx}>"
-    return head if @columns.empty?
+    # The table counts the index as a column, so a frame whose only data is its
+    # index has one to show.  Gate on the same thing render_table does.
+    return head if @columns.empty? && @index.nil?
     head + "\n" + render_table(head: 8, tail: 2, index: true, precision: 6,
                                footer: false)
   end
@@ -252,6 +285,15 @@ class CAFrame
     "#{n} #{noun}#{n == 1 ? '' : 's'}"
   end
 
+  # Render one N-D cell the way Array#inspect would, except that a masked
+  # element prints as the table's missing marker rather than as UNDEF.
+  private def format_nested_cell(v)
+    case v
+    when Array then "[" + v.map { |x| format_nested_cell(x) }.join(", ") + "]"
+    else UNDEF.equal?(v) || v.nil? ? "_" : v.inspect
+    end
+  end
+
   private def format_table_cell(col, i, precision)
     e = elem_at(col, i)
     if UNDEF.equal?(e) || e.nil?
@@ -262,7 +304,10 @@ class CAFrame
       # to read. precision: nil prints the value as Ruby renders it.
       e.round(precision).to_s
     elsif e.is_a?(CArray)
-      e.to_a.inspect
+      # An N-D cell renders its elements, and a masked element among them
+      # takes the same marker a masked scalar does -- UNDEF's own inspect
+      # would put a second spelling of "missing" in the same table.
+      format_nested_cell(e.to_a)
     elsif e.is_a?(String)
       e
     elsif e.respond_to?(:iso8601)
@@ -314,6 +359,13 @@ class CAFrame
       end
     end
     table = CArray.object(rows.size, ncol) { rows }
+    # A missing field is UNDEF, not a Ruby nil sitting in a cell.  The
+    # tokenizer says missing with nil (an unquoted empty field; a quoted
+    # one is the empty string and stays a value), and an object array will
+    # hold that nil quite happily -- so a column read without `types:` used
+    # to carry nil where the same column read with one carried UNDEF, and
+    # the mask a to_csv had written did not survive the trip back.
+    table[:eq, nil] = UNDEF
     names.each_with_index { |name, j| cols[name] = table[nil, j] }
     new(cols)
   end

data/lib/carray/frame/records.rb

@@ -59,30 +59,44 @@ class CAFrame
   # native scalar column (nil -> UNDEF); anything else -> object column.
   def self.build_record_column(values, n)
     present = values.reject(&:nil?)
-    return CArray.object(n) { values } if present.empty?
+    return mask_missing(CArray.object(n) { values }) if present.empty?
 
     if present.all? { |v| v.is_a?(Array) || v.is_a?(CArray) }
       build_nd_column(values, present, n)
     else
       type = numeric_leaf_type(present)
       col = CArray.object(n) { values }
-      type ? col.to_type(type) : col
+      type ? col.to_type(type) : mask_missing(col)
     end
   end
   private_class_method :build_record_column
 
+  # A missing cell is UNDEF, not a Ruby nil sitting in a cell.  to_records
+  # writes a masked cell as nil, so nil on the way back in is the only
+  # spelling a missing cell has; to_type does this conversion for a numeric
+  # column, and an object column would otherwise keep the nil as a value --
+  # a row with no label coming back as a row labelled nil.  The CSV reader
+  # takes the same position (see build_frame in io.rb).
+  def self.mask_missing(col)
+    col[:eq, nil] = UNDEF
+    col
+  end
+  private_class_method :mask_missing
+
   # Stack equal-length array cells into an (N, L) column via an object 2-D fill
   # + to_type (nil rows -> UNDEF, int/float by leaf). Ragged lengths or
   # non-numeric leaves fall back to a 1-D object column of the raw cells.
   def self.build_nd_column(values, present, n)
     lengths = present.map { |v| v.is_a?(CArray) ? v.shape[0] : v.size }
     len = lengths.first
-    return CArray.object(n) { values } unless lengths.all? { |x| x == len }
+    unless lengths.all? { |x| x == len }
+      return mask_missing(CArray.object(n) { values })
+    end
 
     nested = values.map { |v| v.nil? ? Array.new(len) : (v.is_a?(CArray) ? v.to_a : v) }
     type = numeric_leaf_type(nested.flatten.compact)
     table = CArray.object(n, len) { nested }
-    type ? table.to_type(type) : table
+    type ? table.to_type(type) : mask_missing(table)
   end
   private_class_method :build_nd_column
 

data/lib/carray/frame/verbs.rb

@@ -230,7 +230,9 @@ class CAFrame
       return
     end
     yvalid = col.value.float64[present]
-    col[] = yvalid.linear_fetch(addr).to_type(col.data_type).mask_invalid   # write-through
+    # mask_invalid before the cast: linear_fetch marks out of range with NaN,
+    # and casting back to an integer column would turn it into a plausible 0.
+    col[] = yvalid.linear_fetch(addr).mask_invalid.to_type(col.data_type)  # write-through
   end
 
   private def cast_one(name, type)
@@ -267,16 +269,17 @@ class CAFrame
             "promote takes a data type Symbol (got #{type.class}); " \
             "class-shaped targets are not promotion destinations"
     end
-    @columns.each_key do |key|
-      col = @columns[key]
-      # A Face column answers for itself: :object is its surface values, a
-      # numeric target is whatever it declares in #to_numeric (and a TypeError
-      # naming that method when it declares nothing). result_type has nothing
-      # to say about a surface it cannot read, so the widening check -- which
-      # is about primitive promotion -- applies to plain columns only.
-      refuse_narrowing(key, col, type) unless col.face?
-      @columns[key] = col.to_type(type)
-    end
+    # Check every column before rebinding any. A column that would narrow
+    # rejects the whole promote, and rejecting part way through would leave the
+    # frame promoted in whichever columns happened to come first.
+    #
+    # A Face column answers for itself: :object is its surface values, a
+    # numeric target is whatever it declares in #to_numeric (and a TypeError
+    # naming that method when it declares nothing). result_type has nothing
+    # to say about a surface it cannot read, so the widening check -- which
+    # is about primitive promotion -- applies to plain columns only.
+    @columns.each { |key, col| refuse_narrowing(key, col, type) unless col.face? }
+    @columns.each_key { |key| @columns[key] = @columns[key].to_type(type) }
   end
 
   private def refuse_narrowing(key, col, type)

data/lib/carray/inspect.rb

@@ -7,7 +7,12 @@ class CArray::Inspector  # :nodoc:
   end
 
   # @!visibility private
-  def inspect_string
+  #
+  #  `abbrev` false renders every element instead of eliding with `...`.
+  #  It is the one difference between #inspect and #inspect_full: the
+  #  header and the layout are the same, so there is one renderer rather
+  #  than two that could drift apart.
+  def inspect_string (abbrev: true)
     if @carray.ndim == 0
       raise "can't inspect CArray of ndim == 0"
     end
@@ -15,7 +20,7 @@ class CArray::Inspector  # :nodoc:
     class_name = get_class_name()
     type_name  = get_type_name()
     shape      = get_shape()
-    data_spec  = get_data_spec(0, Array.new(@carray.ndim){0}, formatter)
+    data_spec  = get_data_spec(0, Array.new(@carray.ndim){0}, formatter, abbrev)
     info_list  = get_info_list()
     output = ["<",
               format("%s.%s(%s)", class_name, type_name, shape.join(",")),
@@ -146,7 +151,7 @@ class CArray::Inspector  # :nodoc:
     end
   end
 
-  def get_data_spec (level, idx, formatter)
+  def get_data_spec (level, idx, formatter, abbrev = true)
     io = +"[ "  # mutable buffer; `<<` below appends into it
     ndim = @carray.ndim
     dim  = @carray.shape
@@ -163,7 +168,7 @@ class CArray::Inspector  # :nodoc:
         if i != dim[level] - 1
           io << ", "
         end
-        if io.length > 48 - 2*level
+        if abbrev and io.length > 48 - 2*level
           if i < dim[level] - 1
             io << "..."
             over = true
@@ -182,24 +187,24 @@ class CArray::Inspector  # :nodoc:
       end
     else
       over = false
-      show = [dim[level], 5].min
+      show = abbrev ? [dim[level], 5].min : dim[level]
       show.times do |i|
         idx[level] = i
-        io << get_data_spec(level+1, idx, formatter)
+        io << get_data_spec(level+1, idx, formatter, abbrev)
         if i < show - 1
           io << ",\n" + "  " * (level+1)
         end
-        if i >= 2 and dim[level] > 5
+        if abbrev and i >= 2 and dim[level] > 5
           break
         end
       end
-      if dim[level] > 5
+      if abbrev and dim[level] > 5
         io << "... ... ..."
         over = true
       end
       if over
         idx[level] = dim[level] - 1
-        io << "\n"+ "  " * (level+1) + get_data_spec(level+1, idx, formatter)
+        io << "\n"+ "  " * (level+1) + get_data_spec(level+1, idx, formatter, abbrev)
       end
     end
     io << " ]"
@@ -219,6 +224,34 @@ class CArray
     return CArray::Inspector.new(self).inspect_string
   end
 
+  # @overload inspect_full
+  #   The same description as {#inspect}, with every element rendered
+  #   instead of the `...` preview.
+  #
+  #   `inspect` abbreviates on purpose -- it is what `p`, `irb` and an
+  #   error message call, and a million-cell array has to stay readable
+  #   there. `inspect_full` is for the other moment, when the whole
+  #   array is the thing you came to look at:
+  #
+  #     puts a.inspect_full
+  #
+  #   The header, the layout and the `_` for a masked cell are
+  #   `inspect`'s; only the eliding is dropped, so for an array small
+  #   enough that `inspect` was not abbreviating anything the two give
+  #   the same string.
+  #
+  #   Note that neither of these is `to_s`, which returns the **raw
+  #   bytes** of the storage rather than anything printable.
+  #
+  #   The result is one String holding every element, so it is as large
+  #   as the array is: nothing here is streamed, and a line is as long
+  #   as the last axis makes it.
+  #
+  #   @return [String]
+  def inspect_full
+    return CArray::Inspector.new(self).inspect_string(abbrev: false)
+  end
+
   private
 
   def desc  

data/lib/carray/iterator.rb

@@ -36,6 +36,114 @@ class CAIterator
   # rather than reading as "no such method" -- that gap is the member's to close.
   # A member that genuinely cannot provide one overrides it to raise with its
   # own reason.
+  # The methods below are declared here, so that the family surface is
+  # documented in one place; each member generates its own implementation and
+  # a member for which one is ill-defined overrides it to raise with a reason.
+  # "Piece" is the unit a member iterates over: a slab, a window, a block, a
+  # category, or a group.
+  #
+  # @!method sum
+  #   Returns the sum of each piece.
+  #   @return [CArray] one value per piece, shaped {#shape}.
+  # @!method accumulate
+  #   Returns the sum of each piece kept in the source's own `data_type`,
+  #   wrapping at its width, where {#sum} answers in the type the core
+  #   promotes to (`:float64` for integers).
+  #   @return [CArray] one value per piece.
+  # @!method prod
+  #   Returns the product of each piece.
+  #   @return [CArray] one value per piece.
+  # @!method mean
+  #   Returns the arithmetic mean of each piece.
+  #   @return [CArray] one value per piece.
+  # @!method min
+  #   Returns the smallest value in each piece.
+  #   @return [CArray] one value per piece.
+  # @!method max
+  #   Returns the largest value in each piece.
+  #   @return [CArray] one value per piece.
+  # @!method minmax
+  #   Returns the smallest and largest value of each piece, found in one pass.
+  #   @return [Array<CArray>] the pair `[min, max]`.
+  # @!method variance
+  #   Returns the sample variance (divisor `n - 1`) of each piece.
+  #   @return [CArray] one value per piece.
+  # @!method variancep
+  #   Returns the population variance (divisor `n`) of each piece.
+  #   @return [CArray] one value per piece.
+  # @!method stddev
+  #   Returns the sample standard deviation (divisor `n - 1`) of each piece.
+  #   @return [CArray] one value per piece.
+  # @!method stddevp
+  #   Returns the population standard deviation (divisor `n`) of each piece.
+  #   @return [CArray] one value per piece.
+  # @!method all
+  #   Returns whether every cell of each piece is true.
+  #   @return [CArray] `:boolean`, one value per piece.
+  # @!method any
+  #   Returns whether any cell of each piece is true.
+  #   @return [CArray] `:boolean`, one value per piece.
+  # @!method min_index
+  #   Returns the position of the smallest value **within** each piece.
+  #   @return [CArray] one index per piece.
+  # @!method max_index
+  #   Returns the position of the largest value within each piece.
+  #   @return [CArray] one index per piece.
+  # @!method min_addr
+  #   Returns the flat address **in the source** of the smallest value of each
+  #   piece -- which source cell holds it, rather than where it sits inside the
+  #   piece. Use it to read the same cell out of another source-shaped array.
+  #   @return [CArray] one flat address per piece.
+  # @!method max_addr
+  #   Returns the flat address in the source of the largest value of each piece.
+  #   @return [CArray] one flat address per piece.
+  # @!method wsum(weights)
+  #   Returns the weighted sum of each piece.
+  #   @param weights [CArray] one weight per source cell, shaped like the source.
+  #   @return [CArray] one value per piece.
+  # @!method wmean(weights)
+  #   Returns the weighted mean of each piece.
+  #   @param weights [CArray] one weight per source cell, shaped like the source.
+  #   @return [CArray] one value per piece.
+  # @!method median
+  #   Returns the median of each piece.
+  #   @return [CArray] one value per piece.
+  # @!method percentile(*pers)
+  #   Returns the requested percentile(s) of each piece.
+  #   @param pers [Array<Numeric>] percentile positions in `0..100`.
+  #   @return [CArray, Array<CArray>] one array per requested position; a
+  #     single position returns that array directly.
+  # @!method quantile
+  #   Returns the five-number summary of each piece,
+  #   `[min, Q1, median, Q3, max]`.
+  #   @return [Array<CArray>] five arrays, one per position.
+  # @!method count(value = nil)
+  #   Returns a count per piece: with no argument the cells that are not
+  #   masked, with `UNDEF` the masked cells, and with any other value the
+  #   cells equal to it.
+  #   @param value [Object] value to match, or `UNDEF`.
+  #   @return [CArray] one count per piece.
+  # @!method count_not_masked
+  #   Returns the number of cells of each piece that are not masked.
+  #   @return [CArray] one count per piece.
+  # @!method count_masked
+  #   Returns the number of masked cells of each piece.
+  #   @return [CArray] one count per piece.
+  # @!method elements
+  #   Returns the total number of cells in each piece, masked or not.
+  #   @return [CArray] one count per piece.
+  # @!method each
+  #   Yields each piece in turn as a CArray.
+  #   @yieldparam piece [CArray]
+  #   @return [Enumerator, self] an Enumerator when no block is given.
+  # @!method reduce(init = nil)
+  #   Folds the pieces with a block, for a reduction the family does not name.
+  #   Without `init` the first piece seeds the accumulator.
+  #   @param init [Object] initial accumulator value.
+  #   @yieldparam acc [Object] running accumulator.
+  #   @yieldparam piece [CArray] next piece.
+  #   @yieldreturn [Object] updated accumulator.
+  #   @return [Object] the final accumulator.
   [
     :sum, :accumulate, :prod, :mean, :min, :max,                      # tier 1
     :variance, :stddev, :all, :any,
@@ -66,6 +174,41 @@ class CAIterator
   # sort_addr.  Un-overridden each is simply unavailable, not a contract
   # violation.  min_addr / max_addr stay required: a single winner address is
   # well-defined even for an overlapping window.
+  # @!method map
+  #   Returns a source-shaped array built by applying the block to each piece
+  #   and scattering the result back into that piece's cells.
+  #   @yieldparam piece [CArray]
+  #   @yieldreturn [CArray, Numeric] replacement values for the piece.
+  #   @return [CArray] shaped like the source.
+  #   @raise [NotImplementedError] for a member whose pieces overlap, where a
+  #     cell would receive more than one value.
+  # @!method sort_addr
+  #   Returns a source-shaped array whose cells, read piece by piece, give the
+  #   flat source addresses that put that piece in ascending order.
+  #   @return [CArray] `:int64`, shaped like the source.
+  #   @raise [NotImplementedError] for a member whose pieces overlap.
+  # @!method cumsum
+  #   Returns a source-shaped array of the running sum within each piece.
+  #   @return [CArray] shaped like the source.
+  #   @raise [NotImplementedError] for a member whose pieces overlap, where a
+  #     cell has no single running value.
+  # @!method cumprod
+  #   Returns a source-shaped array of the running product within each piece.
+  #   @return [CArray] shaped like the source.
+  #   @raise [NotImplementedError] for a member whose pieces overlap.
+  # @!method cummax
+  #   Returns a source-shaped array of the running maximum within each piece.
+  #   @return [CArray] shaped like the source.
+  #   @raise [NotImplementedError] for a member whose pieces overlap.
+  # @!method cummin
+  #   Returns a source-shaped array of the running minimum within each piece.
+  #   @return [CArray] shaped like the source.
+  #   @raise [NotImplementedError] for a member whose pieces overlap.
+  # @!method cumcount
+  #   Returns a source-shaped array of the running count of cells that are not
+  #   masked within each piece.
+  #   @return [CArray] shaped like the source.
+  #   @raise [NotImplementedError] for a member whose pieces overlap.
   [:map, :sort_addr,
    :cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |name|
     define_method(name) do |*, **, &_blk|

data/lib/carray/lazy.rb

@@ -901,41 +901,4 @@ class << CArray
     end
     CArray::FuseSource.evaluate(block)
   end
-
-  # @overload jit_for (*extents) { |i, j, ...| ... }
-  #   Runs a block over an index space, so that a cell may reach the ones
-  #   around it -- a recurrence, a stencil.  The block is compiled, and the
-  #   compiler is the carray-jit gem.
-  #   @raise [NotImplementedError] when that gem is not installed.
-  def jit_for (*extents, **options)
-    raise NotImplementedError, no_compiler("jit_for")
-  end
-
-  # @overload jit_each { ... }
-  #   Runs a block over the cells of arrays at once, naming each cell by the
-  #   array it came from, and writing what it computes into arrays of yours:
-  #   `CArray.jit_each { out = a + b * c }`.  The block is compiled, and the
-  #   compiler is the carray-jit gem.
-  #   @raise [NotImplementedError] when that gem is not installed.
-  def jit_each (&block)
-    raise NotImplementedError, no_compiler("jit_each")
-  end
-
-  # @overload jit_map { ... }
-  #   The same block with its value asked for: the last statement is what
-  #   every cell of the result gets, and the result comes back.  The block is
-  #   compiled, and the compiler is the carray-jit gem.
-  #   @raise [NotImplementedError] when that gem is not installed.
-  def jit_map (&block)
-    raise NotImplementedError, no_compiler("jit_map")
-  end
-
-  private
-
-  def no_compiler (name)
-    "CArray.#{name} compiles its block, and the compiler is the carray-jit " \
-    "gem, which is not installed.  An expression over whole arrays can be " \
-    "written as `CArray.fuse { ... }` instead, which needs no compiler; " \
-    "reaching a neighbouring cell, or writing the loop itself, does."
-  end
 end

data/lib/carray/mask_gap_fill.rb

@@ -172,7 +172,9 @@ class CArray
       return vec[present].linear_fetch(addr)
     end
     vval = vec.value.float64[present]         # valid values
-    vval.linear_fetch(addr).to_type(vec.data_type).mask_invalid
+    # Mark the out-of-range NaN before casting back: for an integer data_type
+    # the cast turns NaN into 0, and mask_invalid then has nothing left to find.
+    vval.linear_fetch(addr).mask_invalid.to_type(vec.data_type)
   end
 
   # Yield an index key (Array with `nil` at `axis`, integers elsewhere)

data/lib/carray/methods/discovery_along.rb

@@ -0,0 +1,74 @@
+class CArray
+
+  #  The value-hash discovery family (`unique`, `nunique`,
+  #  `mask_duplicates`, ...) interns one **cell** at a time.  `along:`
+  #  widens the unit to a whole sub-array: the sub-arrays enumerated
+  #  along one axis, each compared as a single value.
+  #
+  #  Nothing new has to hash.  `fz_hash`'s third key lane already interns
+  #  a fixed-width block of bytes (FNV-1a, with a memcmp re-check), which
+  #  is what a sub-array is once it sits contiguously; this method is the
+  #  bridge to it, and the three things the bridge has to get right:
+  #
+  #    - the sub-arrays have to be contiguous before their bytes mean
+  #      anything, which a transpose or a strided view does not give;
+  #    - byte equality is not the family's contract for floats, where
+  #      every NaN is one value and -0.0 is +0.0;
+  #    - masks fold for free -- a CARefer that covers several parent
+  #      cells with one view cell OR-reduces their mask bits, so a
+  #      sub-array holding a masked cell is a masked sub-array and the
+  #      kernels skip it.
+  #
+  #  Returns a 1-D fixlen array, one cell per sub-array, for the caller
+  #  to run an ordinary cell-level discovery method over.
+  private def fibers_as_cells (along, caller_name)
+    if ndim < 2
+      raise ArgumentError,
+            "#{caller_name}: along: compares sub-arrays, and a #{ndim}-D array " \
+            "has none -- drop along: for the whole-array form"
+    end
+
+    if data_type == CA_OBJECT
+      raise ArgumentError,
+            "#{caller_name}: along: is not available for an object array -- " \
+            "its cells hold Ruby references, which would compare by identity " \
+            "rather than by value"
+    end
+
+    axis = normalize_axis(along, caller_name)
+
+    #  Bring the named axis to the front so each sub-array is one
+    #  unbroken run of bytes, and take a copy: `refer` reinterprets a
+    #  byte buffer, and a view whose element stride is not its element
+    #  width is not one.  The copy is also what makes the normalisation
+    #  below safe to write.
+    order = [axis] + (0...ndim).to_a.reject { |i| i == axis }
+    block = (axis.zero? ? self : transpose(*order)).copy
+
+    #  Two values the family calls equal are not equal byte for byte.
+    #  Normalise them so the bytes say what the values mean.
+    if block.float?
+      block[:is_nan] = Float::NAN     #  every NaN is one value
+      block[:eq, 0.0] = 0.0           #  -0.0 is +0.0
+    elsif block.complex?
+      [block.real, block.imag].each do |part|
+        part[:is_nan] = Float::NAN
+        part[:eq, 0.0] = 0.0
+      end
+    end
+
+    count = shape[axis]
+    block.refer(:fixlen, [count], bytes: elements / count * bytes)
+  end
+
+  #  `axis:` and `along:` ask different questions of the same array, so
+  #  answering both at once has no meaning.
+  private def reject_axis_with_along (axis, along, caller_name)
+    return unless axis and along
+    raise ArgumentError,
+          "#{caller_name}: axis: and along: cannot be given together -- " \
+          "axis: is about the values inside each sub-array, along: is about " \
+          "the sub-arrays themselves"
+  end
+
+end

data/lib/carray/methods/factorize.rb

@@ -0,0 +1,50 @@
+class CArray
+
+  # @overload factorize
+  #   Returns `[codes, levels]` in one pass: `levels` is a 1-D CArray of
+  #   the distinct values of `self` in first-appearance order — what
+  #   {#unique} answers — and `codes` is an integer CArray of `self`'s
+  #   shape where `levels[codes[i]]` is `self[i]`.
+  #
+  #   This is the member of the value-hash discovery family ({#unique},
+  #   {#value_counts}, {#mask_duplicates}, {#nunique}) that hands back
+  #   the codes as storage. {#unique} answers with the vocabulary alone
+  #   and {#categorize} wraps both in a {CACategorical} Face; a caller
+  #   who wants the codes themselves — a position to scatter into, a key
+  #   to group by, a dense renumbering of sparse keys — would otherwise
+  #   pay a second pass or take the Face and its Ruby label list.
+  #
+  #   `codes` takes the narrowest unsigned data type the vocabulary
+  #   fits, reserving that type's top value as the exclusion sentinel.
+  #   A cell that joins no category — a masked cell — is both masked and
+  #   holds the sentinel, exactly as {CACategorical}'s storage is, so a
+  #   consumer may read either.
+  #
+  #   Distinctness is the family's hash-key judgement (see {#unique}):
+  #   `==` for numeric with all NaN collapsed to one value and
+  #   -0.0 / +0.0 the same value; `eql?` / `hash` for `CA_OBJECT` and
+  #   `CA_FIXLEN`. Complex is not a lane the factorizer takes and raises
+  #   {CArray::DataTypeError}.
+  #
+  #   There is no `sort:` here, unlike {#unique} and {#value_counts}:
+  #   the codes index the levels, so reordering the vocabulary would
+  #   desync them. Take {#unique}`(sort: true)` where the codes are not
+  #   wanted, or sort afterwards and carry the codes through the same
+  #   permutation.
+  #
+  #   @return [Array(CArray, CArray)] `[codes, levels]`.
+  def factorize
+    # One linear pass through the shared value hash (C
+    # __factorize_appearance__). It writes the sentinel into the
+    # excluded cells but leaves the codes unmasked; masking them is what
+    # CACategorical.from_codes does on its way to the Face, and this
+    # surface owes the same, since it hands the storage out bare.
+    codes, levels = __factorize_appearance__
+    excluded = codes.eq(CACategorical::SENTINEL[codes.data_type])
+    if excluded.count(true) > 0
+      codes.mask = codes.has_mask? ? (codes.mask | excluded) : excluded
+    end
+    [codes, levels]
+  end
+
+end