carray 3.0.3 → 3.0.5

data/ext/mkkernel.rb

@@ -44,26 +44,29 @@ module MkKernel
   # value_arg: dispatchers (CF.1) to cast a positional Ruby argument to
   # the kernel's T_IN type.  Signed integers use NUM2LL, unsigned
   # NUM2ULL, floats NUM2DBL, with the dispatcher adding the per-type cast.
+  # `c` of a multi-byte type is its unaligned typedef from carray.h
+  # (ca_u_float64_t): a view's cells may sit at any byte, and a kernel
+  # reads them through `c *`.
   DTYPES = {
     i8:  { c: "int8_t",   ca: "CA_INT8",    ruby: "LL2NUM",  num2c: "NUM2LL",
            limit_hi: "INT8_MAX",   limit_lo: "INT8_MIN" },
     u8:  { c: "uint8_t",  ca: "CA_UINT8",   ruby: "ULL2NUM", num2c: "NUM2ULL",
            limit_hi: "UINT8_MAX",  limit_lo: "0" },
-    i16: { c: "int16_t",  ca: "CA_INT16",   ruby: "LL2NUM",  num2c: "NUM2LL",
+    i16: { c: "ca_u_int16_t",  ca: "CA_INT16",   ruby: "LL2NUM",  num2c: "NUM2LL",
            limit_hi: "INT16_MAX",  limit_lo: "INT16_MIN" },
-    u16: { c: "uint16_t", ca: "CA_UINT16",  ruby: "ULL2NUM", num2c: "NUM2ULL",
+    u16: { c: "ca_u_uint16_t", ca: "CA_UINT16",  ruby: "ULL2NUM", num2c: "NUM2ULL",
            limit_hi: "UINT16_MAX", limit_lo: "0" },
-    i32: { c: "int32_t",  ca: "CA_INT32",   ruby: "LL2NUM",  num2c: "NUM2LL",
+    i32: { c: "ca_u_int32_t",  ca: "CA_INT32",   ruby: "LL2NUM",  num2c: "NUM2LL",
            limit_hi: "INT32_MAX",  limit_lo: "INT32_MIN" },
-    u32: { c: "uint32_t", ca: "CA_UINT32",  ruby: "ULL2NUM", num2c: "NUM2ULL",
+    u32: { c: "ca_u_uint32_t", ca: "CA_UINT32",  ruby: "ULL2NUM", num2c: "NUM2ULL",
            limit_hi: "UINT32_MAX", limit_lo: "0" },
-    i64: { c: "int64_t",  ca: "CA_INT64",   ruby: "LL2NUM",  num2c: "NUM2LL",
+    i64: { c: "ca_u_int64_t",  ca: "CA_INT64",   ruby: "LL2NUM",  num2c: "NUM2LL",
            limit_hi: "INT64_MAX",  limit_lo: "INT64_MIN" },
-    u64: { c: "uint64_t", ca: "CA_UINT64",  ruby: "ULL2NUM", num2c: "NUM2ULL",
+    u64: { c: "ca_u_uint64_t", ca: "CA_UINT64",  ruby: "ULL2NUM", num2c: "NUM2ULL",
            limit_hi: "UINT64_MAX", limit_lo: "0" },
-    f32: { c: "float",    ca: "CA_FLOAT32", ruby: "rb_float_new", num2c: "NUM2DBL",
+    f32: { c: "ca_u_float32_t",    ca: "CA_FLOAT32", ruby: "rb_float_new", num2c: "NUM2DBL",
            limit_hi: "INFINITY",   limit_lo: "-INFINITY" },
-    f64: { c: "double",   ca: "CA_FLOAT64", ruby: "rb_float_new", num2c: "NUM2DBL",
+    f64: { c: "ca_u_float64_t",   ca: "CA_FLOAT64", ruby: "rb_float_new", num2c: "NUM2DBL",
            limit_hi: "INFINITY",   limit_lo: "-INFINITY" },
     # E.6a (2026-06-03): boolean as a first-class source DTYPE.
     # Intentionally NOT included in ALL_NUMERIC -- kernels opt boolean in
@@ -90,11 +93,11 @@ module MkKernel
     # CA_NTYPE coverage parity with mkmath, mkkernel needs to know the
     # complex and object data_types too.  These entries are NOT included in
     # ALL_NUMERIC; pick up via MATH_TYPES / MATH_NUMERIC aliases.
-    cmplx64:  { c: "cmplx64_t",  ca: "CA_CMPLX64",  ruby: "CC2NUM",
+    cmplx64:  { c: "ca_u_cmplx64_t",  ca: "CA_CMPLX64",  ruby: "CC2NUM",
                 num2c: "NUM2CC",  limit_hi: "0", limit_lo: "0" },
-    cmplx128: { c: "cmplx128_t", ca: "CA_CMPLX128", ruby: "CC2NUM",
+    cmplx128: { c: "ca_u_cmplx128_t", ca: "CA_CMPLX128", ruby: "CC2NUM",
                 num2c: "NUM2CC",  limit_hi: "0", limit_lo: "0" },
-    object:   { c: "VALUE",      ca: "CA_OBJECT",   ruby: "(VALUE)",
+    object:   { c: "VALUE",           ca: "CA_OBJECT",   ruby: "(VALUE)",
                 num2c: "(VALUE)", limit_hi: "0", limit_lo: "0" },
     # P.5b.4: fixlen for bincmp (= match, eq/ne/gt/lt/ge/le against
     # fixed-byte strings).  The c: type is `char` because fixlen kernels
@@ -177,10 +180,8 @@ module MkKernel
   #   expr: { object: "..." }.merge(float_widths("(#2) = sin<f>(#1);"))
   #
   # Not every double-taking call wants this.  `fabs`, `fmin` / `fmax`,
-  # `ceil` / `floor` / `trunc` and the `isnan` family are exact on a
-  # float either way, and `round` (`floor(x + 0.5)`) is exact only in
-  # double -- narrowing it would round `x + 0.5f` first and step the
-  # answer at the boundary.
+  # `ceil` / `floor` / `trunc` / `round` and the `isnan` family are exact
+  # on a float either way.
   def self.float_widths(body)
     {
       [:f32] => at_width(body, "f", "float"),
@@ -706,8 +707,13 @@ module MkKernel
   #                    the input is masked).  Aligns core cummax / cummin with
   #                    the reduction contract (empty max / min = UNDEF, no
   #                    identity) and the axis-group scan family.
+  # private:
+  #   false (default) -> bound as the public `<name>_ki`
+  #   true            -> bound only as the private method `<name>`, a part
+  #                      that lib reaches with `send` (not user surface)
   def self.scan(name, source:, output:, init:, step:, fallback: :raise,
-                acc_type: nil, axis_default: nil, empty: nil, face_gate: nil)
+                acc_type: nil, axis_default: nil, empty: nil, face_gate: nil,
+                private: false)
     raise "duplicate kernel #{name}" if KERNELS.any? { |k| k[:name] == name }
     source.each do |s|
       raise "unknown source data_type #{s}" unless DTYPES.key?(s)
@@ -749,6 +755,7 @@ module MkKernel
       axis_default: axis_default,
       empty:        empty,
       face_gate:    face_gate,
+      private:      private,
     }
   end
 
@@ -944,7 +951,7 @@ module MkKernel
   # is up to the body (= value-returning search variants).
   def self.search(name, source:, output:, body:,
                   no_match: "-1", no_match_check: nil, undef_no_match: false,
-                  mask_self: :raise, mask_query: :ignore, runtime_args: nil,
+                  mask_self: :raise, runtime_args: nil,
                   fallback: :raise, semantics: :fiber_local)
     raise "search: unknown semantics #{semantics} (expected :fiber_local or :view_flat)" \
       unless %i[fiber_local view_flat].include?(semantics)
@@ -956,12 +963,10 @@ module MkKernel
     raise "search: no_match must be a String" unless no_match.is_a?(String)
     raise "search: unknown mask_self #{mask_self} (expected :raise | :skip | :ignore)" \
       unless %i[raise skip ignore].include?(mask_self)
-    # mask_query: what a masked query cell means.  :ignore reads the value
-    # under the mask (the historical behaviour).  :undef treats the query as
-    # having no answer and marks the corresponding output cell UNDEF, so an
-    # undetermined query cannot silently become a determined result.
-    raise "search: unknown mask_query #{mask_query} (expected :ignore | :undef)" \
-      unless %i[ignore undef].include?(mask_query)
+    # A masked query cell (or a scalar UNDEF query) has no answer: its output
+    # cell is UNDEF, so an undetermined query cannot become a determined
+    # result from the value that happens to sit under the mask.  There is no
+    # option to read under the mask.
     raise "search: unknown fallback #{fallback} (expected :raise)" \
       unless fallback == :raise
 
@@ -1026,7 +1031,6 @@ module MkKernel
       no_match_check:  no_match_check,
       undef_no_match:  undef_no_match,
       mask_self:       mask_self,
-      mask_query:      mask_query,
       runtime_args:    runtime_args || [],
       fallback:        fallback,
       semantics:       semantics,
@@ -1774,13 +1778,15 @@ module MkKernel
 
   # Emits the native function of one source data type.  The object lane
   # calls Ruby for every cell, so any cell can raise in the middle of a walk;
-  # its function is rebuilt around ca_iter_ensure (see guard_object_native).
-  # Every other lane is emitted as the emitter writes it.
-  def self.emit_native_for(io, k, src)
-    if src == :object
+  # its function is rebuilt around ca_iter_ensure (see guard_native).  So is
+  # every lane of a family that passes guard: true, whose walks allocate
+  # (an output mask made on the first empty slab, a scratch buffer), since a
+  # failed allocation raises too.  Other lanes are emitted as written.
+  def self.emit_native_for(io, k, src, guard: false)
+    if src == :object || guard
       buf = StringIO.new
       yield buf
-      io.print guard_object_native(buf.string, "#{k[:name]}_ki_native_object")
+      io.print guard_native(buf.string, "#{k[:name]}_ki_native_#{src}")
     else
       yield io
     end
@@ -1821,7 +1827,7 @@ module MkKernel
   # frame is gone once the body raises.  The body text is kept as written:
   # the parameters are unpacked from the frame under their own names, and
   # each state name is defined to its slot for the length of the body.
-  def self.guard_object_native(text, fn)
+  def self.guard_native(text, fn)
     header = /^static VALUE\n#{fn} \(([^)]*)\)\n\{\n/
     m = header.match(text) or raise "mkkernel: #{fn} not found"
     close = matching_brace(text, m.end(0) - 2)
@@ -1844,9 +1850,9 @@ module MkKernel
 
     out = StringIO.new
     out.print m.pre_match
-    out.puts "/* The object lane calls Ruby for every cell, so a walk can raise part"
-    out.puts "   way.  Its states live in the caller's frame, and ca_iter_ensure"
-    out.puts "   finishes them however the body leaves. */"
+    out.puts "/* The walks can raise part way (a Ruby call per cell in the object"
+    out.puts "   lane, an allocation in any lane).  Their states live in the caller's"
+    out.puts "   frame, and ca_iter_ensure finishes them however the body leaves. */"
     out.puts "typedef struct {"
     out.puts "  ca_iter_state states[#{states.size}];"
     params.each { |type, name| out.puts "  #{type} #{name};".sub(/\* /, "*") }
@@ -1881,7 +1887,7 @@ module MkKernel
     io.puts
     io.puts "/* ===== #{k[:name]}_ki ============================================ */"
     k[:source].each do |src|
-      emit_native_for(io, k, src) { |o| emit_reduce_native(o, k, src) }
+      emit_native_for(io, k, src, guard: true) { |o| emit_reduce_native(o, k, src) }
     end
     emit_reduce_dispatch(io, k)
   end
@@ -2178,20 +2184,22 @@ module MkKernel
     else
       if k[:array_arg]
         wc = (k[:array_arg][:data_type] == :promote) ? "double" : si[:c]
-        io.puts "      CA_SLAB_REDUCE_ARRAY_T(#{si[:c]}, #{wc}, st, p, m, st_w, p_w, #{acc_var}, #{acc_init}, #{reduce_stmt});"
+        io.puts "      ca_size_t __sra_throwaway_mc = 0;"
+        io.puts "      CA_SLAB_REDUCE_ARRAY_T_EX(#{si[:c]}, #{wc}, st, p, m, st_w, p_w, #{acc_var}, #{acc_init}, #{reduce_stmt}, __sra_throwaway_mc);"
+        io.puts "      (void) __sra_throwaway_mc;"
       else
         suffix = MkKernel.reduce_macro_suffix(k, src)
-        # Non-mask_policy path: emit a throwaway masked_cnt for the 8-way
-        # macros (which always take one).  Falls through to the legacy
-        # non-EX macro when the 8-way path is not eligible (:object src).
-        if MkKernel.reduce_8way_eligible?(k, src, suffix)
+        # Non-mask_policy path: the slab macros always take a masked_cnt,
+        # so hand them a throwaway.  The generic form (suffix "", the
+        # :object lane) has a wrapper that hides it.
+        if suffix == ""
+          io.puts "      CA_SLAB_REDUCE_T(#{si[:c]}, st, p, m, #{acc_var}, #{acc_init}, #{reduce_stmt});"
+        else
           io.puts "      ca_size_t __sr_throwaway_mc = 0;"
           emit_reduce_slab_call(io, k, src, si, oi, suffix, acc_var, acc_init,
                                 reduce_stmt, "__sr_throwaway_mc",
                                 indent: "      ")
           io.puts "      (void) __sr_throwaway_mc;"
-        else
-          io.puts "      CA_SLAB_REDUCE_T#{suffix}(#{si[:c]}, st, p, m, #{acc_var}, #{acc_init}, #{reduce_stmt});"
         end
       end
       finish_emit = view_flat \
@@ -3549,9 +3557,12 @@ module MkKernel
       wio.puts "  else {"
       wio.puts "    /* CArray val: coerce to the weight storage type (f64 for :promote, so a"
       wio.puts "       float weight against an int source is not truncated) */"
-      wio.puts "    vcw = rb_ca_wrap_readonly(rweights, INT2NUM(#{waca}));"
+      wio.puts "    /* vcv holds the view while cv is read: W-A2 below moves vcw on to"
+      wio.puts "       the materialised copy before it is done with cv. */"
+      wio.puts "    volatile VALUE vcv = rb_ca_wrap_readonly(rweights, INT2NUM(#{waca}));"
       wio.puts "    CArray *cv;"
-      wio.puts "    GetCArray(vcw, cv);"
+      wio.puts "    GetCArray(vcv, cv);"
+      wio.puts "    vcw = vcv;"
       wio.puts ""
       wio.puts "    /* rev5 form detection: A3 (same ndim) -> A2 (1-D axis-broadcast) -> raise */"
       wio.puts "    if ( cv->ndim == src->ndim ) {"
@@ -3565,17 +3576,21 @@ module MkKernel
       wio.puts "      cw = cv;"
       wio.puts "    }"
       wio.puts "    else if ( naxes == 1 && cv->ndim == 1 && cv->dim[0] == src->dim[slab_axes[0]] ) {"
-      wio.puts "      /* W-A2: 1-D axis-broadcast -> materialize to full shape */"
-      wio.puts "      ca_attach(cv);   /* window: nothing raises inside */"
+      wio.puts "      /* W-A2: 1-D axis-broadcast -> materialize to full shape.  cw and"
+      wio.puts "         its mask are made before cv is attached: making them allocates. */"
       wio.puts "      vcw = rb_carray_new(#{waca}, src->ndim, src->dim, 0, NULL);"
       wio.puts "      GetCArray(vcw, cw);"
+      wio.puts "      if ( ca_has_mask(cv) ) {"
+      wio.puts "        ca_create_mask(cw);"
+      wio.puts "      }"
+      wio.puts "      ca_attach(cv);   /* window: nothing raises inside */"
       wio.puts "      /* broadcast cv (1-D, length src->dim[slab_axes[0]]) along reduce axis */"
       wio.puts "      {"
       wio.puts "        int8_t baxis = slab_axes[0];"
       wio.puts "        ca_size_t inner_stride = 1;"
       wio.puts "        for ( int8_t ii = baxis + 1; ii < src->ndim; ii++ ) inner_stride *= src->dim[ii];"
       wio.puts "        ca_size_t axis_n = src->dim[baxis];"
-      wio.puts "        ca_size_t outer  = src->elements / (axis_n * inner_stride);"
+      wio.puts "        ca_size_t outer  = ( axis_n && inner_stride ) ? src->elements / (axis_n * inner_stride) : 0;   /* a zero-length axis leaves nothing to fill */"
       wio.puts "        for ( ca_size_t o = 0; o < outer; o++ ) {"
       wio.puts "          for ( ca_size_t a = 0; a < axis_n; a++ ) {"
       wio.puts "            for ( ca_size_t in = 0; in < inner_stride; in++ ) {"
@@ -3587,15 +3602,14 @@ module MkKernel
       wio.puts "      }"
       wio.puts "      /* Propagate cv mask via W-A3 overlay path below (= mask materialize) */"
       wio.puts "      if ( ca_has_mask(cv) ) {"
-      wio.puts "        /* Allocate cw mask + broadcast cv mask along axis */"
-      wio.puts "        ca_create_mask(cw);"
+      wio.puts "        /* broadcast cv mask along axis */"
       wio.puts "        boolean8_t *cw_m = (boolean8_t *)cw->mask->ptr;"
       wio.puts "        boolean8_t *cv_m = (boolean8_t *)cv->mask->ptr;"
       wio.puts "        int8_t baxis = slab_axes[0];"
       wio.puts "        ca_size_t inner_stride = 1;"
       wio.puts "        for ( int8_t ii = baxis + 1; ii < src->ndim; ii++ ) inner_stride *= src->dim[ii];"
       wio.puts "        ca_size_t axis_n = src->dim[baxis];"
-      wio.puts "        ca_size_t outer  = src->elements / (axis_n * inner_stride);"
+      wio.puts "        ca_size_t outer  = ( axis_n && inner_stride ) ? src->elements / (axis_n * inner_stride) : 0;   /* a zero-length axis leaves nothing to fill */"
       wio.puts "        for ( ca_size_t o = 0; o < outer; o++ ) {"
       wio.puts "          for ( ca_size_t a = 0; a < axis_n; a++ ) {"
       wio.puts "            for ( ca_size_t in = 0; in < inner_stride; in++ ) {"
@@ -3813,14 +3827,18 @@ module MkKernel
 
     # empty: :undef -> allocate the output mask up front when the input
     # carries one (a fiber can only go UNDEF on its leading masked cells,
-    # which exist only when the input is masked).  co_root / op_mask are the
-    # value / mask bases used to derive the per-fiber mask base parallel to
-    # the aliased output value slab.
+    # which exist only when the input is masked).  The output is a fresh
+    # row-major entity, and the walk cuts its slabs from a buffer laid out
+    # the same way (the output's memory, or the walk's scratch when it
+    # delivers by transfer), so a slab's offset from st_out.alias_ptr is
+    # its offset in the output, and op_mask + that offset / cell size is the
+    # fiber's mask base.
     mask_setup =
       if gated
+        # Before the walks open: creating the mask allocates, and a raise
+        # inside them would jump past their finish.
         "\n" \
         "    boolean8_t *op_mask = NULL;\n" \
-        "    char       *co_root = (char *) co->ptr;\n" \
         "    if ( ca_has_mask(ca) ) {\n" \
         "      ca_create_mask(co);\n" \
         "      op_mask = (boolean8_t *) co->mask->ptr;\n" \
@@ -3835,7 +3853,7 @@ module MkKernel
         "                         st_out, po, #{init_expr}, #{step_stmt});"
       elsif gated
         "boolean8_t *mo_fiber = op_mask\n" \
-        "            ? op_mask + (ca_size_t)(po - co_root) / (ca_size_t) co->bytes\n" \
+        "            ? op_mask + (ca_size_t)(po - st_out.alias_ptr) / (ca_size_t) co->bytes\n" \
         "            : NULL;\n" \
         "          CA_SLAB_SCAN_T_GATED(#{si[:c]}, #{oi[:c]}, st_in, pi, mi,\n" \
         "                         st_out, po, mo_fiber, #{init_expr}, #{step_stmt});"
@@ -3859,6 +3877,7 @@ module MkKernel
            INIT inside the macro. */
         int8_t slab_axes[CA_RANK_MAX];
         slab_axes[0] = (int8_t) axis;
+      #{mask_setup}
         ca_iter_state st_in, st_out;
         int rc;
         rc = ca_iter_state_init_l2(&st_in,  ca, CA_SLAB_AXES,
@@ -3874,7 +3893,6 @@ module MkKernel
           rb_raise(rb_eRuntimeError,
                    "%s: output init failed rc=%d", ca_calling_method_name(), rc);
         }
-      #{mask_setup}
         char       *pi, *po;
         boolean8_t *mi, *mo;
         while ( ca_iter_state_next_slab_axes(&st_in,  &pi, &mi) &&
@@ -3968,11 +3986,7 @@ module MkKernel
     io.puts "  if ( TYPE(raxis) == T_ARRAY ) {"
     io.puts %Q[    rb_raise(rb_eArgError, "%s: axis must be a single Integer (got Array); a scan over several axes is ambiguous, chain explicitly: a.#{name}(axis: 0).#{name}(axis: 1)", ca_calling_method_name());]
     io.puts "  }"
-    io.puts "  int axis = (int) ca_axis_integer(raxis, NULL);"
-    io.puts "  if ( axis < 0 ) axis += src->ndim;"
-    io.puts "  if ( axis < 0 || axis >= src->ndim ) {"
-    io.puts %Q[    rb_raise(rb_eArgError, "%s: axis %d out of range for ndim %d", ca_calling_method_name(), NUM2INT(raxis), src->ndim);]
-    io.puts "  }"
+    io.puts "  int axis = ca_normalize_axis_arg(raxis, src->ndim, NULL);"
     io.puts "  switch ( src->data_type ) {"
     k[:source].each do |s|
       si = DTYPES[s]
@@ -4134,35 +4148,15 @@ module MkKernel
     io.puts "  const #{name}_pair_#{src} *pa = (const #{name}_pair_#{src} *) a;"
     io.puts "  const #{name}_pair_#{src} *pb = (const #{name}_pair_#{src} *) b;"
     if is_object
-      # PROPOSAL_MKKERNEL_OBJECT_DTYPE_BRANCH Phase 2: CA_OBJECT cmp via
-      # Ruby Comparable (= `<=>`).  Any Comparable element type (Numeric,
-      # String, custom class with <=> defined) works.  Raises Ruby
-      # TypeError mid-sort if a cell pair is uncomparable -- consistent
-      # with Array#sort behavior.
-      # A Float NaN sorts after every other value, as it does in a float
-      # array; two NaNs fall to the index tie-break.
-      io.puts "  int nan_a = CA_OBJ_ISNAN(pa->v);"
-      io.puts "  int nan_b = CA_OBJ_ISNAN(pb->v);"
-      io.puts "  if ( nan_a && !nan_b ) return 1;"
-      io.puts "  if ( nan_b && !nan_a ) return -1;"
-      io.puts "  if ( !nan_a ) {"
-      io.puts %Q{    VALUE r = rb_funcall(pa->v, rb_intern("<=>"), 1, pb->v);}
-      io.puts "    if ( NIL_P(r) ) {"
-      io.puts "      ca_obj_compare_refused(pa->v, pb->v);"
-      io.puts "    }"
-      io.puts "    int c = NUM2INT(r);"
-      io.puts "    if ( c > 0 ) return 1;"
-      io.puts "    if ( c < 0 ) return -1;"
-      io.puts "  }"
+      # The object order: `<=>`, a Float NaN after every other value
+      # (ca_obj_order_cmp); two NaNs fall to the index tie-break.
+      io.puts "  int c = ca_obj_order_cmp(pa->v, pb->v);"
+      io.puts "  if ( c != 0 ) return c;"
+    elsif is_fp
+      # The float order: NaN after every number (CA_FLOAT_ORDER_CMP).
+      io.puts "  int c = CA_FLOAT_ORDER_CMP(pa->v, pb->v);"
+      io.puts "  if ( c != 0 ) return c;"
     else
-      if is_fp
-        # NaN policy :end -- NaN sorts to the end (matches qcmp_f_type in
-        # carray_sort.c).
-        io.puts "  int nan_a = isnan(pa->v);"
-        io.puts "  int nan_b = isnan(pb->v);"
-        io.puts "  if ( nan_a && !nan_b ) return 1;"
-        io.puts "  if ( nan_b && !nan_a ) return -1;"
-      end
       io.puts "  if ( pa->v > pb->v ) return 1;"
       io.puts "  if ( pa->v < pb->v ) return -1;"
     end
@@ -4183,7 +4177,7 @@ module MkKernel
     if is_fixlen
       io.puts "          buf[#{dst}].vp = pi + #{kexpr} * istride; buf[#{dst}].nb = (ca_size_t) ca->bytes;"
     else
-      io.puts "          buf[#{dst}].v = *(#{si[:c]} *) (pi + #{kexpr} * istride);"
+      io.puts "          memcpy(&buf[#{dst}].v, pi + #{kexpr} * istride, sizeof(buf[#{dst}].v));"
     end
   end
 
@@ -4279,21 +4273,10 @@ module MkKernel
       io.puts "     when input has any mask. */"
       io.puts "  if ( ca_has_mask(ca) ) ca_create_mask(co);"
     end
-    io.puts "  ca_iter_state st_in, st_out;"
-    io.puts "  int rc;"
-    io.puts "  rc = ca_iter_state_init_l2(&st_in,  ca, CA_SLAB_AXES,"
-    io.puts "                             slab_axes, 1, 0);"
-    io.puts "  if ( rc != CA_ITER_OK ) {"
-    io.puts %Q[    rb_raise(rb_eRuntimeError, "%s: input init failed rc=%d", ca_calling_method_name(), rc);]
-    io.puts "  }"
-    io.puts "  rc = ca_iter_state_init_l2_paired(&st_out, &st_in, co, CA_SLAB_AXES,"
-    io.puts "                             slab_axes, 1, CA_KERNEL_WRITE);"
-    io.puts "  if ( rc != CA_ITER_OK ) {"
-    io.puts "    ca_iter_state_finish(&st_in);"
-    io.puts %Q[    rb_raise(rb_eRuntimeError, "%s: output init failed rc=%d", ca_calling_method_name(), rc);]
-    io.puts "  }"
-    io.puts
-    io.puts "  ca_size_t fiber_n = st_in.slab_elements;"
+    # The buffers are taken before the walks open: an allocation that
+    # raises inside them would jump past their finish.  The slab is the
+    # sort axis alone, so a fiber holds ca->dim[axis] cells.
+    io.puts "  ca_size_t fiber_n = ca->dim[axis];"
     if src == :object
       # The object comparator calls <=>, so the sort can raise part way.
       # ALLOCV frees these buffers whether the function returns or unwinds.
@@ -4306,19 +4289,39 @@ module MkKernel
         io.puts "  #{name}_pair_#{src} *aux = ALLOCV_N(#{name}_pair_#{src}, haux, fiber_n);"
       end
     else
-      io.puts "  #{name}_pair_#{src} *buf ="
-      io.puts "    xmalloc(sizeof(#{name}_pair_#{src}) * (size_t) fiber_n);"
+      io.puts "  volatile VALUE hbuf = 0, haux = 0;"
+      io.puts "  #{name}_pair_#{src} *buf = ALLOCV_N(#{name}_pair_#{src}, hbuf, fiber_n);"
     end
-    if has_kind && src != :object
-      # PROPOSAL_PORTABLE_TEXTBOOK_SORT §9.5.2 Option γ: aux pair buffer
-      # for ca_sort_merge_pair_<src> (= ping-pong bottom-up mergesort).
-      # Allocated once outside the fiber loop, reused across fibers.
+    if has_kind && %i[f32 f64].include?(src)
+      # aux is the mergesort's ping-pong buffer and, before that, the
+      # scratch of the NaN pre-partition, so a float kernel always takes it.
+      io.puts "  #{name}_pair_#{src} *aux = ALLOCV_N(#{name}_pair_#{src}, haux, fiber_n);"
+    elsif has_kind && src != :object && src != :fixlen
+      # The aux pair buffer for ca_sort_merge_pair_<src> (a ping-pong
+      # bottom-up mergesort), taken once outside the fiber loop.  The fixlen
+      # lane sorts with qsort and needs none.
       io.puts "  #{name}_pair_#{src} *aux = NULL;"
       io.puts "  if ( do_stable ) {"
-      io.puts "    aux = xmalloc(sizeof(#{name}_pair_#{src}) * (size_t) fiber_n);"
+      io.puts "    aux = ALLOCV_N(#{name}_pair_#{src}, haux, fiber_n);"
       io.puts "  }"
+    elsif src != :object
+      io.puts "  (void) haux;"
     end
 
+    io.puts "  ca_iter_state st_in, st_out;"
+    io.puts "  int rc;"
+    io.puts "  rc = ca_iter_state_init_l2(&st_in,  ca, CA_SLAB_AXES,"
+    io.puts "                             slab_axes, 1, 0);"
+    io.puts "  if ( rc != CA_ITER_OK ) {"
+    io.puts %Q[    rb_raise(rb_eRuntimeError, "%s: input init failed rc=%d", ca_calling_method_name(), rc);]
+    io.puts "  }"
+    io.puts "  rc = ca_iter_state_init_l2_paired(&st_out, &st_in, co, CA_SLAB_AXES,"
+    io.puts "                             slab_axes, 1, CA_KERNEL_WRITE);"
+    io.puts "  if ( rc != CA_ITER_OK ) {"
+    io.puts "    ca_iter_state_finish(&st_in);"
+    io.puts %Q[    rb_raise(rb_eRuntimeError, "%s: output init failed rc=%d", ca_calling_method_name(), rc);]
+    io.puts "  }"
+
     if view_flat
       io.puts
       io.puts "  /* :view_flat semantics: kernel computes view-flat address"
@@ -4370,7 +4373,7 @@ module MkKernel
         if is_fixlen
           io.puts "      buf[k].vp = pi + k * istride; buf[k].nb = (ca_size_t) ca->bytes;"
         else
-          io.puts "      buf[k].v = *(#{si[:c]} *) (pi + k * istride);"
+          io.puts "      memcpy(&buf[k].v, pi + k * istride, sizeof(buf[k].v));"
         end
         io.puts "      buf[k].i = #{payload.call("k")};"
         io.puts "    }"
@@ -4385,7 +4388,7 @@ module MkKernel
       if is_fixlen
         io.puts "      buf[k_unmasked].vp = pi + k * istride; buf[k_unmasked].nb = (ca_size_t) ca->bytes;"
       else
-        io.puts "      buf[k_unmasked].v = *(#{si[:c]} *) (pi + k * istride);"
+        io.puts "      memcpy(&buf[k_unmasked].v, pi + k * istride, sizeof(buf[k_unmasked].v));"
       end
       io.puts "      buf[k_unmasked].i = k;"
       io.puts "      k_unmasked++;"
@@ -4401,7 +4404,7 @@ module MkKernel
       if is_fixlen
         io.puts "      buf[k].vp = pi + k * istride; buf[k].nb = (ca_size_t) ca->bytes;"
       else
-        io.puts "      buf[k].v = *(#{si[:c]} *) (pi + k * istride);"
+        io.puts "      memcpy(&buf[k].v, pi + k * istride, sizeof(buf[k].v));"
       end
       io.puts "      buf[k].i = k;"
       io.puts "    }"
@@ -4474,7 +4477,7 @@ module MkKernel
         io.puts "    /* P.3/P.4: NaN pre-partition for float argsort -- NaN pairs are"
         io.puts "       pushed to the tail (= NaN-at-end policy, matches sort_copy"
         io.puts "       AC8).  Only the finite slice is sorted. */"
-        io.puts "    ca_size_t finite_n = ca_partition_nan_pair_#{src}((ca_pair_#{src} *) #{buf_expr}, (ca_size_t) #{sort_n_expr});"
+        io.puts "    ca_size_t finite_n = ca_partition_nan_pair_#{src}((ca_pair_#{src} *) #{buf_expr}, (ca_size_t) #{sort_n_expr}, (ca_pair_#{src} *) aux);"
         io.puts "    if ( do_stable ) {"
         io.puts "      ca_sort_merge_pair_#{src}((ca_pair_#{src} *) #{buf_expr}, (ca_pair_#{src} *) aux, finite_n);"
         io.puts "    } else {"
@@ -4502,16 +4505,21 @@ module MkKernel
       io.puts "       gaps ordinal would have spent on the tie are never used, so a"
       io.puts "       later out_r stays <= the ordinal r at that position, preserving"
       io.puts "       both the ascending order and (via the order() method's"
-      io.puts "       (n-1)-out_r descending transform) the descending order.  NaN"
-      io.puts "       cells never tie with each other under dense (IEEE754 x != x). */"
+      io.puts "       descending transform) the descending order.  NaN cells tie"
+      io.puts "       with each other, as sort_addr and the value-hash family treat"
+      io.puts "       every NaN as one value. */"
       io.puts "    ca_size_t out_r = 0;"
       io.puts "    for ( ca_size_t r = 0; r < #{rank_n}; r++ ) {"
       io.puts "      if ( r > 0 ) {"
       io.puts "        if ( dense ) {"
       if is_object
-        io.puts "          if ( ! ca_obj_equal(buf[r].v, buf[r - 1].v) ) out_r++;"
+        io.puts "          if ( ! ( ( CA_OBJ_ISNAN(buf[r].v) && CA_OBJ_ISNAN(buf[r - 1].v) )"
+        io.puts "                   || ca_obj_equal(buf[r].v, buf[r - 1].v) ) ) out_r++;"
       elsif is_fixlen
         io.puts "          if ( memcmp(buf[r].vp, buf[r - 1].vp, (size_t) buf[r].nb) != 0 ) out_r++;"
+      elsif FLOAT_DTYPES.include?(src)
+        io.puts "          if ( buf[r].v != buf[r - 1].v"
+        io.puts "               && ! ( isnan(buf[r].v) && isnan(buf[r - 1].v) ) ) out_r++;"
       else
         io.puts "          if ( buf[r].v != buf[r - 1].v ) out_r++;"
       end
@@ -4521,15 +4529,32 @@ module MkKernel
       io.puts "      }"
       io.puts "      *(#{oi[:c]} *) (po + buf[r].i * ostride) = (#{oi[:c]}) out_r;"
       io.puts "    }"
+      # dense == 2 (from CArray#order only): descending dense ranks, the
+      # largest value 0, the NaN cells (sorted to the end) after the numbers.
+      nan_test = if is_object then "CA_OBJ_ISNAN(buf[nan_start - 1].v)"
+                 elsif FLOAT_DTYPES.include?(src) then "isnan(buf[nan_start - 1].v)"
+                 end
+      io.puts "    if ( dense == 2 && #{rank_n} > 0 ) {"
+      io.puts "      ca_size_t nan_start = #{rank_n};"
+      io.puts "      while ( nan_start > 0 && #{nan_test} ) nan_start--;" if nan_test
+      io.puts "      #{oi[:c]} top = nan_start > 0"
+      io.puts "        ? *(#{oi[:c]} *) (po + buf[nan_start - 1].i * ostride) : (#{oi[:c]}) -1;"
+      io.puts "      for ( ca_size_t r = 0; r < #{rank_n}; r++ ) {"
+      io.puts "        #{oi[:c]} *o = (#{oi[:c]} *) (po + buf[r].i * ostride);"
+      io.puts "        *o = ( r < nan_start ) ? top - *o : top + 1;"
+      io.puts "      }"
+      io.puts "    }"
       if mask_skip
         io.puts "    /* Mirror input mask to output (= UNDEF rank for masked"
-        io.puts "       cells).  Write via co->mask->ptr directly with the"
-        io.puts "       output slab offset, because the iter's mo isn't"
-        io.puts "       populated in CA_KERNEL_WRITE mode (= reduce form's"
-        io.puts "       op_mask pattern). */"
+        io.puts "       cells).  Write via co->mask->ptr directly, because a"
+        io.puts "       mask written through a WRITE walk's cursor does not"
+        io.puts "       reach the array.  co is a fresh row-major entity and"
+        io.puts "       the walk cuts its slabs from a buffer laid out the"
+        io.puts "       same way, so the slab's offset from st_out.alias_ptr"
+        io.puts "       is its offset in co. */"
         io.puts "    if ( mi && co->mask ) {"
         io.puts "      boolean8_t *omask_base = (boolean8_t *) co->mask->ptr"
-        io.puts "                              + ((po - (char *) co->ptr) / (ca_size_t) co->bytes);"
+        io.puts "                              + ((po - st_out.alias_ptr) / (ca_size_t) co->bytes);"
         io.puts "      ca_size_t mostride = st_out.slab_mask_strides[0];"
         io.puts "      for ( ca_size_t k = 0; k < fiber_n; k++ ) {"
         io.puts "        if ( mi[k * mstride] ) omask_base[k * mostride] = 1;"
@@ -4555,45 +4580,33 @@ module MkKernel
     end
 
     io.puts "  }"
-    if src == :object
-      io.puts "  ALLOCV_END(hbuf);"
-      io.puts "  ALLOCV_END(haux);"
-    else
-      io.puts "  xfree(buf);"
-      if has_kind
-        io.puts "  if ( aux ) xfree(aux);"
-      end
-    end
+    io.puts "  ALLOCV_END(hbuf);"
+    io.puts "  ALLOCV_END(haux);"
     io.puts "  ca_iter_state_finish(&st_in);"
     io.puts "  ca_iter_state_finish(&st_out);"
     io.puts "  return vout;"
     io.puts "}"
   end
 
-  # Shared sort-family Face gate.  Descends a Face to its storage: a fixlen
-  # storage sorts by memcmp (the default order for fixlen, the same as a plain
-  # fixlen array -- a struct/record Face gets a deterministic byte order, and
-  # opts into a semantic order separately); a numeric storage requires
-  # ORDERABLE so the numeric order equals the surface order, else it raises.
-  # `src`/`self` are rebound to the stripped storage.
+  # Shared sort-family Face gate.  Every kernel of the family answers
+  # positions, so it runs on the array the Face's order is read from
+  # (ca_order_key): the storage of an ORDERABLE Face, the Face's own dense
+  # rank for a fixlen storage that is not ORDERABLE (CAConstString's offsets,
+  # a record's bytes), and a refusal otherwise.  `src`/`self` are rebound to
+  # that array.
   def self.emit_sort_face_gate(io, name)
     io.puts "  if ( ca_is_face(src) ) {"
-    io.puts "    VALUE _face = self;"
-    io.puts "    int _orderable = ca_test_flag(src, CA_FLAG_FACE_ORDERABLE_STORAGE);"
-    io.puts "    VALUE _st = rb_ca_strip_face_value(self);"
-    io.puts "    CArray *_sc;"
-    io.puts "    GetCArray(_st, _sc);"
-    io.puts "    if ( _sc->data_type != CA_FIXLEN && ! _orderable ) {"
-    io.puts %Q[      ca_face_refuse_order(ca_calling_method_name(), _face, "order");]
-    io.puts "    }"
-    io.puts "    self = _st;"
-    io.puts "    src = _sc;"
+    io.puts "    self = ca_order_key(self, ca_calling_method_name());"
+    io.puts "    GetCArray(self, src);"
     io.puts "  }"
   end
 
-  # Reduce-family Face gate.  Same ORDERABLE-descent policy as
-  # emit_sort_face_gate (fixlen storage -> memcmp default; numeric storage
-  # -> ORDERABLE required, else raise), rebinding self/src to storage.  When
+  # Reduce-family Face gate.  Descends an ORDERABLE Face to its storage and
+  # refuses any other (ca_face_require_orderable): a fixlen storage is
+  # reduced by memcmp only when the Face declares that its bytes are its
+  # order (CAFixlenString).  CAConstString and CARecord, whose bytes are no
+  # order, answer min / max / minmax in Ruby and do not reach this gate.
+  # self/src are rebound to storage.  When
   # `relift` is set (value-returning kernels), the original Face is saved to
   # the function-scope `_reduce_face` so the dispatcher can re-lift the output.
   def self.emit_reduce_face_gate(io, name, relift)
@@ -4602,13 +4615,11 @@ module MkKernel
     end
     io.puts "  if ( ca_is_face(src) ) {"
     io.puts "    VALUE _face = self;"
-    io.puts "    int _orderable = ca_test_flag(src, CA_FLAG_FACE_ORDERABLE_STORAGE);"
-    io.puts "    VALUE _st = rb_ca_strip_face_value(self);"
+    io.puts "    VALUE _st;"
     io.puts "    CArray *_sc;"
+    io.puts %Q[    ca_face_require_orderable(_face, ca_calling_method_name(), "order");]
+    io.puts "    _st = rb_ca_strip_face_value(self);"
     io.puts "    GetCArray(_st, _sc);"
-    io.puts "    if ( _sc->data_type != CA_FIXLEN && ! _orderable ) {"
-    io.puts %Q[      ca_face_refuse_order(ca_calling_method_name(), _face, "order");]
-    io.puts "    }"
     if relift
       io.puts "    _reduce_face = _face;"
     end
@@ -4713,10 +4724,10 @@ module MkKernel
       io.puts "  CArray *src;"
       io.puts "  GetCArray(self, src);"
       emit_lazy_evaluated(io)
-      io.puts "  /* Face gate: descend to storage (fixlen -> memcmp default,"
-      io.puts "     numeric -> ORDERABLE required).  The strip also makes the"
-      io.puts "     switch below and every index/addr template output build on"
-      io.puts "     the plain storage, not a face-lifted (NULL-ptr) view. */"
+      io.puts "  /* Face gate: run on the array the Face's order is read from."
+      io.puts "     It is never a Face, so the switch below and every index/addr"
+      io.puts "     template output build on a plain array, not a face-lifted"
+      io.puts "     (NULL-ptr) view. */"
       emit_sort_face_gate(io, name)
       if k[:mask_self] == :raise
         io.puts "  /* sort family rejects masked sources globally (R3 / Q3): "
@@ -4725,11 +4736,7 @@ module MkKernel
         io.puts %Q[    rb_raise(rb_eArgError, "%s: masked input not supported (use ca.value or ca.strip_mask(fill))", ca_calling_method_name());]
         io.puts "  }"
       end
-      io.puts "  int axis = (int) ca_axis_integer(vaxis, NULL);"
-      io.puts "  if ( axis < 0 ) axis += src->ndim;"
-      io.puts "  if ( axis < 0 || axis >= src->ndim ) {"
-      io.puts %Q[    rb_raise(rb_eArgError, "%s: axis %d out of range for ndim %d", ca_calling_method_name(), NUM2INT(vaxis), src->ndim);]
-      io.puts "  }"
+      io.puts "  int axis = ca_normalize_axis_arg(vaxis, src->ndim, NULL);"
       io.puts "  switch ( src->data_type ) {"
       k[:source].each do |s|
         si = DTYPES[s]
@@ -4825,8 +4832,7 @@ module MkKernel
       io.puts "  CArray *src;"
       io.puts "  GetCArray(self, src);"
       emit_lazy_evaluated(io)
-      io.puts "  /* Face gate (partition): same as the sort dispatcher --"
-      io.puts "     fixlen -> memcmp default, numeric -> ORDERABLE required. */"
+      io.puts "  /* Face gate (partition): same as the sort dispatcher. */"
       emit_sort_face_gate(io, name)
       if k[:mask_self] == :raise
         io.puts "  /* sort family rejects masked sources globally (R3 / Q3): "
@@ -4835,11 +4841,7 @@ module MkKernel
         io.puts %Q[    rb_raise(rb_eArgError, "%s: masked input not supported (use ca.value or ca.strip_mask(fill))", ca_calling_method_name());]
         io.puts "  }"
       end
-      io.puts "  int axis = (int) ca_axis_integer(vaxis, NULL);"
-      io.puts "  if ( axis < 0 ) axis += src->ndim;"
-      io.puts "  if ( axis < 0 || axis >= src->ndim ) {"
-      io.puts %Q[    rb_raise(rb_eArgError, "%s: axis %d out of range for ndim %d", ca_calling_method_name(), NUM2INT(vaxis), src->ndim);]
-      io.puts "  }"
+      io.puts "  int axis = ca_normalize_axis_arg(vaxis, src->ndim, NULL);"
       io.puts "  ca_size_t kth_raw = (ca_size_t) ca_integer_arg(vkth, \"kth\", NULL);"
       io.puts "  ca_size_t kth = kth_raw;"
       io.puts "  ca_size_t axis_n = src->dim[axis];"
@@ -5017,7 +5019,7 @@ module MkKernel
     io.puts
     io.puts "/* ===== #{k[:name]}_ki ============================================ */"
     k[:source].each do |src|
-      emit_native_for(io, k, src) { |o| emit_search_native(o, k, src) }
+      emit_native_for(io, k, src, guard: true) { |o| emit_search_native(o, k, src) }
     end
     emit_search_dispatch(io, k)
   end
@@ -5175,56 +5177,41 @@ module MkKernel
     # flat array path.  op_mask is lazily allocated on the first no-match.
     undef_nm = k[:undef_no_match]
     nm_cond  = "result == (#{oi[:c]})(#{no_match})"
-    # mask_query: :undef -- a masked query cell yields a masked output cell,
-    # so an undetermined query cannot be answered from the value that happens
-    # to sit under the mask.  Shares op_mask with undef_no_match.
-    undef_q  = (k[:mask_query] == :undef)
-    # op_mask declaration (only when an output cell can be masked);
-    # content-only (the heredoc carries indentation).
-    op_mask_decl = (undef_nm || undef_q) \
-      ? "boolean8_t *op_mask = NULL;   /* lazily allocated on the first masked output cell */" \
-      : "/* (no masked output cell) */"
+    # A masked query cell yields a masked output cell, so an undetermined
+    # query cannot be answered from the value that happens to sit under the
+    # mask.  Shares op_mask with undef_no_match.
+    op_mask_decl = "boolean8_t *op_mask = NULL;   /* lazily allocated on the first masked output cell */"
 
     # Case A scalar UNDEF query (also reached by the A1 single-element
     # reduction when that element is masked): every output cell is
     # undetermined, so short-circuit before the query is coerced to a number.
-    undef_query_a = if undef_q
-                      <<~C.chomp
-                        if ( rval == CA_UNDEF ) {
-                              int8_t  undef_axes[1] = { (int8_t) axis };
-                              VALUE   voutu = rb_ca_new_reduced(self, undef_axes, 1, #{oi[:ca]}, 0);
-                              CArray *cou;
-                              GetCArray(voutu, cou);
-                              MEMZERO((#{oi[:c]} *) cou->ptr, #{oi[:c]}, cou->elements);
-                              ca_create_mask(cou);
-                              memset(cou->mask->ptr, 1, (size_t) cou->elements);
-                              if ( ca->ndim == 1 ) { return Qnil; }
-                              return voutu;
-                            }
-                      C
-                    else
-                      "/* (mask_query: :ignore) */"
-                    end
+    undef_query_a = <<~C.chomp
+      if ( rval == CA_UNDEF ) {
+            int8_t  undef_axes[1] = { (int8_t) axis };
+            VALUE   voutu = rb_ca_new_reduced(self, undef_axes, 1, #{oi[:ca]}, 0);
+            CArray *cou;
+            GetCArray(voutu, cou);
+            MEMZERO((#{oi[:c]} *) cou->ptr, #{oi[:c]}, cou->elements);
+            ca_create_mask(cou);
+            memset(cou->mask->ptr, 1, (size_t) cou->elements);
+            if ( ca->ndim == 1 ) { return Qnil; }
+            return voutu;
+          }
+    C
 
     # Case B/C query mask base.  NULL when the query carries no mask.
-    cv_mask_decl_bc = undef_q \
-      ? "boolean8_t *cv_mask_base = ca_has_mask(cv) ? (boolean8_t *) cv->mask->ptr : NULL;" \
-      : "/* (mask_query: :ignore) */"
+    cv_mask_decl_bc = "boolean8_t *cv_mask_base = ca_has_mask(cv) ? (boolean8_t *) cv->mask->ptr : NULL;"
 
     # Per-cell guard wrapping the body + write in case B/C.  A masked query
     # cell skips the body entirely and marks the output cell instead.
-    undef_query_open_bc = if undef_q
-                            <<~C.chomp
-                              if ( cv_mask_base && cv_mask_base[val_offset / cv->bytes] ) {
-                                    if ( ! op_mask ) { ca_create_mask(co); op_mask = (boolean8_t *) co->mask->ptr; }
-                                    op_mask[flat] = 1;
-                                    op[flat] = (#{oi[:c]}) 0;   /* undetermined query; the mask bit is what counts */
-                                  } else {
-                            C
-                          else
-                            "/* (mask_query: :ignore) */"
-                          end
-    undef_query_close_bc = undef_q ? "}" : "/* (mask_query: :ignore) */"
+    undef_query_open_bc = <<~C.chomp
+      if ( cv_mask_base && cv_mask_base[val_offset / cv->bytes] ) {
+            if ( ! op_mask ) { ca_create_mask(co); op_mask = (boolean8_t *) co->mask->ptr; }
+            op_mask[flat] = 1;
+            op[flat] = (#{oi[:c]}) 0;   /* undetermined query; the mask bit is what counts */
+          } else {
+    C
+    undef_query_close_bc = "}"
 
     # Per-cell write block for case A (scalar query).  `out_i` is the output
     # cursor, `co` the output CArray.  When undef_no_match: a no-match masks
@@ -5292,7 +5279,7 @@ module MkKernel
           CArray *co;
           GetCArray(vout, co);
           #{oi[:c]} *op = (#{oi[:c]} *) co->ptr;
-          #{op_mask_decl}
+          #{undef_nm ? op_mask_decl : ""}
 
           ca_iter_state st;
           int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
@@ -5514,6 +5501,12 @@ module MkKernel
         #{op_mask_decl}
 
         #{entity_bc}
+        /* The output mask is made before the window opens, since making it
+           allocates; it comes off again below when no cell needed it. */
+        if ( #{undef_nm ? "1" : "ca_has_mask(cv)"} ) {
+          ca_create_mask(co);
+          op_mask = (boolean8_t *) co->mask->ptr;
+        }
         /* Attach self + val; rev4 uses raw row-major byte addressing. */
         ca_attach_n(2, ca, cv);   /* window: numeric bodies do not raise; the object lane reads entities */
 
@@ -5620,6 +5613,11 @@ module MkKernel
 
         ca_detach(ca);
         ca_detach(cv);
+        if ( op_mask && ! memchr(op_mask, 1, (size_t) co->elements) ) {
+          CArray *unused = co->mask;
+          co->mask = NULL;
+          ca_free(unused);
+        }
         return vout;
       }
     C
@@ -5642,21 +5640,15 @@ module MkKernel
     end
     io.puts "{"
     io.puts "  CArray *src;"
-    io.puts "  VALUE self_ref = self;         /* pre-strip Face, for to_comparable */"
     io.puts "  int self_was_face = 0;"
     io.puts "  GetCArray(self, src);"
     emit_lazy_evaluated(io)
-    io.puts "  /* Face gate (self): descend a Face reference to storage so the"
-    io.puts "     search runs on the numeric storage.  ORDERABLE licenses the"
-    io.puts "     ordered/searchable descent; the query gate below brings the"
-    io.puts "     query into the same space. */"
+    io.puts "  /* Face gate (self): a Face reference is searched on its numeric"
+    io.puts "     storage, which ORDERABLE licenses; the query gate below brings"
+    io.puts "     the reference and the query into one space and descends. */"
     io.puts "  if ( ca_is_face(src) ) {"
-    io.puts "    if ( ! ca_test_flag(src, CA_FLAG_FACE_ORDERABLE_STORAGE) ) {"
-    io.puts %Q[      ca_face_refuse_order(ca_calling_method_name(), self, "order");]
-    io.puts "    }"
+    io.puts %Q[    ca_face_require_orderable(self, ca_calling_method_name(), "order");]
     io.puts "    self_was_face = 1;"
-    io.puts "    self = rb_ca_strip_face_value(self);"
-    io.puts "    GetCArray(self, src);"
     io.puts "  }"
 
     if has_eps
@@ -5667,38 +5659,38 @@ module MkKernel
       io.puts "  VALUE rval  = argv[0];"
       io.puts "  VALUE raxis = argv[1];"
       io.puts "  VALUE reps  = (argc >= 3) ? argv[2] : Qnil;"
+      io.puts "  /* A tolerance is a distance: a negative one (or NaN) means nothing. */"
+      io.puts "  if ( ! NIL_P(reps) ) {"
+      io.puts "    double eps_given = NUM2DBL(reps);"
+      io.puts "    if ( ! ( eps_given >= 0.0 ) ) {"
+      io.puts %Q[      rb_raise(rb_eArgError, "%s: eps must be zero or positive (got %"PRIsVALUE")", ca_calling_method_name(), rb_inspect(reps));]
+      io.puts "    }"
+      io.puts "  }"
     end
-    io.puts "  /* Query gate: the reference Face reconciles the query into its own"
-    io.puts "     space (ca_face_reconcile_operand). */"
+    io.puts "  /* Query gate: the reference Face and the query are brought into one"
+    io.puts "     space (ca_face_reconcile_pair), then the reference descends to"
+    io.puts "     its storage. */"
     io.puts "  if ( self_was_face ) {"
-    io.puts "    rval = ca_face_reconcile_operand(self_ref, rval, ca_calling_method_name());"
+    io.puts "    volatile VALUE ref = self, query = rval;"
+    io.puts "    ca_face_reconcile_pair(&ref, &query, ca_calling_method_name());"
+    io.puts "    rval = query;"
+    io.puts "    self = rb_ca_strip_face_value(ref);"
+    io.puts "    GetCArray(self, src);"
     io.puts "  }"
 
-    # The query and the reference are compared in the type they share, as
-    # `a.eq(q)` compares, not with the query cast to the reference's type:
-    # 3.9 must not match 3, a float64 query must not be rounded to a float32
-    # reference, and -1 must not read as 255.  A scalar takes the type the
-    # operators give it (ca_promote_scalar_type); a CArray query the type
-    # the two match in (ca_value_match_type).  The reference is read through
-    # a cast view of that type.  An Integer scalar keeps the reference's type
-    # and is refused when it does not fit.
-    io.puts "  if ( ca_is_integer_type(src) || ca_is_float_type(src) || src->data_type == CA_BOOLEAN ) {"
-    io.puts "    int8_t qt = src->data_type;"
-    io.puts "    if ( RB_FLOAT_TYPE_P(rval) ) {"
-    io.puts "      qt = ca_promote_scalar_type(src->data_type, rval, CA_FLOAT64);"
-    io.puts "    } else if ( rb_obj_is_kind_of(rval, rb_cCArray) ) {"
-    io.puts "      CArray *qv;"
-    io.puts "      TypedData_Get_Struct(rval, CArray, &carray_data_type, qv);"
-    io.puts "      if ( ca_is_integer_type(qv) || ca_is_float_type(qv) || qv->data_type == CA_BOOLEAN ) {"
-    io.puts "        qt = ca_value_match_type(ca_promote_type(src->data_type, qv->data_type),"
-    io.puts "                                 src->data_type, qv->data_type);"
-    io.puts "      }"
-    io.puts "    }"
+    # The query and the reference are compared in the type they match in,
+    # as `a.eq(q)` compares (ca_query_match_type): the reference is read
+    # through a cast view of that type, so the query is never cut to the
+    # reference's type.
+    io.puts "  if ( ca_query_match_applies(src, rval) ) {"
+    io.puts "    int8_t qt = ca_query_match_type(src->data_type, rval);"
     io.puts "    if ( qt != src->data_type ) {"
     io.puts "      self = rb_ca_wrap_readonly(self, INT2NUM(qt));"
     io.puts "      GetCArray(self, src);"
     io.puts "    }"
     io.puts "  }"
+    io.puts "  self = ca_fixlen_widen_for_query(self, rval);"
+    io.puts "  GetCArray(self, src);"
 
     case k[:mask_self]
     when :raise
@@ -5710,11 +5702,7 @@ module MkKernel
       # body handles mask_in or ignores it
     end
 
-    io.puts "  int axis = (int) ca_axis_integer(raxis, NULL);"
-    io.puts "  if ( axis < 0 ) axis += src->ndim;"
-    io.puts "  if ( axis < 0 || axis >= src->ndim ) {"
-    io.puts %Q[    rb_raise(rb_eArgError, "%s: axis %d out of range for ndim %d", ca_calling_method_name(), NUM2INT(raxis), src->ndim);]
-    io.puts "  }"
+    io.puts "  int axis = ca_normalize_axis_arg(raxis, src->ndim, NULL);"
     io.puts "  switch ( src->data_type ) {"
     k[:source].each do |s|
       si = DTYPES[s]
@@ -5734,13 +5722,14 @@ module MkKernel
   # (`ca_monop_<name>_<suffix>`).  mkmath uses typedef names
   # `float32_t` / `float64_t` here, not the raw C types `float` / `double`,
   # so that `ca_monop_dispatch.c`'s extern references stay ABI-stable.
-  # For all other data_types, the suffix matches `DTYPES[src][:c]`.
+  # For all other data_types, the suffix is `DTYPES[src][:c]` without the
+  # `ca_u_` of its unaligned cell typedef (ca_u_int32_t -> int32_t).
   def self.math_suffix(src)
     case src
     when :f32 then "float32_t"
     when :f64 then "float64_t"
     when :fixlen then "fixlen"   # symbol suffix differs from c-type "char"
-    else DTYPES[src][:c]
+    else DTYPES[src][:c].sub(/\Aca_u_/, "")
     end
   end
 
@@ -5819,7 +5808,8 @@ module MkKernel
     uses_input = raw.include?("#1")
     # SL.2.x phase 2 (monop emit substrate hardening, MEMO_MONOP_STRIDED_AUDIT.md):
     # 3-way split (mask / contig / strided) symmetric with binop SL.2.x phase 1.
-    # contig branch (i1==1 && i2==1) uses index form + `__restrict__` + `omp simd`.
+    # contig branch (i1==1 && i2==1) uses index form + `omp simd` (`__restrict__`
+    # only when input and output are apart).
     # strided branch uses `p += i` increment form (per-iter multiply removed).
     # mask branch unchanged (per-cell predicate, not SIMD-amenable).
     if uses_input
@@ -5827,11 +5817,11 @@ module MkKernel
 
         static void
         ca_monop_#{k[:name]}_#{suffix} (ca_size_t n, boolean8_t *m,
-                                         char * __restrict__ ptr1, ca_size_t i1,
-                                         char * __restrict__ ptr2, ca_size_t i2)
+                                         char *ptr1, ca_size_t i1,
+                                         char *ptr2, ca_size_t i2)
         {
-          #{c_type_in}  * __restrict__ q1 = (#{c_type_in}  *) ptr1;
-          #{c_type_out} * __restrict__ q2 = (#{c_type_out} *) ptr2;
+          #{c_type_in}  *q1 = (#{c_type_in}  *) ptr1;
+          #{c_type_out} *q2 = (#{c_type_out} *) ptr2;
           ca_size_t k;
           if ( m ) {
             #{c_type_in}  *p1 = q1;
@@ -5847,17 +5837,38 @@ module MkKernel
             }
           } else if ( i1 == 1 && i2 == 1 ) {
             /* contig branch: SIMD-friendly form.  Index-form `q[k]` exposes
-               the unit-stride access to the vectorizer; `__restrict__` on
-               q1/q2 promises no aliasing so the omp simd pragma can license
-               lane-parallel loads/stores.  Element-wise monop has no
-               reassoc, so ε-close numeric contract is unaffected. */
+               the unit-stride access to the vectorizer.  Element-wise monop
+               has no reassoc, so ε-close numeric contract is unaffected.
+               The input and output are either apart, the same cells (a
+               bang, or the lazy scratch applied in place), or overlapping
+               otherwise; `__restrict__` is promised only when apart, since
+               the body may store part of a cell before it has read it. */
             #{c_type_in}  *p1;
             #{c_type_out} *p2;
             (void)p1; (void)p2;
-            #pragma omp simd
-            for (k=0; k<n; k++) {
-              p1 = q1 + k; p2 = q2 + k;
-              { #{expr_body} }
+            uintptr_t a1 = (uintptr_t) q1, a2 = (uintptr_t) q2;
+            if ( a2 + n * sizeof(*q2) <= a1 || a1 + n * sizeof(*q1) <= a2 ) {
+              #{c_type_in}  * __restrict__ r1 = q1;
+              #{c_type_out} * __restrict__ r2 = q2;
+              #pragma omp simd
+              for (k=0; k<n; k++) {
+                p1 = r1 + k; p2 = r2 + k;
+                { #{expr_body} }
+              }
+            }
+            else if ( a1 == a2 && sizeof(*q1) == sizeof(*q2) ) {
+              /* in place: an iteration reads and writes only its own cell */
+              #pragma omp simd
+              for (k=0; k<n; k++) {
+                p1 = q1 + k; p2 = q2 + k;
+                { #{expr_body} }
+              }
+            }
+            else {
+              for (k=0; k<n; k++) {
+                p1 = q1 + k; p2 = q2 + k;
+                { #{expr_body} }
+              }
             }
           } else {
             /* strided branch: numo-shape `p += s` increment form. */
@@ -6043,7 +6054,8 @@ module MkKernel
     expr_body = raw.gsub(/#1/, "*p1").gsub(/#2/, "*p2").gsub(/#3/, "*p3").gsub(/<type>/, suffix)
     # Option B+C pilot (binop emit substrate hardening, MEMO_BINOP_STRIDED_AUDIT.md):
     # The no-mask body is split into a contig branch (i1==i2==i3==1) using the
-    # `q[k]` index form with `__restrict__`-qualified pointers + `#pragma omp simd`
+    # `q[k]` index form with `#pragma omp simd` (`__restrict__`-qualified only
+    # when the output is apart from both inputs)
     # (SL.2.x-style insurance, element-wise has no reassoc -> ε-close unaffected),
     # and a strided branch using `p += i` increment form (numo-shape inner loop).
     # `i1`/`i2`/`i3` remain element units (NOT byte) -- byte-flip is Option A scope.
@@ -6053,13 +6065,13 @@ module MkKernel
 
       static void
       ca_binop_#{k[:name]}_#{suffix} (ca_size_t n, boolean8_t *m,
-                                       char * __restrict__ ptr1, ca_size_t i1,
-                                       char * __restrict__ ptr2, ca_size_t i2,
-                                       char * __restrict__ ptr3, ca_size_t i3)
+                                       char *ptr1, ca_size_t i1,
+                                       char *ptr2, ca_size_t i2,
+                                       char *ptr3, ca_size_t i3)
       {
-        #{c_type} * __restrict__ q1 = (#{c_type} *) ptr1;
-        #{c_type} * __restrict__ q2 = (#{c_type} *) ptr2;
-        #{c_type} * __restrict__ q3 = (#{c_type} *) ptr3;
+        #{c_type} *q1 = (#{c_type} *) ptr1;
+        #{c_type} *q2 = (#{c_type} *) ptr2;
+        #{c_type} *q3 = (#{c_type} *) ptr3;
         ca_size_t k;
         if ( m ) {
           #{c_type} *p1 = q1, *p2 = q2, *p3 = q3;
@@ -6075,16 +6087,41 @@ module MkKernel
           }
         } else if ( i1 == 1 && i2 == 1 && i3 == 1 ) {
           /* contig branch: SIMD-friendly form.  Index-form `q[k]` exposes
-             the unit-stride access to the vectorizer; `__restrict__` on
-             q1/q2/q3 promises no aliasing so the omp simd pragma can
-             license lane-parallel loads/stores.  Element-wise binop has
-             no reassoc, so ε-close numeric contract is unaffected. */
+             the unit-stride access to the vectorizer.  Element-wise binop
+             has no reassoc, so ε-close numeric contract is unaffected.
+             The output is apart from each input, the same cells as it (a
+             bang), or overlaps it otherwise; `__restrict__` is promised
+             only when apart from both, since the body may store part of a
+             cell before it has read it. */
           #{c_type} *p1, *p2, *p3;
+          uintptr_t a1 = (uintptr_t) q1, a2 = (uintptr_t) q2, a3 = (uintptr_t) q3;
+          uintptr_t nb = (uintptr_t) n * sizeof(*q3);
+          int apart1 = ( a3 + nb <= a1 || a1 + nb <= a3 );
+          int apart2 = ( a3 + nb <= a2 || a2 + nb <= a3 );
           (void)p1; (void)p2; (void)p3;
-          #pragma omp simd
-          for (k=0; k<n; k++) {
-            p1 = q1 + k; p2 = q2 + k; p3 = q3 + k;
-            { #{expr_body} }
+          if ( apart1 && apart2 ) {
+            #{c_type} * __restrict__ r1 = q1;
+            #{c_type} * __restrict__ r2 = q2;
+            #{c_type} * __restrict__ r3 = q3;
+            #pragma omp simd
+            for (k=0; k<n; k++) {
+              p1 = r1 + k; p2 = r2 + k; p3 = r3 + k;
+              { #{expr_body} }
+            }
+          }
+          else if ( ( apart1 || a1 == a3 ) && ( apart2 || a2 == a3 ) ) {
+            /* in place: an iteration reads and writes only its own cell */
+            #pragma omp simd
+            for (k=0; k<n; k++) {
+              p1 = q1 + k; p2 = q2 + k; p3 = q3 + k;
+              { #{expr_body} }
+            }
+          }
+          else {
+            for (k=0; k<n; k++) {
+              p1 = q1 + k; p2 = q2 + k; p3 = q3 + k;
+              { #{expr_body} }
+            }
           }
         } else {
           /* strided branch: numo-shape `p += s` increment form. */
@@ -6539,19 +6576,29 @@ module MkKernel
   # the source's data type.  Left out: a boolean target, which checks its
   # input and raises, and a float or complex source to an integer target,
   # where an out-of-range value or NaN has no defined C result for a
-  # second compilation to agree on.
+  # second compilation to agree on.  An int64 or uint64 source to float32
+  # or cmplx64 goes through double, as ca_cast_block does.
   CAST_SOURCES = %i[bool i8 u8 i16 u16 i32 u32 i64 u64 f32 f64
                     cmplx64 cmplx128].freeze
   CAST_TARGETS = %i[i8 u8 i16 u16 i32 u32 i64 u64 f32 f64
                     cmplx64 cmplx128].freeze
   CAST_INEXACT = %i[f32 f64 cmplx64 cmplx128].freeze
 
+  # The plain C type of a data type, for a body that leaves this file
+  # (CArray.__kernel_body__, a fusion plan): the unaligned typedef in :c is
+  # about how a kernel reaches a cell, not about the value.
+  def self.value_c(src)
+    { "ca_u_float32_t" => "float", "ca_u_float64_t" => "double" }
+      .fetch(DTYPES[src][:c]) { |c| c.sub(/\Aca_u_/, "") }
+  end
+
   def self.cast_body_rows
     CAST_SOURCES.product(CAST_TARGETS).filter_map { |src, dst|
       next if CAST_INEXACT.include?(src) && !CAST_INEXACT.include?(dst)
       name = DTYPES[dst][:ca].sub(/\ACA_/, "").downcase
+      via  = ( %i[i64 u64].include?(src) && %i[f32 cmplx64].include?(dst) ) ? "(double)" : ""
       ["monop", "cast_#{name}", DTYPES[src][:ca].sub(/\ACA_/, "").downcase,
-       "(#2) = (#{DTYPES[dst][:c]})(#1);"]
+       "(#2) = (#{value_c(dst)})#{via}(#1);"]
     }
   end
 
@@ -6756,12 +6803,16 @@ module MkKernel
       #   reduce: default false (E.8 retire), opt-in via bind_ruby: true
       #           (used by CF.1 count_equal smoke + future kernel testing)
       #   sort:   default true,  opt-out via bind_ruby: false
-      #   scan:   always bound (no opt-out)
+      #   scan:   bound as <name>_ki, or only privately as <name> (private:)
       bind = case k[:kind]
              when :reduce then k[:bind_ruby] == true
              when :sort   then k[:bind_ruby] != false
+             when :scan   then ! k[:private]
              else true
              end
+      if k[:kind] == :scan && k[:private]
+        io.puts %Q[  rb_define_private_method(rb_cCArray, "#{k[:name]}", rb_ca_#{k[:name]}_ki, #{arity});]
+      end
       # Smoke kernels (e.g. count_equal) expose their _ki binding only
       # in dev builds, gated by CARRAY_DEV_BUILD.  See
       # devel/PROPOSAL_SMOKE_DEV_BUILD_GATE.md.
@@ -6774,7 +6825,7 @@ module MkKernel
       # directly without a Ruby wrapper.  This is paired with
       # axis_default: :flatten because both express the legacy-compat
       # convenience of these specific scan kernels.
-      if k[:kind] == :scan && k[:axis_default] == :flatten
+      if k[:kind] == :scan && k[:axis_default] == :flatten && ! k[:private]
         io.puts %Q[  rb_define_method(rb_cCArray, "#{k[:name]}", rb_ca_#{k[:name]}_ki, #{arity});]
       end
       # Phase E rewire: when this kernel replaces a legacy CArray method,
@@ -7398,7 +7449,7 @@ MkKernel.reduce :wsum,
                      shape: :rev5_strict, mask: :overlay },
   mask_policy:     :min_count,
   identity_on_empty: true,       # ERI.2: empty / all-masked -> 0.0 (weighted sum identity)
-  object_escape:   :__wsum_object__,   # CA_OBJECT -> (w*self).sum (runtime.rb)
+  object_escape:   :wsum_object,   # CA_OBJECT -> (w*self).sum (runtime.rb)
   bind_ruby:       true,
   public_method: true
 
@@ -7415,7 +7466,7 @@ MkKernel.reduce :wsum,
 #   - min_count semantic "max masked" -> "min valid required" (Phase E)
 #   - complex (CMPLX64/128) dropped (= ALL_NUMERIC + :raise)
 #   - per-axis support gained
-# CA_OBJECT is handled via object_escape (runtime.rb __wmean_object__), not
+# CA_OBJECT is handled via object_escape (runtime.rb wmean_object), not
 # the numeric kernel: weight-coerce vs object arithmetic is cleaner as a
 # (w*self).sum / Σw composition than a dedicated :object kernel branch.
 
@@ -7432,7 +7483,7 @@ MkKernel.reduce :wmean,
   array_arg:       { name: :weights, data_type: :promote,
                      shape: :rev5_strict, mask: :overlay },
   mask_policy:     :min_count,
-  object_escape:   :__wmean_object__,  # CA_OBJECT -> weighted mean via runtime.rb
+  object_escape:   :wmean_object,  # CA_OBJECT -> weighted mean via runtime.rb
   bind_ruby:       true,
   public_method: true
 
@@ -7457,6 +7508,18 @@ MkKernel.scan :cumsum,
   fallback:     :raise,
   axis_default: :flatten
 
+# The running sum in the source's own integer type, added without a range
+# check (int64 wraps, as accumulate does).  CATimedelta#cumsum reads it
+# for an exact running sum of durations.
+MkKernel.scan :cumsum_in_own_type,
+  source:       MkKernel::INT_DTYPES,
+  output:       :preserve,
+  init:         "0",
+  step:         "acc += v; r = acc",
+  fallback:     :raise,
+  axis_default: :flatten,
+  private:      true
+
 MkKernel.scan :cumprod,
   source:       MkKernel::ALL_NUMERIC + MkKernel::CMPLX_DTYPES + [:bool, :object],
   output:       { numeric: :f64, complex: :cmplx128, bool: :u64, object: :object },
@@ -7707,8 +7770,8 @@ MkKernel.search :find_value_index,
   mask_self: :raise,
   fallback:  :raise
 
-# bsearch_ki: per-slice binary search returning the matching position
-# (assumes a sorted slab).  NaN handling: NaN sorts to end; query == NaN
+# bsearch_ki: per-slice binary search returning the first matching position
+# (assumes a sorted slab; of equal cells, the first).  NaN handling: NaN sorts to end; query == NaN
 # never matches.  Hand-rolled strided binary search (= libc bsearch()
 # requires contig memory; slab_stride may be > sizeof(T_LOAD) for a
 # non-innermost search axis).  NaN guards are dead code for integer
@@ -7732,47 +7795,41 @@ MkKernel.search :bsearch,
         while ( lo < hi ) {
           ca_size_t mid = lo + (hi - lo) / 2;
           T_LOAD v = *(T_LOAD *)(slab_ptr + mid * slab_stride);
-          if ( isnan((double) v) ) { hi = mid; continue; }
           if ( v < query_val ) lo = mid + 1;
           else if ( v > query_val ) hi = mid;
-          else { result = mid; break; }
+          else { result = mid; hi = mid; }   /* keep looking left: the first equal cell */
         }
       }
     C
     float: <<~C,
       result = (ca_size_t) -1;
-      if ( ! isnan((double) query_val) ) {
+      if ( ! isnan((double) query_val) ) {   /* a NaN query matches nothing */
         ca_size_t lo = 0;
         ca_size_t hi = slab_n;
         while ( lo < hi ) {
           ca_size_t mid = lo + (hi - lo) / 2;
           T_LOAD v = *(T_LOAD *)(slab_ptr + mid * slab_stride);
-          if ( isnan((double) v) ) { hi = mid; continue; }
-          if ( v < query_val ) lo = mid + 1;
-          else if ( v > query_val ) hi = mid;
-          else { result = mid; break; }
+          int c = CA_FLOAT_ORDER_CMP(v, query_val);
+          if ( c < 0 ) lo = mid + 1;
+          else if ( c > 0 ) hi = mid;
+          else { result = mid; hi = mid; }   /* keep looking left: the first equal cell */
         }
       }
     C
     object: <<~C,
-      /* CA_OBJECT bsearch: ordering via rb_funcall(<=>).  Assumes the
-         slab is sorted by the same comparator used to build it, which puts
-         a Float NaN last; a NaN query matches nothing, as in float. */
+      /* CA_OBJECT bsearch: the object order (ca_obj_order_cmp), which the
+         slab is assumed to be sorted by; a NaN query matches nothing, as in
+         float. */
       result = (ca_size_t) -1;
       ca_size_t lo = 0;
       ca_size_t hi = CA_OBJ_ISNAN(query_val) ? 0 : slab_n;   /* NaN matches nothing */
       while ( lo < hi ) {
         ca_size_t mid = lo + (hi - lo) / 2;
         T_LOAD v = *(T_LOAD *)(slab_ptr + mid * slab_stride);
-        if ( CA_OBJ_ISNAN(v) ) { hi = mid; continue; }   /* NaN sorts last */
-        VALUE r = rb_funcall(v, rb_intern("<=>"), 1, query_val);
-        if ( NIL_P(r) ) {
-          ca_obj_compare_refused(v, query_val);
-        }
-        int c = NUM2INT(r);
+        int c = ca_obj_order_cmp(v, query_val);
         if ( c < 0 ) lo = mid + 1;
         else if ( c > 0 ) hi = mid;
-        else { result = mid; break; }
+        else { result = mid; hi = mid; }   /* keep looking left: the first equal cell */
       }
     C
     fixlen: <<~C,
@@ -7788,7 +7845,7 @@ MkKernel.search :bsearch,
         int c = memcmp(v, query_val, (size_t) ca->bytes);
         if ( c < 0 ) lo = mid + 1;
         else if ( c > 0 ) hi = mid;
-        else { result = mid; break; }
+        else { result = mid; hi = mid; }   /* keep looking left: the first equal cell */
       }
     C
   },
@@ -7971,25 +8028,24 @@ MkKernel.search :bsearch_addr,
         while ( lo < hi ) {
           ca_size_t mid = lo + (hi - lo) / 2;
           T_LOAD v = *(T_LOAD *)(slab_ptr + mid * slab_stride);
-          if ( isnan((double) v) ) { hi = mid; continue; }
           if ( v < query_val ) lo = mid + 1;
           else if ( v > query_val ) hi = mid;
-          else { result = mid; break; }
+          else { result = mid; hi = mid; }   /* keep looking left: the first equal cell */
         }
       }
     C
     float: <<~C,
       result = (ca_size_t) -1;
-      if ( ! isnan((double) query_val) ) {
+      if ( ! isnan((double) query_val) ) {   /* a NaN query matches nothing */
         ca_size_t lo = 0;
         ca_size_t hi = slab_n;
         while ( lo < hi ) {
           ca_size_t mid = lo + (hi - lo) / 2;
           T_LOAD v = *(T_LOAD *)(slab_ptr + mid * slab_stride);
-          if ( isnan((double) v) ) { hi = mid; continue; }
-          if ( v < query_val ) lo = mid + 1;
-          else if ( v > query_val ) hi = mid;
-          else { result = mid; break; }
+          int c = CA_FLOAT_ORDER_CMP(v, query_val);
+          if ( c < 0 ) lo = mid + 1;
+          else if ( c > 0 ) hi = mid;
+          else { result = mid; hi = mid; }   /* keep looking left: the first equal cell */
         }
       }
     C
@@ -8000,15 +8056,10 @@ MkKernel.search :bsearch_addr,
       while ( lo < hi ) {
         ca_size_t mid = lo + (hi - lo) / 2;
         T_LOAD v = *(T_LOAD *)(slab_ptr + mid * slab_stride);
-        if ( CA_OBJ_ISNAN(v) ) { hi = mid; continue; }   /* NaN sorts last */
-        VALUE r = rb_funcall(v, rb_intern("<=>"), 1, query_val);
-        if ( NIL_P(r) ) {
-          ca_obj_compare_refused(v, query_val);
-        }
-        int c = NUM2INT(r);
+        int c = ca_obj_order_cmp(v, query_val);
         if ( c < 0 ) lo = mid + 1;
         else if ( c > 0 ) hi = mid;
-        else { result = mid; break; }
+        else { result = mid; hi = mid; }   /* keep looking left: the first equal cell */
       }
     C
     fixlen: <<~C,
@@ -8022,7 +8073,7 @@ MkKernel.search :bsearch_addr,
         int c = memcmp(v, query_val, (size_t) ca->bytes);
         if ( c < 0 ) lo = mid + 1;
         else if ( c > 0 ) hi = mid;
-        else { result = mid; break; }
+        else { result = mid; hi = mid; }   /* keep looking left: the first equal cell */
       }
     C
   },
@@ -8188,19 +8239,18 @@ MkKernel.search :linear_section_binary,
   no_match:       "NAN",
   no_match_check: "isnan(op[0])",
   mask_self:      :raise,
-  mask_query:     :undef,
   body: <<~C,
     result = NAN;
     if ( slab_n <= 1 ) {
       /* A one-point axis has no segment: only the point itself is in
        * range, at 0.  An empty one has nothing in range. */
-      if ( slab_n == 1 && query_val == *(double *)slab_ptr ) {
+      if ( slab_n == 1 && query_val == *(ca_u_float64_t *)slab_ptr ) {
         result = 0.0;
       }
     }
     else {
-      double y0 = *(double *)(slab_ptr + 0 * slab_stride);
-      double yn = *(double *)(slab_ptr + (slab_n - 1) * slab_stride);
+      double y0 = *(ca_u_float64_t *)(slab_ptr + 0 * slab_stride);
+      double yn = *(ca_u_float64_t *)(slab_ptr + (slab_n - 1) * slab_stride);
       int ascending = ( yn >= y0 );   /* grid direction (monotonic assumed) */
       ca_size_t x1 = 0;
       int       found = 0;
@@ -8223,8 +8273,8 @@ MkKernel.search :linear_section_binary,
           double  gd = (query_val - y0) / (yn - y0) * (double)(slab_n - 1);
           if ( gd >= 0.0 && gd < (double)(slab_n - 1) ) {
             ca_size_t ag = (ca_size_t) gd;
-            double yga  = *(double *)(slab_ptr + ag * slab_stride);
-            double ygb  = *(double *)(slab_ptr + (ag + 1) * slab_stride);
+            double yga  = *(ca_u_float64_t *)(slab_ptr + ag * slab_stride);
+            double ygb  = *(ca_u_float64_t *)(slab_ptr + (ag + 1) * slab_stride);
             if ( (yga - query_val) * (ygb - query_val) <= 0 ) {
               x1 = ag; found = 1;
             }
@@ -8239,7 +8289,7 @@ MkKernel.search :linear_section_binary,
           double    ya = y0;
           while ( (b - a) > 1 ) {
             ca_size_t c  = (a + b) / 2;
-            double    yc = *(double *)(slab_ptr + c * slab_stride);
+            double    yc = *(ca_u_float64_t *)(slab_ptr + c * slab_stride);
             if ( (ya - query_val) * (yc - query_val) <= 0 ) { b = c; }
             else { a = c; ya = yc; }
           }
@@ -8248,8 +8298,8 @@ MkKernel.search :linear_section_binary,
       }
 
       if ( found ) {
-        double y1 = *(double *)(slab_ptr + x1 * slab_stride);
-        double y2 = *(double *)(slab_ptr + (x1 + 1) * slab_stride);
+        double y1 = *(ca_u_float64_t *)(slab_ptr + x1 * slab_stride);
+        double y2 = *(ca_u_float64_t *)(slab_ptr + (x1 + 1) * slab_stride);
         double rest = (query_val - y1) / (y2 - y1);
         if ( y2 != 0.0 && fabs(y2 - query_val) / fabs(y2) < DBL_EPSILON * 100 ) {
           result = (double)(x1 + 1);
@@ -8280,13 +8330,12 @@ MkKernel.search :linear_section_linear,
   no_match:       "NAN",
   no_match_check: "isnan(op[0])",
   mask_self:      :raise,
-  mask_query:     :undef,
   body: <<~C,
     result = NAN;
     if ( slab_n <= 1 ) {
       /* A one-point axis has no segment: only the point itself is in
        * range, at 0.  An empty one has nothing in range. */
-      if ( slab_n == 1 && query_val == *(double *)slab_ptr ) {
+      if ( slab_n == 1 && query_val == *(ca_u_float64_t *)slab_ptr ) {
         result = 0.0;
       }
     }
@@ -8294,15 +8343,15 @@ MkKernel.search :linear_section_linear,
       ca_size_t x1 = 0;
       int       found = 0;
       for ( ca_size_t k = 0; k < slab_n - 1; k++ ) {
-        double yk  = *(double *)(slab_ptr + k * slab_stride);
-        double yk1 = *(double *)(slab_ptr + (k + 1) * slab_stride);
+        double yk  = *(ca_u_float64_t *)(slab_ptr + k * slab_stride);
+        double yk1 = *(ca_u_float64_t *)(slab_ptr + (k + 1) * slab_stride);
         if ( (query_val - yk) * (query_val - yk1) <= 0 ) {
           x1 = k; found = 1; break;
         }
       }
       if ( found ) {
-        double y1 = *(double *)(slab_ptr + x1 * slab_stride);
-        double y2 = *(double *)(slab_ptr + (x1 + 1) * slab_stride);
+        double y1 = *(ca_u_float64_t *)(slab_ptr + x1 * slab_stride);
+        double y2 = *(ca_u_float64_t *)(slab_ptr + (x1 + 1) * slab_stride);
         result = (query_val - y1) / (y2 - y1) + (double) x1;
       }
 
@@ -8323,15 +8372,14 @@ MkKernel.search :linear_fetch,
   no_match:       "NAN",
   no_match_check: "isnan(op[0])",
   mask_self:      :raise,
-  mask_query:     :undef,
   body: <<~C,
     result = NAN;
     if ( slab_n >= 1 && query_val >= 0.0 && query_val <= (double)(slab_n - 1) ) {
       ca_size_t il = (ca_size_t) floor(query_val);
       ca_size_t iu = (ca_size_t) ceil(query_val);
       double    w  = query_val - floor(query_val);
-      double    yl = *(double *)(slab_ptr + il * slab_stride);
-      double    yu = *(double *)(slab_ptr + iu * slab_stride);
+      double    yl = *(ca_u_float64_t *)(slab_ptr + il * slab_stride);
+      double    yu = *(ca_u_float64_t *)(slab_ptr + iu * slab_stride);
       result = yu * w + yl * (1.0 - w);
     }
   C
@@ -8588,7 +8636,9 @@ MkKernel.monfunc :round,
   source: MkKernel::ALL_NUMERIC + [:object],
   expr:   {
     int:    "(#2) = (#1);",
-    float:  "(#2) = ((#1)>0.0) ? floor((#1)+0.5) : ((#1)<0.0) ? ceil((#1)-0.5) : (#1);",
+    # Half away from zero, as Float#round.  floor(x + 0.5) would round in
+    # the addition: 0.49999999999999994 + 0.5 is 1.0.
+    float:  "(#2) = round(#1);",
     object: '(#2) = rb_funcall((#1), rb_intern("round"), 0);',
   }
 
@@ -8889,7 +8939,7 @@ MkKernel.monfunc :rad_pi,
 # Pair-wise max / min — three flavours:
 #
 #   pmax / pmin      CArray legacy names (kept for backwards compat),
-#                    NaN-skip semantics via C99 fmax / fmin.
+#                    NaN-skip semantics, as C99 fmax / fmin.
 #   fmax / fmin      Ruby-level aliases of pmax / pmin (= same kernel,
 #                    same NaN-skip behaviour).  Matches the C99 library
 #                    function of the same name.
@@ -8899,12 +8949,15 @@ MkKernel.monfunc :rad_pi,
 #
 # Integer branches are identical across all three (no NaN).  The float
 # and object branches differ: pmax / pmin let a NaN lose (an object cell's
-# Float NaN included), maximum / minimum let it win.
+# Float NaN included), maximum / minimum let it win.  In both flavours, of
+# two zeros the max is +0 and the min is -0 in either order.  C's fmax /
+# fmin leave that choice to the library (glibc and macOS answer
+# differently), so the float branch does not call them.
 MkKernel.binop :pmax,
   source: MkKernel::ALL_NUMERIC + [:object],
   expr:   {
     int:    "(#3) = (#1) > (#2) ? (#1) : (#2);",
-    float:  "(#3) = fmax(#1, #2);",
+    float:  "(#3) = isnan(#1) ? (#2) : isnan(#2) ? (#1) : ((#1) > (#2) || ((#1) == (#2) && !signbit(#1))) ? (#1) : (#2);",
     object: '(#3) = ca_obj_extremum_replaces((#2), (#1), rb_intern(">")) ? (#2) : (#1);',
   }
 
@@ -8912,7 +8965,7 @@ MkKernel.binop :pmin,
   source: MkKernel::ALL_NUMERIC + [:object],
   expr:   {
     int:    "(#3) = (#1) < (#2) ? (#1) : (#2);",
-    float:  "(#3) = fmin(#1, #2);",
+    float:  "(#3) = isnan(#1) ? (#2) : isnan(#2) ? (#1) : ((#1) < (#2) || ((#1) == (#2) && signbit(#1))) ? (#1) : (#2);",
     object: '(#3) = ca_obj_extremum_replaces((#2), (#1), rb_intern("<")) ? (#2) : (#1);',
   }
 
@@ -9259,13 +9312,13 @@ MkKernel.triop :fms,
 #     UNDEF-fill case can now be expressed as `out = a.clip(lo, hi);
 #     out[a.lt(lo) | a.gt(hi)] = UNDEF`).  Pinned in test.
 #
-# Bound at the C level as `__clip_ki__` (private kernel entry).  The
+# Bound at the C level as `clamp_between` (made private by lazy.rb).  The
 # public `CArray#clip` is a Ruby wrapper in lib/carray/math.rb that
 # restores the rich pre-3.0 API surface (nil-bound one-sided clip +
 # optional fill).  `clip!` is retired in 3.0 (view-by-default
 # convention) so the bang form is suppressed via `bang: false`.
 MkKernel.triop :clip,
-  op:     "__clip_ki__",
+  op:     "clamp_between",
   bang:   false,
   source: MkKernel::ALL_NUMERIC + [:object],
   expr:   {