carray 3.0.0 → 3.0.1

data/ext/carray_kernels_sort.c

@@ -34,6 +34,7 @@ static inline VALUE BOOL2VAL (boolean8_t x) { return x ? Qtrue : Qfalse; }
 #endif
 
 #include "ca_op_powi.h"
+#include "ca_op_cmplx64.h"
 
 /* ===== sort_index_ki ============================================ */
 

data/ext/carray_kernels_triop.c

@@ -34,6 +34,7 @@ static inline VALUE BOOL2VAL (boolean8_t x) { return x ? Qtrue : Qfalse; }
 #endif
 
 #include "ca_op_powi.h"
+#include "ca_op_cmplx64.h"
 
 /* ===== triop fma ============================================ */
 

data/ext/carray_lazy.c

@@ -93,6 +93,10 @@ ca_lazy_marker_setup (CALazyMarker *ca, CArray *parent)
      the parent and violate the shadow semantics. */
   ca_set_flag(ca, CA_FLAG_READ_ONLY);
 
+  /* Storage-identical wrapper: the kernel_iterator entry strip and the
+     view-creation lift both ask for this. */
+  ca_set_flag(ca, CA_FLAG_IS_LAZY_MARKER);
+
   return 0;
 }
 
@@ -253,6 +257,105 @@ rb_ca_lazy (VALUE self)
   return rb_ca_lazy_marker_new(self);
 }
 
+/* ------------------------------------------------------------------- */
+/* CA_OBJECT GC guard                                                   */
+/* ------------------------------------------------------------------- */
+
+/* A CA_OBJECT cell is a VALUE.  While a view materialises into a raw
+   buffer -- the destination `copy` will hand out, a view's own attach
+   buffer, or an arena scratch -- those VALUEs live somewhere the GC
+   cannot see: an xmalloc'd block owned by no Ruby object.  For every
+   other data_type that is harmless, but an object-lane kernel calls
+   rb_funcall per cell, and any of those can collect.  The results
+   written so far are then freed under the buffer, and the array comes
+   back holding recycled slots: wrong values, and a crash once a slot is
+   reused as something else.
+
+   So such buffers are registered here for the duration and marked from
+   a hidden guard object that lives as long as the process.  Two kinds
+   are registered:
+
+     holds   a contiguous run of cells, pushed and popped around a
+             whole-view transfer or the handoff to a Ruby owner
+     slots   an arena scratch, tagged at acquire and held until release
+
+   Only contiguous runs are held.  A strided window would cover the same
+   cells but its extent has to be taken on trust from the caller's
+   strides, and a window wider than the buffer behind it would be filled
+   and marked out of bounds.  The whole-view entry knows the extent
+   exactly, and every partial transfer writes inside it. */
+
+#define CA_GC_HOLD_MAX 64
+
+typedef struct ca_gc_hold {
+  VALUE    *ptr;
+  ca_size_t elements;
+} ca_gc_hold_t;
+
+static ca_gc_hold_t ca_gc_holds[CA_GC_HOLD_MAX];
+static int          ca_gc_hold_depth = 0;
+
+static void ca_lazy_arena_mark_object_slots (void);
+
+/* Returns the depth to hand back to ca_gc_hold_pop_to, so nested holds
+   unwind in order.  Pushing is best-effort at the ceiling: a chain
+   deeper than CA_GC_HOLD_MAX loses the protection for its innermost
+   buffers rather than raising in the middle of a materialise.  The cells
+   must already be valid VALUEs -- the caller fills a fresh buffer with
+   Qnil first. */
+int
+ca_gc_hold_push (void *ptr, ca_size_t n_elements)
+{
+  int depth = ca_gc_hold_depth;
+
+  if ( depth >= CA_GC_HOLD_MAX || n_elements <= 0 || ptr == NULL ) {
+    return -1;
+  }
+  ca_gc_holds[depth].ptr      = (VALUE *) ptr;
+  ca_gc_holds[depth].elements = n_elements;
+  ca_gc_hold_depth = depth + 1;
+  return depth;
+}
+
+void
+ca_gc_hold_pop_to (int depth)
+{
+  if ( depth >= 0 && depth < ca_gc_hold_depth ) {
+    ca_gc_hold_depth = depth;
+  }
+}
+
+static VALUE ca_gc_guard = Qnil;
+
+/* The wrapped pointer must be non-NULL: Ruby's GC skips the mark
+   function of a TypedData whose data pointer is NULL. */
+static int ca_gc_guard_body = 0;
+
+static void
+ca_gc_guard_mark (void *ptr)
+{
+  int i;
+  (void) ptr;
+  for ( i = 0; i < ca_gc_hold_depth; i++ ) {
+    VALUE    *p = ca_gc_holds[i].ptr;
+    ca_size_t n = ca_gc_holds[i].elements;
+    while ( n-- ) rb_gc_mark(*p++);
+  }
+  ca_lazy_arena_mark_object_slots();
+}
+
+static void
+ca_gc_guard_free (void *ptr)
+{
+  (void) ptr;
+}
+
+static const rb_data_type_t ca_gc_guard_data_type = {
+  "carray_object_gc_guard",
+  { ca_gc_guard_mark, ca_gc_guard_free, NULL, },
+  0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+};
+
 /* ------------------------------------------------------------------- */
 /* ca_lazy_arena                                                        */
 /* ------------------------------------------------------------------- */
@@ -277,8 +380,9 @@ rb_ca_lazy (VALUE self)
    programming error or a pathological chain rather than a graceful
    fallback.
 
-   CAREFUL: single-thread only (see CLAUDE.md "thread-safety is not a
-   goal").  The arena is process-global static state. */
+   CAREFUL: single-thread only -- thread-safety across concurrent access
+   is not a goal (see guides/devel/04_attach_lifecycle.md).  The arena is
+   process-global static state. */
 
 #define CA_LAZY_ARENA_SLOTS 32
 
@@ -286,6 +390,7 @@ typedef struct ca_lazy_arena_slot {
   void     *ptr;     /* xmalloc'd buffer, NULL = unused-and-unallocated */
   ca_size_t bytes;   /* allocated size of ptr (= capacity of this slot) */
   int       in_use;  /* 1 = currently acquired, 0 = available */
+  ca_size_t object_elements;  /* >0 = holds VALUEs; marked while in_use */
 } ca_lazy_arena_slot_t;
 
 typedef struct ca_lazy_arena {
@@ -308,6 +413,7 @@ ca_lazy_arena_enter (void)
     int i;
     for ( i = 0; i < CA_LAZY_ARENA_SLOTS; i++ ) {
       ca_lazy_arena.slots[i].in_use = 0;
+      ca_lazy_arena.slots[i].object_elements = 0;
     }
   }
   ca_lazy_arena.depth++;
@@ -319,7 +425,16 @@ ca_lazy_arena_exit (void)
   if ( ca_lazy_arena.depth > 0 ) {
     ca_lazy_arena.depth--;
   }
-  /* Keep all slot buffers; they amortise into subsequent calls. */
+  /* Keep all slot buffers, and leave in_use and the object tags alone.
+     A scratch can be acquired outside any enter/exit bracket -- a lazy
+     view's attach path does exactly that -- so an inner materialise
+     dropping to depth 0 is not evidence that nothing is held.  Recovery
+     from an abandoned slot happens on the next depth-0 entry instead.
+
+     An abandoned object slot stays marked until then, which is a leak
+     but not a hazard: acquire only ever hands out a slot that is not in
+     use, so nothing overwrites it, and marking is what keeps its cells
+     from being collected in the first place. */
 }
 
 void *
@@ -350,6 +465,7 @@ ca_lazy_arena_acquire (ca_size_t bytes)
   }
   if ( best >= 0 ) {
     ca_lazy_arena.slots[best].in_use = 1;
+    ca_lazy_arena.slots[best].object_elements = 0;
     ca_lazy_arena.debug_reuse_count++;
     return ca_lazy_arena.slots[best].ptr;
   }
@@ -367,6 +483,7 @@ ca_lazy_arena_acquire (ca_size_t bytes)
     ca_lazy_arena.slots[empty].ptr    = xmalloc(bytes);
     ca_lazy_arena.slots[empty].bytes  = bytes;
     ca_lazy_arena.slots[empty].in_use = 1;
+    ca_lazy_arena.slots[empty].object_elements = 0;
     ca_lazy_arena.debug_xmalloc_count++;
     return ca_lazy_arena.slots[empty].ptr;
   }
@@ -380,6 +497,7 @@ ca_lazy_arena_acquire (ca_size_t bytes)
       ca_lazy_arena.slots[i].ptr    = xmalloc(bytes);
       ca_lazy_arena.slots[i].bytes  = bytes;
       ca_lazy_arena.slots[i].in_use = 1;
+      ca_lazy_arena.slots[i].object_elements = 0;
       ca_lazy_arena.debug_xmalloc_count++;
       return ca_lazy_arena.slots[i].ptr;
     }
@@ -402,6 +520,7 @@ ca_lazy_arena_release (void *ptr)
   for ( i = 0; i < CA_LAZY_ARENA_SLOTS; i++ ) {
     if ( ca_lazy_arena.slots[i].ptr == ptr ) {
       ca_lazy_arena.slots[i].in_use = 0;
+      ca_lazy_arena.slots[i].object_elements = 0;
       return;
     }
   }
@@ -410,6 +529,52 @@ ca_lazy_arena_release (void *ptr)
      genuine bugs at unrelated callsites.  */
 }
 
+/* Acquire a scratch that will hold `n_elements` VALUEs.
+
+   Unlike the byte-sized acquire this one Qnil-fills the buffer and tags
+   the slot, so the cells stay markable for as long as the caller holds
+   it.  Object-lane kernels run rb_funcall over their operands, so a
+   scratch pulled from a lazy operand holds VALUEs that exist nowhere
+   else until the kernel has consumed them. */
+void *
+ca_lazy_arena_acquire_object (ca_size_t n_elements)
+{
+  void *ptr;
+  int i;
+
+  if ( n_elements < 0 ) n_elements = 0;
+  ptr = ca_lazy_arena_acquire(n_elements * (ca_size_t) sizeof(VALUE));
+
+  {
+    VALUE *p = (VALUE *) ptr;
+    ca_size_t n = n_elements;
+    while ( n-- ) *p++ = Qnil;
+  }
+
+  for ( i = 0; i < CA_LAZY_ARENA_SLOTS; i++ ) {
+    if ( ca_lazy_arena.slots[i].ptr == ptr ) {
+      ca_lazy_arena.slots[i].object_elements = n_elements;
+      break;
+    }
+  }
+  return ptr;
+}
+
+static void
+ca_lazy_arena_mark_object_slots (void)
+{
+  int i;
+  for ( i = 0; i < CA_LAZY_ARENA_SLOTS; i++ ) {
+    ca_lazy_arena_slot_t *s = &ca_lazy_arena.slots[i];
+    ca_size_t n;
+    VALUE *p;
+    if ( ! s->in_use || s->object_elements <= 0 ) continue;
+    p = (VALUE *) s->ptr;
+    n = s->object_elements;
+    while ( n-- ) rb_gc_mark(*p++);
+  }
+}
+
 /* Test instrumentation — exposed to Ruby for the arena smoke and
    regression assertions.  Not user-facing API. */
 static VALUE
@@ -490,8 +655,7 @@ rb_ca_lazy_arena_s_slot_capacities (VALUE klass)
  *                       just let the scalar choose)
  * numeric self      -> self_ca->data_type
  *
- * Callers broadcast against the array via stride 0 in xfer_stride
- * (or wrap the scalar in CAUnboundRepeat when the chain needs it). */
+ * Callers broadcast against the array via stride 0 in xfer_stride. */
 VALUE
 ca_lazy_wrap_scalar (VALUE other, CArray *self_ca)
 {
@@ -543,6 +707,12 @@ Init_carray_lazy (void)
 
   rb_define_method(rb_cCArray, "lazy", rb_ca_lazy, 0);
 
+  /* Hidden, never collected: its mark function is what keeps the
+     in-flight CA_OBJECT buffers reachable. */
+  ca_gc_guard = TypedData_Wrap_Struct(rb_cObject, &ca_gc_guard_data_type,
+                                      &ca_gc_guard_body);
+  rb_gc_register_mark_object(ca_gc_guard);
+
   rb_define_alloc_func(rb_cCALazyMarker, rb_ca_lazy_marker_s_allocate);
   rb_define_method(rb_cCALazyMarker, "initialize_copy",
                                       rb_ca_lazy_marker_initialize_copy, 1);

data/ext/carray_mask.c

@@ -607,7 +607,7 @@ ca_count_not_masked (void *ap)
 /* Scalar-fill every masked cell with fill_value, then clear its mask bit.
    This depends only on the cell BYTE WIDTH, not the data type (the work is
    a width-sized copy), so a single width switch covers every numeric /
-   complex / object dtype; the default arm handles fixlen and any other
+   complex / object data type; the default arm handles fixlen and any other
    width via memcpy.  The typed arms let the compiler emit a plain store.  */
 #define proc_fill_bang_width(T)                 \
   {                                             \
@@ -894,6 +894,7 @@ rb_ca_value_array (VALUE self)
   TypedData_Get_Struct(self, CArray, &carray_data_type, ca);
 
   obj = rb_ca_refer_new(self, ca->data_type, ca->ndim, ca->dim, ca->bytes, 0);
+
   TypedData_Get_Struct(obj, CArray, &carray_data_type, co);
 
   /* Value arrays ignore the mask by definition.  ca_stride_setup
@@ -1112,7 +1113,7 @@ The returned array doesn't have the mask array.
 static VALUE
 rb_ca_unmask_method (int argc, VALUE *argv, VALUE self)
 {
-  volatile VALUE rfval = CA_NIL, rcs;
+  volatile VALUE rfval = CA_UNSPECIFIED, rcs;
   CArray *ca;
   CScalar *cv;
   char *fval = NULL;
@@ -1125,7 +1126,7 @@ rb_ca_unmask_method (int argc, VALUE *argv, VALUE self)
 
   TypedData_Get_Struct(self, CArray, &carray_data_type, ca);
 
-  if ( rfval != CA_NIL ) {
+  if ( rfval != CA_UNSPECIFIED ) {
     /* Face has surface != storage; the fill value is cast in the storage data_type. */
     int8_t conv_type = ca->data_type;
     if ( ca_is_face(ca) ) {
@@ -1171,7 +1172,7 @@ The returned array doesn't have the mask array.
 static VALUE
 rb_ca_unmask_copy_method (int argc, VALUE *argv, VALUE self)
 {
-  volatile VALUE obj, rfval = CA_NIL, rcs;
+  volatile VALUE obj, rfval = CA_UNSPECIFIED, rcs;
   CArray *ca, *co;
   CScalar *cv;
   char *fval = NULL;
@@ -1182,7 +1183,7 @@ rb_ca_unmask_copy_method (int argc, VALUE *argv, VALUE self)
 
   TypedData_Get_Struct(self, CArray, &carray_data_type, ca);
 
-  if ( rfval != CA_NIL ) {
+  if ( rfval != CA_UNSPECIFIED ) {
     /* Face has surface != storage; the fill value is cast in the storage data_type. */
     int8_t conv_type = ca->data_type;
     if ( ca_is_face(ca) ) {

data/ext/carray_median_percentile.c

@@ -152,7 +152,7 @@ typedef enum { PCT_LOWER_ONLY, PCT_UPPER_ONLY, PCT_BOTH } pct_need_t;
 
 /* [shared] which of lower (sorted[k]) / upper (sorted[k+1]) the method
    needs.  Lets the caller skip the fetch it won't use.  Used by both
-   lanes (the need classification is dtype-independent). */
+   lanes (the need classification is data type-independent). */
 static pct_need_t
 pct_need (VALUE method, long k, double r, long n)
 {

data/ext/carray_memory_view.c

@@ -99,46 +99,70 @@ static ca_mv_runtime_type_t ca_mv_runtime_types[CA_MV_NUM_RUNTIME_OBJ_TYPES] = {
 
 /* ---------------- format / data_type mapping ----------------
    Bidirectional table.  Outbound table (ca_mv_format_for) picks the
-   canonical PEP 3118 specifier for each CArray data_type.  Inbound
-   table (ca_mv_data_type_from_format) is Postel: it accepts a wider
-   set of synonyms that other producers emit (they describe int32
-   differently). */
+   specifier each CArray data_type publishes at the top level of a
+   MemoryView.  Inbound table (ca_mv_data_type_from_format) is Postel:
+   it accepts a wider set of synonyms that other producers emit (they
+   describe int32 differently).
+
+   The outbound spellings follow ruby/memory_view.h, which documents
+   `format` as a sequence of pack-derived specifiers and expects
+   item_size to equal rb_memory_view_item_size_from_format(format).
+   PEP 3118 reuses several of those letters for other types, so the two
+   vocabularies agree on `i` / `I` / `q` / `Q` / `f` / `d` and disagree
+   below 32 bits: what PEP calls `b` / `B` / `h` / `H`, Ruby calls
+   `c` / `C` / `s` / `S`.  Emitting Ruby's spelling is what lets a
+   generic Ruby consumer -- Fiddle::MemoryView, say -- read elements out
+   of a CArray at all; PEP-speaking consumers are reached across the
+   language boundary, where pycall-memoryview translates.
+
+   Three types have no Ruby spelling at all: boolean and the two
+   complex widths.  They keep their PEP form, which Ruby's parser
+   rejects, because the alternative ("C" for a boolean, "dd" for a
+   complex) would misdescribe the data rather than merely fail to
+   describe it. */
 
 static const char *
 ca_mv_format_for (int8_t data_type)
 {
   switch (data_type) {
-  case CA_BOOLEAN: return "?";
-  case CA_INT8:    return "b";
-  case CA_UINT8:   return "B";
-  case CA_INT16:   return "h";
-  case CA_UINT16:  return "H";
+  case CA_BOOLEAN: return "?";   /* no Ruby spelling */
+  case CA_INT8:    return "c";
+  case CA_UINT8:   return "C";
+  case CA_INT16:   return "s";
+  case CA_UINT16:  return "S";
   case CA_INT32:   return "i";
   case CA_UINT32:  return "I";
   case CA_INT64:   return "q";
   case CA_UINT64:  return "Q";
   case CA_FLOAT32: return "f";
   case CA_FLOAT64: return "d";
-  case CA_CMPLX64:  return "Zf";
-  case CA_CMPLX128: return "Zd";
+  case CA_CMPLX64:  return "Zf";   /* no Ruby spelling */
+  case CA_CMPLX128: return "Zd";   /* no Ruby spelling */
   default:
     return NULL;
   }
 }
 
 /* PEP 3118 single-character specifier for a primitive data_type, for
-   use inside a `T{...}` struct format body.  Identical to the top-level
-   producer specifier: since the 2026-06-29 PEP 3118 strict flip both the
-   top-level and struct-body contexts emit the same chars, so this
-   delegates to ca_mv_format_for.  Kept as a distinct entry point so the
-   struct-body context can diverge again without touching call sites.
+   use inside a `T{...}` struct format body.  This is the context the
+   distinct entry point was kept for: `T{...}` is a PEP 3118 construct
+   with no Ruby counterpart, so a record body stays in PEP's vocabulary
+   even though the top level now publishes Ruby's.  A consumer reading
+   the body reads it as PEP, and a bridge that forwards the record
+   whole -- as pycall-memoryview does -- keeps it intact.
 
    Returns NULL for non-primitive (CA_FIXLEN / CA_OBJECT) or for the
    retired float128 / cmplx256 enum slots. */
 static const char *
 ca_mv_pep3118_char (int8_t data_type)
 {
-  return ca_mv_format_for(data_type);
+  switch (data_type) {
+  case CA_INT8:   return "b";
+  case CA_UINT8:  return "B";
+  case CA_INT16:  return "h";
+  case CA_UINT16: return "H";
+  default:        return ca_mv_format_for(data_type);
+  }
 }
 
 /* Build (or fetch from cache) the PEP 3118 struct format string for
@@ -362,7 +386,7 @@ ca_mv_data_type_from_format (const char *format, ssize_t item_size)
 }
 
 /* Derive the data_type a MemoryView producer would import as, by parsing its
-   format — without importing the data.  This is the canonical "what dtype is
+   format — without importing the data.  This is the canonical "what data type is
    this MV" question, decoupled from the import strategy (copy via
    from_memory_view vs zero-copy via wrap_memory_view).  Used by
    ca_arg_to_data_type so CArray.result_type / promote_list can treat an MV
@@ -414,7 +438,6 @@ ca_mv_strategy_for (int16_t obj_type)
   if (obj_type == CA_OBJ_SELECT) return CA_MV_ATTACH;
   if (obj_type == CA_OBJ_REPEAT) return CA_MV_REPEAT;
   if (obj_type == CA_OBJ_OBJECT) return CA_MV_REJECT;
-  if (obj_type == CA_OBJ_UNBOUND_REPEAT) return CA_MV_REJECT;
   /* Plain CAStride (created via #as_strided, or wrap_memory_view from
      a strided producer).  Reuse the CA_MV_TRANS handler since it
      already reads strides[] / base_offset directly from the CAStride
@@ -510,7 +533,6 @@ ca_mv_reject_reason_for (CArray *ca)
   /* Compile-time obj_types we know about */
   if (ca->obj_type == CA_OBJ_SELECT)          return "boolean-mask selection (positions not expressible as strides)";
   if (ca->obj_type == CA_OBJ_OBJECT)          return "stores Ruby VALUEs, not raw bytes";
-  if (ca->obj_type == CA_OBJ_UNBOUND_REPEAT)  return "shape is not bound";
   /* Runtime-assigned obj_types: look up by name in the strategy table */
   for (int i = 0; i < CA_MV_NUM_RUNTIME_OBJ_TYPES; i++) {
     if (ca_mv_runtime_types[i].id == ca->obj_type) {
@@ -1110,7 +1132,10 @@ ca_mv_acquire_mask_view (VALUE mask_src, int8_t data_ndim,
     if (c == '<' || c == '>' || c == '=' || c == '!' || c == '@') {
       c = mv->format[1];
     }
-    if (c != '?' && c != 'B' && c != 'b') {
+    /* One byte per element, under either vocabulary: PEP 3118 spells the
+       integer widths 'b' / 'B', Ruby's MemoryView spells them 'c' / 'C'.
+       '?' is PEP's boolean, which Ruby has no spelling for. */
+    if (c != '?' && c != 'B' && c != 'b' && c != 'C' && c != 'c') {
       char fmt_buf[64];
       snprintf(fmt_buf, sizeof(fmt_buf), "%s", mv->format);
       MV_MASK_CLEANUP_AND_RAISE("mask: format must be bool/uint8/int8 (got %s)",