carray 3.0.0 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. checksums.yaml +4 -4
  2. data/.yardopts +2 -1
  3. data/CHANGELOG.md +325 -3
  4. data/{NEWS.md → CHANGELOG.v1.md} +1 -1
  5. data/README.md +32 -32
  6. data/carray.gemspec +12 -8
  7. data/ext/ca_axis_group.c +55 -19
  8. data/ext/ca_binop_dispatch.c +5 -6
  9. data/ext/ca_binop_dispatch.h +0 -7
  10. data/ext/ca_categorical_iterator.c +13 -13
  11. data/ext/ca_for_each_element.h +6 -8
  12. data/ext/ca_group_iter.c +1 -1
  13. data/ext/ca_kernel_iterator.c +69 -23
  14. data/ext/ca_kernel_iterator.h +3 -10
  15. data/ext/ca_obj_bincmp.c +57 -27
  16. data/ext/ca_obj_binop.c +86 -30
  17. data/ext/ca_obj_const_string.c +3 -1
  18. data/ext/ca_obj_face.c +80 -0
  19. data/ext/ca_obj_face.h +33 -1
  20. data/ext/ca_obj_fake.c +11 -0
  21. data/ext/ca_obj_fixlen_string.c +3 -1
  22. data/ext/ca_obj_grid.c +11 -3
  23. data/ext/ca_obj_meld.c +13 -8
  24. data/ext/ca_obj_moncmp.c +33 -14
  25. data/ext/ca_obj_monop.c +19 -7
  26. data/ext/ca_obj_object.c +99 -4
  27. data/ext/ca_obj_record.c +3 -1
  28. data/ext/ca_obj_refer.c +30 -6
  29. data/ext/ca_obj_roll.c +13 -5
  30. data/ext/ca_obj_select_axis.c +16 -6
  31. data/ext/ca_obj_shift.c +3 -3
  32. data/ext/ca_obj_stride.c +106 -23
  33. data/ext/ca_obj_string.c +3 -1
  34. data/ext/ca_obj_tile.c +13 -5
  35. data/ext/ca_obj_time.c +3 -1
  36. data/ext/ca_obj_timedelta.c +3 -1
  37. data/ext/ca_obj_transpose.c +1 -1
  38. data/ext/ca_obj_triop.c +62 -23
  39. data/ext/ca_obj_window.c +86 -17
  40. data/ext/ca_op_cmplx64.h +123 -0
  41. data/ext/ca_op_ipower.c +0 -3
  42. data/ext/ca_sort_kernels.h +5 -5
  43. data/ext/ca_sweep_engine.c +78 -35
  44. data/ext/ca_sweep_engine.h +13 -4
  45. data/ext/ca_transform_common.c +7 -0
  46. data/ext/carray.h +127 -130
  47. data/ext/carray_access.c +56 -34
  48. data/ext/carray_bincount.c +8 -8
  49. data/ext/carray_broadcast.c +99 -6
  50. data/ext/carray_build_flags.h +3 -0
  51. data/ext/carray_call_cfunc.c +1491 -0
  52. data/ext/carray_call_cfunc.h +153 -0
  53. data/ext/carray_cast.c +69 -31
  54. data/ext/carray_conversion.c +22 -22
  55. data/ext/carray_copy.c +12 -2
  56. data/ext/carray_core.c +57 -10
  57. data/ext/carray_count.c +1 -1
  58. data/ext/carray_element.c +7 -7
  59. data/ext/carray_factorize.c +28 -28
  60. data/ext/carray_hold.c +1 -1
  61. data/ext/carray_index_classifier.c +7 -21
  62. data/ext/carray_internal.h +19 -3
  63. data/ext/carray_kernels_bincmp.c +1 -0
  64. data/ext/carray_kernels_binop.c +320 -298
  65. data/ext/carray_kernels_init.c +1095 -0
  66. data/ext/carray_kernels_map.c +1 -0
  67. data/ext/carray_kernels_moncmp.c +1 -0
  68. data/ext/carray_kernels_monop.c +412 -411
  69. data/ext/carray_kernels_reduce_aggregate.c +88 -87
  70. data/ext/carray_kernels_reduce_boolean.c +1 -0
  71. data/ext/carray_kernels_reduce_cumulative.c +54 -53
  72. data/ext/carray_kernels_reduce_extreme.c +51 -50
  73. data/ext/carray_kernels_reduce_variance.c +1 -0
  74. data/ext/carray_kernels_scan.c +1 -0
  75. data/ext/carray_kernels_search.c +93 -92
  76. data/ext/carray_kernels_sort.c +1 -0
  77. data/ext/carray_kernels_triop.c +1 -0
  78. data/ext/carray_lazy.c +175 -5
  79. data/ext/carray_mask.c +6 -5
  80. data/ext/carray_median_percentile.c +1 -1
  81. data/ext/carray_memory_view.c +46 -21
  82. data/ext/carray_operator.c +46 -49
  83. data/ext/carray_partition.c +4 -4
  84. data/ext/carray_random.c +7 -7
  85. data/ext/carray_scatter.c +1 -1
  86. data/ext/carray_sort.c +3 -3
  87. data/ext/carray_sort_kernel.c +10 -10
  88. data/ext/carray_test.c +1 -1
  89. data/ext/extconf.rb +18 -0
  90. data/ext/mk_call_cfunc.rb +243 -52
  91. data/ext/mkkernel.rb +354 -100
  92. data/ext/ruby_carray.c +18 -16
  93. data/ext/version.h +4 -4
  94. data/lib/carray/autoload_carray.rb +19 -0
  95. data/lib/carray/autoload_method_extension.rb +2 -1
  96. data/lib/carray/axis_group.rb +7 -7
  97. data/lib/carray/basics.rb +59 -59
  98. data/lib/carray/bincount_nd.rb +12 -12
  99. data/lib/carray/block_iterator.rb +24 -14
  100. data/lib/carray/categorical.rb +4 -4
  101. data/lib/carray/categorical_iterator.rb +98 -76
  102. data/lib/carray/conditional.rb +14 -14
  103. data/lib/carray/construct.rb +21 -0
  104. data/lib/carray/core_extensions.rb +62 -5
  105. data/lib/carray/data_type_extension.rb +18 -1
  106. data/lib/carray/frame/concat.rb +3 -3
  107. data/lib/carray/frame/convert.rb +1 -1
  108. data/lib/carray/frame/frame.rb +4 -11
  109. data/lib/carray/frame/io.rb +159 -2
  110. data/lib/carray/frame/sort.rb +1 -1
  111. data/lib/carray/frame/verbs.rb +18 -1
  112. data/lib/carray/fuse_source.rb +123 -0
  113. data/lib/carray/fusion.rb +218 -0
  114. data/lib/carray/histogram.rb +16 -16
  115. data/lib/carray/inspect.rb +1 -7
  116. data/lib/carray/iterator.rb +4 -3
  117. data/lib/carray/lazy.rb +125 -73
  118. data/lib/carray/meld_reduce.rb +2 -2
  119. data/lib/carray/methods/align_addr.rb +1 -1
  120. data/lib/carray/methods/composition.rb +1 -1
  121. data/lib/carray/methods/is_in.rb +12 -12
  122. data/lib/carray/methods/locate_addr.rb +6 -1
  123. data/lib/carray/methods/mask_duplicates.rb +1 -1
  124. data/lib/carray/methods/meshgrid.rb +4 -5
  125. data/lib/carray/methods/mode.rb +2 -2
  126. data/lib/carray/methods/nunique.rb +1 -1
  127. data/lib/carray/methods/snap.rb +7 -2
  128. data/lib/carray/methods/unique.rb +3 -3
  129. data/lib/carray/methods/value_counts.rb +2 -2
  130. data/lib/carray/runtime.rb +0 -19
  131. data/lib/carray/slab_iterator.rb +20 -7
  132. data/lib/carray/string_operation_extension.rb +5 -5
  133. data/lib/carray/time.rb +890 -491
  134. data/lib/carray/window_iterator.rb +285 -13
  135. data/lib/carray.rb +5 -5
  136. data/yard-stubs/ca_obj_array.rb +385 -0
  137. data/yard-stubs/ca_obj_bitarray.rb +38 -0
  138. data/yard-stubs/ca_obj_bitfield.rb +43 -0
  139. data/yard-stubs/ca_obj_block.rb +73 -0
  140. data/yard-stubs/ca_obj_byte_swap.rb +56 -0
  141. data/yard-stubs/ca_obj_fake.rb +31 -0
  142. data/yard-stubs/ca_obj_farray.rb +32 -0
  143. data/yard-stubs/ca_obj_field.rb +45 -0
  144. data/yard-stubs/ca_obj_grid.rb +35 -0
  145. data/yard-stubs/ca_obj_refer.rb +72 -0
  146. data/yard-stubs/ca_obj_roll.rb +45 -0
  147. data/yard-stubs/ca_obj_shift.rb +43 -0
  148. data/yard-stubs/ca_obj_stride.rb +181 -0
  149. data/yard-stubs/ca_obj_tile.rb +29 -0
  150. data/yard-stubs/ca_obj_transpose.rb +40 -0
  151. data/yard-stubs/ca_obj_window.rb +49 -0
  152. data/yard-stubs/carray_access.rb +131 -0
  153. data/yard-stubs/carray_attribute.rb +246 -0
  154. data/yard-stubs/carray_broadcast.rb +37 -0
  155. data/yard-stubs/carray_cast.rb +489 -0
  156. data/yard-stubs/carray_class.rb +65 -0
  157. data/yard-stubs/carray_conversion.rb +76 -0
  158. data/yard-stubs/carray_copy.rb +79 -0
  159. data/yard-stubs/carray_core.rb +114 -0
  160. data/yard-stubs/carray_count.rb +79 -0
  161. data/yard-stubs/carray_element.rb +108 -0
  162. data/yard-stubs/carray_generate.rb +66 -0
  163. data/yard-stubs/carray_lazy.rb +23 -0
  164. data/yard-stubs/carray_loop.rb +140 -0
  165. data/yard-stubs/carray_mask.rb +259 -0
  166. data/yard-stubs/carray_math.rb +132 -0
  167. data/yard-stubs/carray_mathfunc.rb +45 -0
  168. data/yard-stubs/carray_median_percentile.rb +89 -0
  169. data/yard-stubs/carray_memory_view.rb +163 -0
  170. data/yard-stubs/carray_order.rb +312 -0
  171. data/yard-stubs/carray_random.rb +89 -0
  172. data/yard-stubs/carray_scatter.rb +106 -0
  173. data/yard-stubs/carray_slab.rb +57 -0
  174. data/yard-stubs/carray_sort.rb +163 -0
  175. data/yard-stubs/carray_test.rb +85 -0
  176. data/yard-stubs/carray_undef.rb +64 -0
  177. data/yard-stubs/carray_utils.rb +97 -0
  178. data/yard-stubs/ruby_carray.rb +193 -0
  179. metadata +59 -10
  180. data/ext/ca_obj_unbound_repeat.c +0 -496
data/ext/ca_axis_group.c CHANGED
@@ -8,7 +8,7 @@
8
8
  reduce dispatch, GroupLabels) is wired in ca_group_iter.c (the `[]` gate and
9
9
  the iterator) + `axis_group` (lib/carray/axis_group.rb). Both kernels are
10
10
  written in the general form: several group axes + rank-N categorical (an N-D
11
- codes map) via a per-slab-element composite code, native dtype dispatch,
11
+ codes map) via a per-slab-element composite code, native data type dispatch,
12
12
  mask support. They take pre-built code bundles, so they are independent of
13
13
  how the classifier is constructed.
14
14
 
@@ -40,7 +40,8 @@
40
40
  enum {
41
41
  GR_SUM = 0, GR_PROD, GR_MEAN, GR_MIN, GR_MAX,
42
42
  GR_VARIANCE, GR_STDDEV, GR_COUNT, GR_ALL, GR_ANY,
43
- GR_VARIANCEP, GR_STDDEVP, GR_MINADDR, GR_MAXADDR
43
+ GR_VARIANCEP, GR_STDDEVP, GR_MINADDR, GR_MAXADDR,
44
+ GR_ACCUM
44
45
  };
45
46
 
46
47
  static int
@@ -48,6 +49,7 @@ group_op_code (VALUE vop)
48
49
  {
49
50
  ID id = SYM2ID(vop);
50
51
  if ( id == rb_intern("sum") ) return GR_SUM;
52
+ else if ( id == rb_intern("accumulate") ) return GR_ACCUM;
51
53
  else if ( id == rb_intern("prod") ) return GR_PROD;
52
54
  else if ( id == rb_intern("mean") ) return GR_MEAN;
53
55
  else if ( id == rb_intern("min") ) return GR_MIN;
@@ -72,14 +74,14 @@ group_op_code (VALUE vop)
72
74
  group code, applies the mask, and folds the value into whichever
73
75
  accumulator buffers are non-NULL. Accumulation is always in `double` (or
74
76
  counts in ca_size_t), so the op-finalisation below is type-agnostic — only
75
- the LOAD is monomorphised per dtype, keeping the inner loop autovectorisable
77
+ the LOAD is monomorphised per data type, keeping the inner loop autovectorisable
76
78
  while avoiding a forced float64 materialise of the (large) source. */
77
79
 
78
80
  /* GROUP_WALK(T, ACCUM): one CA_FOR_EACH_SLAB pass. For each slab element it
79
81
  computes the composite group code, applies the mask, then runs ACCUM with
80
82
  `v` (the element, widened to double) and `o` (the output flat index =
81
83
  code * band + band_flat) in scope. ACCUM is the only per-op-varying part,
82
- so the dtype is monomorphised once per type while sum / mean / variance /
84
+ so the data type is monomorphised once per type while sum / mean / variance /
83
85
  ... reuse the same walk. */
84
86
  /* Sentinel o-code for a slab element whose composite code is out of range
85
87
  (excluded categorical), stored in the precomputed plan below. */
@@ -171,7 +173,7 @@ group_op_code (VALUE vop)
171
173
  if ( gw_gaddr ) xfree(gw_gaddr); \
172
174
  } while (0)
173
175
 
174
- /* Run one walk over every supported native dtype. Dispatched on the source
176
+ /* Run one walk over every supported native data type. Dispatched on the source
175
177
  data_type so the inner loop stays monomorphic (no forced float64 cast). */
176
178
  #define GROUP_DISPATCH(ACCUM) \
177
179
  switch ( ca->data_type ) { \
@@ -189,6 +191,19 @@ group_op_code (VALUE vop)
189
191
  default: break; \
190
192
  }
191
193
 
194
+ /* `accumulate` is the one op that folds in the SOURCE's own type instead of in
195
+ double, so it wraps at that width exactly as CArray#accumulate does. It
196
+ folds straight into the (zeroed) output — whose data type is the source's —
197
+ so an empty group already holds the additive identity 0 and needs no mask.
198
+ The type cannot travel inside a GROUP_WALK ACCUM argument (an argument's own
199
+ tokens are not substituted for the macro's parameters), so the walk below
200
+ names its type twice. */
201
+ #define GACC_ADD(T) ( ((T *) co->ptr)[o] += *(T *)(p + gw_doff[e]) )
202
+ /* A boolean accumulate is XOR parity, matching the core: the result stays
203
+ boolean, so a second `true` has nowhere to carry into. */
204
+ #define GACC_XOR \
205
+ ( ((boolean8_t *) co->ptr)[o] ^= (*(boolean8_t *)(p + gw_doff[e]) ? 1 : 0) )
206
+
192
207
  /* __axis_group_reduce__(group_axes, bundles, op) — group-reduces self along
193
208
  * the union of `group_axes` (ascending source-axis indices = the slab) into
194
209
  * composite groups described by `bundles`, preserving the band (= non-grouped)
@@ -358,7 +373,7 @@ rb_ca_axis_group_reduce (VALUE self, VALUE vgaxes, VALUE vbundles, VALUE vop)
358
373
  }
359
374
  ca_size_t nout = K_total * band;
360
375
 
361
- /* --- supported dtype gate (before any allocation) --- */
376
+ /* --- supported data type gate (before any allocation) --- */
362
377
  ca = src;
363
378
  switch ( src->data_type ) {
364
379
  case CA_BOOLEAN: case CA_INT8: case CA_UINT8: case CA_INT16: case CA_UINT16:
@@ -371,11 +386,12 @@ rb_ca_axis_group_reduce (VALUE self, VALUE vgaxes, VALUE vbundles, VALUE vop)
371
386
  src->data_type);
372
387
  }
373
388
 
374
- /* output dtype per op */
389
+ /* output data type per op */
375
390
  int8_t out_dt = CA_FLOAT64;
376
391
  if ( op == GR_COUNT ) out_dt = CA_INT64;
377
392
  else if ( op == GR_MINADDR || op == GR_MAXADDR ) out_dt = CA_INT64;
378
393
  else if ( op == GR_ALL || op == GR_ANY ) out_dt = CA_BOOLEAN;
394
+ else if ( op == GR_ACCUM ) out_dt = src->data_type;
379
395
  VALUE vout = rb_carray_new(out_dt, ondim, odim, 0, NULL);
380
396
  GetCArray(vout, co);
381
397
 
@@ -432,7 +448,7 @@ rb_ca_axis_group_reduce (VALUE self, VALUE vgaxes, VALUE vbundles, VALUE vop)
432
448
  }
433
449
  }
434
450
 
435
- /* --- compute pass(es), native dtype dispatch, no forced float64 cast ---
451
+ /* --- compute pass(es), native data type dispatch, no forced float64 cast ---
436
452
  variance / stddev use a centred two-pass (= matches CArray's own
437
453
  variance, avoids the one-pass sumsq cancellation that breaks ε-close
438
454
  for small near-constant groups). Pass 1 fills sum + cnt; sum is then
@@ -464,6 +480,23 @@ rb_ca_axis_group_reduce (VALUE self, VALUE vgaxes, VALUE vbundles, VALUE vop)
464
480
  }
465
481
  );
466
482
  }
483
+ else if ( op == GR_ACCUM ) {
484
+ MEMZERO(co->ptr, char, (size_t) nout * co->bytes);
485
+ switch ( ca->data_type ) {
486
+ case CA_BOOLEAN: GROUP_WALK(boolean8_t, GACC_XOR); break;
487
+ case CA_INT8: GROUP_WALK(int8_t, GACC_ADD(int8_t)); break;
488
+ case CA_UINT8: GROUP_WALK(uint8_t, GACC_ADD(uint8_t)); break;
489
+ case CA_INT16: GROUP_WALK(int16_t, GACC_ADD(int16_t)); break;
490
+ case CA_UINT16: GROUP_WALK(uint16_t, GACC_ADD(uint16_t)); break;
491
+ case CA_INT32: GROUP_WALK(int32_t, GACC_ADD(int32_t)); break;
492
+ case CA_UINT32: GROUP_WALK(uint32_t, GACC_ADD(uint32_t)); break;
493
+ case CA_INT64: GROUP_WALK(int64_t, GACC_ADD(int64_t)); break;
494
+ case CA_UINT64: GROUP_WALK(uint64_t, GACC_ADD(uint64_t)); break;
495
+ case CA_FLOAT32: GROUP_WALK(float, GACC_ADD(float)); break;
496
+ case CA_FLOAT64: GROUP_WALK(double, GACC_ADD(double)); break;
497
+ default: break;
498
+ }
499
+ }
467
500
  else {
468
501
  GROUP_DISPATCH(
469
502
  cnt[o] += 1;
@@ -502,6 +535,9 @@ rb_ca_axis_group_reduce (VALUE self, VALUE vgaxes, VALUE vbundles, VALUE vop)
502
535
  else out[o] = addr[o];
503
536
  }
504
537
  }
538
+ else if ( op == GR_ACCUM ) {
539
+ /* already folded in place, in the source's own type; empty groups hold 0 */
540
+ }
505
541
  else if ( op == GR_ALL ) {
506
542
  boolean8_t *out = (boolean8_t *) co->ptr; /* empty -> true (vacuous) */
507
543
  for ( ca_size_t o = 0; o < nout; o++ )
@@ -722,10 +758,10 @@ group_scan_build_plan (ca_iter_state *st, boolean8_t *m,
722
758
  }
723
759
 
724
760
  /* GROUP_SCAN_EXTREMUM_WALK(T, CMP): running extremum (cummax / cummin). The
725
- extremum keeps the source dtype (its magnitude never grows), so it holds a
761
+ extremum keeps the source data type (its magnitude never grows), so it holds a
726
762
  native T accumulator, not a widened double. The first member of a group
727
763
  emits its own value: a per-group `seen` byte initialises the accumulator
728
- lazily on first hit — no sentinel like HUGE_VAL, which an integer dtype could
764
+ lazily on first hit — no sentinel like HUGE_VAL, which an integer data type could
729
765
  not represent. CMP is > for max, < for min: a later member replaces the
730
766
  running extremum when `rv CMP acc`. A cell masked within its group holds the
731
767
  current extremum once a member has been seen (output NOT masked, like sum);
@@ -863,18 +899,18 @@ group_scan_op_code (VALUE vop)
863
899
  * its group up to and including that cell, in row-major position order along
864
900
  * the grouped axes (per band).
865
901
 
866
- op / output dtype:
902
+ op / output data type:
867
903
  :cumsum -> float64, inclusive within-group running sum.
868
904
  :cumprod -> float64, inclusive within-group running product (init 1.0;
869
905
  float64 like cumsum since the product grows).
870
- :cummax -> source dtype, running within-group maximum (extrema do not
871
- grow magnitude, so the dtype is preserved; int stays int).
872
- :cummin -> source dtype, running within-group minimum.
906
+ :cummax -> source data type, running within-group maximum (extrema do not
907
+ grow magnitude, so the data type is preserved; int stays int).
908
+ :cummin -> source data type, running within-group minimum.
873
909
  :cumcount -> int64, 1-based within-group running count of present cells
874
910
  (matching the core CArray#cumcount): the first present member
875
911
  of a group emits 1, the next 2, ...
876
912
  cumsum / cumprod keep float64 (matching the reduce siblings sum / prod);
877
- integer-preserving sum / prod is a deliberate non-goal (overflow / dtype
913
+ integer-preserving sum / prod is a deliberate non-goal (overflow / data type
878
914
  consistency), as on the reduce side. A CA_OBJECT source emits a CA_OBJECT
879
915
  result for cumsum / cumprod / cummax / cummin (cumcount stays int64).
880
916
 
@@ -1011,7 +1047,7 @@ rb_ca_axis_group_scan (VALUE self, VALUE vgaxes, VALUE vbundles, VALUE vop)
1011
1047
 
1012
1048
  ca_size_t band = (group_prod > 0) ? (src->elements / group_prod) : 0;
1013
1049
 
1014
- /* --- supported dtype gate (CA_OBJECT handled by its own lane below) --- */
1050
+ /* --- supported data type gate (CA_OBJECT handled by its own lane below) --- */
1015
1051
  ca = src;
1016
1052
  switch ( src->data_type ) {
1017
1053
  case CA_BOOLEAN: case CA_INT8: case CA_UINT8: case CA_INT16: case CA_UINT16:
@@ -1024,8 +1060,8 @@ rb_ca_axis_group_scan (VALUE self, VALUE vgaxes, VALUE vbundles, VALUE vop)
1024
1060
  src->data_type);
1025
1061
  }
1026
1062
 
1027
- /* --- output dtype per op: cumcount int64; cumsum / cumprod float64 (object
1028
- source -> object); cummax / cummin preserve the source dtype (object ->
1063
+ /* --- output data type per op: cumcount int64; cumsum / cumprod float64 (object
1064
+ source -> object); cummax / cummin preserve the source data type (object ->
1029
1065
  object). --- */
1030
1066
  int8_t out_dt;
1031
1067
  if ( op == GS_CUMCOUNT ) { out_dt = CA_INT64; }
@@ -1153,7 +1189,7 @@ rb_ca_axis_group_scan (VALUE self, VALUE vgaxes, VALUE vbundles, VALUE vop)
1153
1189
  }
1154
1190
  }
1155
1191
  }
1156
- else { /* native dtype dispatch */
1192
+ else { /* native data type dispatch */
1157
1193
  switch ( op ) {
1158
1194
  case GS_CUMSUM: {
1159
1195
  double *outd = (double *) co->ptr;
@@ -14,9 +14,9 @@
14
14
  operands arrive at the common type. Two ops break the uniform rule:
15
15
 
16
16
  QUO -- only has a CA_OBJECT kernel; both operands promoted to
17
- CA_OBJECT regardless of input dtypes
17
+ CA_OBJECT regardless of input data types
18
18
  IPOWER -- heterogeneous (left: float/cmplx preserved, right: int64);
19
- output preserves left dtype. See ca_op_ipower.c.
19
+ output preserves left data type. See ca_op_ipower.c.
20
20
 
21
21
  ---------------------------------------------------------------------------- */
22
22
 
@@ -58,7 +58,6 @@ ca_binop_kernel_lookup_vv (uint16_t op_id, int8_t common_dt)
58
58
  case CA_BINOP_AND: return ca_binop_and [common_dt];
59
59
  case CA_BINOP_OR: return ca_binop_or [common_dt];
60
60
  case CA_BINOP_XOR: return ca_binop_xor [common_dt];
61
- case CA_BINOP_REMINDER: return ca_binop_reminder [common_dt];
62
61
  default: return NULL;
63
62
  }
64
63
  }
@@ -107,8 +106,8 @@ ca_binop_kernel_input_data_types (uint16_t op_id, int8_t l_dt, int8_t r_dt,
107
106
  int
108
107
  ca_binop_is_trapping (uint16_t op_id, int8_t common_dt)
109
108
  {
110
- /* Integer DIV / MOD / QUO can SIGFPE on a zero divisor. Float DIV
111
- returns NaN/Inf and does NOT trap, so it is not classified
109
+ /* Integer DIV / MOD / QUO / FMOD can SIGFPE on a zero divisor. Float
110
+ DIV returns NaN/Inf and does NOT trap, so it is not classified
112
111
  trapping. */
113
112
  int is_integer = ( common_dt >= CA_INT8 && common_dt <= CA_UINT64 );
114
113
  if ( ! is_integer ) return 0;
@@ -117,7 +116,7 @@ ca_binop_is_trapping (uint16_t op_id, int8_t common_dt)
117
116
  case CA_BINOP_DIV:
118
117
  case CA_BINOP_MOD:
119
118
  case CA_BINOP_QUO:
120
- case CA_BINOP_REMINDER:
119
+ case CA_BINOP_FMOD:
121
120
  return 1;
122
121
  default:
123
122
  return 0;
@@ -80,12 +80,6 @@ enum {
80
80
  CA_BINOP_OR = 25,
81
81
  CA_BINOP_XOR = 26,
82
82
 
83
- /* IEEE 754 remainder (round-half-to-even, distinct from MOD which
84
- truncates toward zero via C99 fmod on the float branch). Integer
85
- branch is identical to MOD (`%`). All numeric + object. Traps on
86
- integer zero divisor, like MOD. */
87
- CA_BINOP_REMINDER = 27,
88
-
89
83
  CA_BINOP_COUNT
90
84
  };
91
85
 
@@ -127,7 +121,6 @@ extern ca_binop_func_t ca_binop_minimum [CA_NTYPE];
127
121
  extern ca_binop_func_t ca_binop_and [CA_NTYPE];
128
122
  extern ca_binop_func_t ca_binop_or [CA_NTYPE];
129
123
  extern ca_binop_func_t ca_binop_xor [CA_NTYPE];
130
- extern ca_binop_func_t ca_binop_reminder [CA_NTYPE];
131
124
 
132
125
  /* ------------------------------------------------------------------- */
133
126
  /* Dispatch API */
@@ -21,14 +21,14 @@
21
21
  kernel_iterator macros do not model, so the flat inputs are materialised here:
22
22
  ca_attach aliases a contiguous entity (codes / a contiguous value) and gathers
23
23
  a view. Codes dispatch on their native integer type (no coercion); the value
24
- move is a bytes-wide memcpy (grouped shares the value dtype, so no value-dtype
24
+ move is a bytes-wide memcpy (grouped shares the value data type, so no value-type
25
25
  dispatch is needed).
26
26
 
27
27
  Surface (private): codes.__categorical_scatter__(value, cursor, grouped, k)
28
28
  self = codes (integer, carries the exclusion mask), read flat
29
- value = payload (any dtype, may carry a mask), read flat, same length
29
+ value = payload (any data type, may carry a mask), read flat, same length
30
30
  cursor = int64 length-k segment starts (mutated in place, consumed)
31
- grouped = pre-allocated contiguous entity of the value dtype, length nvalid
31
+ grouped = pre-allocated contiguous entity of the value data type, length nvalid
32
32
  k = number of categories
33
33
  Returns grouped.
34
34
 
@@ -79,7 +79,7 @@ rb_ca_categorical_scatter (VALUE self, VALUE rvalue, VALUE rcursor,
79
79
  rb_raise(rb_eArgError, "__categorical_scatter__: cursor must be int64[k]");
80
80
  }
81
81
  if ( grouped->bytes != bytes ) {
82
- rb_raise(rb_eArgError, "__categorical_scatter__: grouped/value dtype mismatch");
82
+ rb_raise(rb_eArgError, "__categorical_scatter__: grouped/value data type mismatch");
83
83
  }
84
84
 
85
85
  ca_attach(codes);
@@ -127,12 +127,12 @@ rb_ca_categorical_scatter (VALUE self, VALUE rvalue, VALUE rcursor,
127
127
  (empty / all-masked -> sum 0 identity, count 0, min/max masked).
128
128
 
129
129
  Surface (private): grouped.__reduceat_moments__(offsets, counts, sums, mins, maxs)
130
- self = grouped (numeric value dtype, may carry a mask), contiguous entity
130
+ self = grouped (numeric value data type, may carry a mask), contiguous entity
131
131
  offsets = int64[k] segment STARTS; segment c = [offsets[c], offsets[c+1]),
132
132
  the last ends at grouped.elements
133
133
  counts = int64[k] output: present (non-masked) cells per segment
134
134
  sums = float64[k] output: sum per segment (0 for empty, unmasked)
135
- mins/maxs = value-dtype[k] output: min / max per segment; the kernel masks
135
+ mins/maxs = value-type[k] output: min / max per segment; the kernel masks
136
136
  the empty/all-masked segments (no value to report)
137
137
  Derived on the Ruby side: mean = sum/count, count_masked = sizes - count, etc.
138
138
 
@@ -190,7 +190,7 @@ rb_ca_reduceat_moments (VALUE self, VALUE roffsets, VALUE rcounts,
190
190
  if ( counts->elements != k || sums->elements != k ||
191
191
  mins->elements != k || maxs->elements != k ||
192
192
  mins->data_type != grouped->data_type || maxs->data_type != grouped->data_type ) {
193
- rb_raise(rb_eArgError, "__reduceat_moments__: output shape/dtype mismatch");
193
+ rb_raise(rb_eArgError, "__reduceat_moments__: output shape/data type mismatch");
194
194
  }
195
195
 
196
196
  offs = (int64_t *) offsets->ptr;
@@ -235,7 +235,7 @@ rb_ca_reduceat_moments (VALUE self, VALUE roffsets, VALUE rcounts,
235
235
  (f = (m-1)*p/100, k = floor(f), lo + (f-k)*(hi-lo)). No per-segment view.
236
236
 
237
237
  Surface (private): grouped.__reduceat_percentile__(offsets, p, out)
238
- self = grouped (numeric value dtype, may carry a mask)
238
+ self = grouped (numeric value data type, may carry a mask)
239
239
  offsets = int64[k] segment STARTS (last ends at grouped.elements)
240
240
  p = percentile in 0..100 (median = 50, quantile(q) = q*100)
241
241
  out = float64[k] output; empty / all-masked segments are masked
@@ -354,7 +354,7 @@ rb_ca_reduceat_percentile (VALUE self, VALUE roffsets, VALUE rp, VALUE rout)
354
354
  SS / (count-1).
355
355
 
356
356
  Surface (private): grouped.__reduceat_variance__(offsets, means, counts, out)
357
- self = grouped (numeric value dtype, may carry a mask)
357
+ self = grouped (numeric value data type, may carry a mask)
358
358
  offsets = int64[k] segment STARTS
359
359
  means = float64[k] per-segment mean (ignored where count < 2)
360
360
  counts = int64[k] per-segment present count
@@ -577,7 +577,7 @@ rb_ca_reduceat_argminmax (VALUE self, VALUE roffsets, VALUE rminidx, VALUE rmaxi
577
577
  }
578
578
 
579
579
  /* __reduceat_all_any__(offsets, all_out, any_out) — per-segment boolean AND / OR
580
- over present cells. Value dtype must be boolean. Empty segment: all -> true,
580
+ over present cells. Value data type must be boolean. Empty segment: all -> true,
581
581
  any -> false. */
582
582
  static VALUE
583
583
  rb_ca_reduceat_all_any (VALUE self, VALUE roffsets, VALUE rall, VALUE rany)
@@ -817,8 +817,8 @@ rb_ca_reduceat_wsum_wmean (VALUE self, VALUE roffsets, VALUE rwg,
817
817
  counts_out = int64, shape [K, ...H.band] (present cells per group)
818
818
  sums_out = float64, shape [K, ...H.band] (per-group sum, 0 for empty)
819
819
 
820
- Sums as float64 mirrors __reduceat_moments__; Ruby side casts to h dtype in
821
- #sum (matches existing empty→0 identity contract). Mins/maxs are in h dtype
820
+ Sums as float64 mirrors __reduceat_moments__; Ruby side casts to h data type in
821
+ #sum (matches existing empty→0 identity contract). Mins/maxs are in h data type
822
822
  (empty group cell → 0 + masked, matching __reduceat_moments__).
823
823
  --------------------------------------------------------------------------- */
824
824
 
@@ -921,7 +921,7 @@ rb_ca_fiber_scatter_moments (VALUE self, VALUE rcodes, VALUE raxis, VALUE rk,
921
921
  }
922
922
  if ( mins->data_type != h->data_type || maxs->data_type != h->data_type ) {
923
923
  rb_raise(rb_eArgError,
924
- "__fiber_scatter_moments__: mins/maxs must match h dtype");
924
+ "__fiber_scatter_moments__: mins/maxs must match h data type");
925
925
  }
926
926
  if ( counts->ndim != h->ndim || sums->ndim != h->ndim ||
927
927
  mins->ndim != h->ndim || maxs->ndim != h->ndim ||
@@ -125,8 +125,9 @@ typedef struct {
125
125
  * inner : per-cell loop within chunk_n
126
126
  *
127
127
  * Memory peak per AC2: INPUT non-alias views materialise into a single
128
- * chunk scratch (~32KB at f64). m0 (= masked form) is full size (=
129
- * n_kernel bytes) for simplicity; macro reads at m0[chunk_off + k].
128
+ * chunk scratch (~32KB at f64). m0 (= masked form) is chunk-sized too and
129
+ * re-gathered per chunk, so the macro reads at m0[k] -- indexed within the
130
+ * chunk, not by the flat cell index.
130
131
  *
131
132
  * For MASKED forms (m / m_in / m_out): when source has no mask, m0 is
132
133
  * NULL and m / m_in == 0 always. m_out writes during INOUT_MASKED are
@@ -162,8 +163,7 @@ typedef struct {
162
163
  && (((x) = *(T *)((_st1).core.base[0] \
163
164
  + __cfem_k * (_st1).core.stride[0])), \
164
165
  ((m) = (_st1).core.m0 \
165
- ? (_st1).core.m0[(_st1).core.chunk_off \
166
- + __cfem_k] \
166
+ ? (_st1).core.m0[__cfem_k] \
167
167
  : (boolean8_t)0), \
168
168
  1); \
169
169
  __cfem_k++ )
@@ -211,16 +211,14 @@ typedef struct {
211
211
  && (((in) = *(T_IN *)((_st2).core.base[0] \
212
212
  + __cfeim_k * (_st2).core.stride[0])), \
213
213
  ((m_in) = (_st2).core.m0 \
214
- ? (_st2).core.m0[(_st2).core.chunk_off \
215
- + __cfeim_k] \
214
+ ? (_st2).core.m0[__cfeim_k] \
216
215
  : (boolean8_t)0), \
217
216
  ((m_out) = (m_in)), \
218
217
  1); \
219
218
  (*(T_OUT *)((_st2).core.base[1] \
220
219
  + __cfeim_k * (_st2).core.stride[1]) = (out)), \
221
220
  (((_st2).core.m0) \
222
- ? ((_st2).core.m0[(_st2).core.chunk_off + __cfeim_k] \
223
- = (m_out)) \
221
+ ? ((_st2).core.m0[__cfeim_k] = (m_out)) \
224
222
  : (boolean8_t)0), \
225
223
  __cfeim_k++ )
226
224
 
data/ext/ca_group_iter.c CHANGED
@@ -292,7 +292,7 @@ Init_ca_group_iter (void)
292
292
  address of the extremum (there is no group-local min_index for the group
293
293
  iterator -- the order-preserving flat address is the meaningful position). */
294
294
  {
295
- const char *ops[] = { "sum", "prod", "mean", "min", "max",
295
+ const char *ops[] = { "sum", "accumulate", "prod", "mean", "min", "max",
296
296
  "variance", "stddev", "variancep", "stddevp",
297
297
  "count", "count_not_masked", "min_addr", "max_addr",
298
298
  "all", "any", NULL };
@@ -186,9 +186,7 @@ ca_iter_classify_source (CArray *src)
186
186
  if ( attach == ca_window_func.attach ) return CA_ITER_SRC_DESCRIPTOR; /* + CAShift */
187
187
 
188
188
  /* Step 9: SRC_ATTACH 5 view. Each view-specific attach materialises
189
- src->ptr via per-element transform; kernel sees a flat contig slab.
190
- CAUnboundRepeat shares ca_stride_func.attach so it was already
191
- classified as SRC_CASTRIDE above (prep doc §2.6). */
189
+ src->ptr via per-element transform; kernel sees a flat contig slab. */
192
190
  if ( attach == ca_fake_func.attach ) return CA_ITER_SRC_ATTACH;
193
191
  if ( attach == ca_byte_swap_func.attach ) return CA_ITER_SRC_ATTACH;
194
192
  if ( attach == ca_bitfield_func.attach ) return CA_ITER_SRC_ATTACH;
@@ -526,6 +524,35 @@ ca_iter_validate_inputs (ca_iter_state *st,
526
524
  return CA_ITER_OK;
527
525
  }
528
526
 
527
+ /* Storage-identical wrapper strip for the kernel-compute entry.
528
+
529
+ A Face layers a semantic identifier and a CALazyMarker layers "read this
530
+ as the leaf of a lazy chain"; neither changes the storage, so routing and
531
+ alias decisions belong to what they wrap. ca_strip_face answers this for
532
+ Face alone and is read that way in ~60 other places, so the marker is
533
+ added here rather than inside it.
534
+
535
+ The marker is NOT classified as a source in its own right. Routing it to
536
+ CA_ITER_SRC_ATTACH would work but would allocate elements * bytes and pull
537
+ the whole array through ca_xfer_all first -- a.lazy.sum(axis: 0) would copy
538
+ what a.sum(axis: 0) aliases. Descending instead lets the parent be
539
+ classified on its own merits, which costs nothing.
540
+
541
+ Unlike Face, nothing re-wraps the result: a reduction over a lazy marker
542
+ yields a plain entity, not a lazy view.
543
+
544
+ CAREFUL: a marker carries CA_FLAG_READ_ONLY and its parent usually does
545
+ not, so the WRITE rejection has to happen against the wrapper. The caller
546
+ does that before stripping. */
547
+ static CArray *
548
+ ca_iter_strip_storage_wrapper (CArray *src)
549
+ {
550
+ while ( src && ( ca_is_face(src) || ca_is_lazy_marker(src) ) ) {
551
+ src = ((CAView *) src)->parent;
552
+ }
553
+ return src;
554
+ }
555
+
529
556
  int
530
557
  ca_iter_state_init_l1 (ca_iter_state *st,
531
558
  struct _CArray *src,
@@ -534,10 +561,19 @@ ca_iter_state_init_l1 (ca_iter_state *st,
534
561
  int8_t naxes,
535
562
  uint32_t flags)
536
563
  {
537
- /* PROPOSAL_CAFACE_PHASE_2 F.2.6: Face strip at entry (= same rationale as
538
- init_l2 above). Strip before validate_inputs. */
539
- if ( ca_is_face(src) ) {
540
- src = ca_strip_face(src);
564
+ /* PROPOSAL_CAFACE_PHASE_2 F.2.6 + PROPOSAL_LAZY_MARKER_LIFT Phase 0:
565
+ storage-identical wrapper strip at entry (= same rationale as init_l2
566
+ below). Strip before validate_inputs.
567
+
568
+ The WRITE rejection is taken against the wrapper, not what it wraps: a
569
+ CALazyMarker is read-only while its parent is not, and stripping first
570
+ would let a destructive kernel through to the parent. validate_inputs
571
+ re-checks the stripped source, which is harmless. */
572
+ if ( (flags & CA_KERNEL_WRITE) && ca_is_readonly(src) ) {
573
+ return CA_ITER_ERR_READONLY;
574
+ }
575
+ if ( ca_is_face(src) || ca_is_lazy_marker(src) ) {
576
+ src = ca_iter_strip_storage_wrapper(src);
541
577
  }
542
578
 
543
579
  int rc = ca_iter_validate_inputs(st, src, policy, flags);
@@ -712,16 +748,27 @@ ca_iter_state_init_l2 (ca_iter_state *st,
712
748
  int8_t naxes,
713
749
  uint32_t flags)
714
750
  {
715
- /* PROPOSAL_CAFACE_PHASE_2 F.2.6 (= MEMO §3.5 kernel_iterator entry strip):
716
- Face only layers a semantic identifier; storage is identical to parent,
717
- so strip at the kernel-compute entry and descend to parent. The Face
718
- identifier is re-wrapped onto the result by the caller's lift hook
719
- (= primary operators / reductions / etc.). Because Face is an identity
720
- mask, routing / alias decisions should be based on parent.
721
- Strip before validate_inputs — `classify_source` would reject Face as
722
- knows-no. */
723
- if ( ca_is_face(src) ) {
724
- src = ca_strip_face(src);
751
+ /* PROPOSAL_CAFACE_PHASE_2 F.2.6 (= MEMO §3.5 kernel_iterator entry strip)
752
+ + PROPOSAL_LAZY_MARKER_LIFT Phase 0.
753
+
754
+ A Face only layers a semantic identifier and a CALazyMarker only layers
755
+ "leaf of a lazy chain"; storage is identical to the parent either way, so
756
+ strip at the kernel-compute entry and descend. Routing and alias
757
+ decisions belong to the parent. Strip before validate_inputs —
758
+ classify_source rejects both as knows-no.
759
+
760
+ The Face identifier is re-wrapped onto the result by the caller's lift
761
+ hook (= primary operators / reductions / etc.). The marker is not:
762
+ a reduction over a lazy marker yields a plain entity.
763
+
764
+ WRITE is rejected against the wrapper, before the strip — a marker is
765
+ read-only while its parent is not. validate_inputs re-checks the
766
+ stripped source, which is harmless. */
767
+ if ( (flags & CA_KERNEL_WRITE) && ca_is_readonly(src) ) {
768
+ return CA_ITER_ERR_READONLY;
769
+ }
770
+ if ( ca_is_face(src) || ca_is_lazy_marker(src) ) {
771
+ src = ca_iter_strip_storage_wrapper(src);
725
772
  }
726
773
 
727
774
  int rc = ca_iter_validate_inputs(st, src, policy, flags);
@@ -1028,8 +1075,8 @@ ca_iter_state_init_l2 (ca_iter_state *st,
1028
1075
  }
1029
1076
  /* slab is all-STRIDE: fall through to B.1.5 / Phase B paths. */
1030
1077
  /* Phase B.1.5: outer SHIFT axis → materialise downgrade. Alias
1031
- can't deliver OOB cells (= need fill_value), so per the
1032
- delivery principle (CLAUDE.md §"deliver" + D1.3 confirmed),
1078
+ can't deliver OOB cells (= need fill_value), and the view-family
1079
+ surface prioritises delivering the cells over avoiding a copy, so
1033
1080
  we materialise the entire view into a row-major scratch
1034
1081
  buffer via ca_axis_dispatch_attach (= same engine as the
1035
1082
  existing SRC_DESCRIPTOR L2 NONE path), then walk it with
@@ -1804,9 +1851,9 @@ ca_iter_state_init_l2 (ca_iter_state *st,
1804
1851
  return CA_ITER_OK;
1805
1852
  }
1806
1853
 
1807
- /* Sub-step 5.3: L2 dispatch over descriptor sources. Per the
1808
- delivery principle (CLAUDE.md "view-family surface prioritises delivery",
1809
- PROPOSAL_T1_STEP5_DESCRIPTOR.md §0): always materialise into a
1854
+ /* Sub-step 5.3: L2 dispatch over descriptor sources. The view-family
1855
+ surface prioritises delivering the cells over avoiding a copy: always
1856
+ materialise into a
1810
1857
  scratch buffer via ca_axis_dispatch_attach and yield a single
1811
1858
  strided slab (stride = bytes). CASelect/CAMapping always reach
1812
1859
  here, CSA/CAGrid/CAWindow/CAShift when INDEX/SHIFT axes are
@@ -4316,6 +4363,5 @@ Init_ca_kernel_iterator (void)
4316
4363
  rb_define_const(rb_cCArray, "T1_ITER_ALIAS_CONTIG", INT2NUM(CA_ITER_ALIAS_CONTIG));
4317
4364
  rb_define_const(rb_cCArray, "T1_ITER_ALIAS_STRIDED", INT2NUM(CA_ITER_ALIAS_STRIDED));
4318
4365
  rb_define_const(rb_cCArray, "T1_ITER_ALIAS_ATTACH", INT2NUM(CA_ITER_ALIAS_ATTACH));
4319
- rb_define_const(rb_cCArray, "T1_ITER_ERR_UNBOUND_SHAPE", INT2NUM(CA_ITER_ERR_UNBOUND_SHAPE));
4320
4366
  #endif /* CARRAY_DEV_BUILD */
4321
4367
  }
@@ -198,15 +198,9 @@ typedef enum {
198
198
  #define CA_ITER_ERR_READONLY 4 /* WRITE on readonly view (CARepeat etc.) */
199
199
  #define CA_ITER_ERR_MASK 5 /* masked source — step 4-5 only, lifted in step 6 */
200
200
  #define CA_ITER_ERR_MASK_NOT_ALLOWED 6 /* NO_MASK flag set on a masked source (step 7) */
201
- #define CA_ITER_ERR_UNBOUND_SHAPE 7 /* CAUnboundRepeat passed before bind() — reserved for
202
- sub-step 9.3 (= used iff unbound CAUbrep smoke shows
203
- unsafe behavior on the existing SRC_CASTRIDE path) */
204
-
205
201
  /* ---- source kind (step 5+, internal routing) ------------------------- */
206
202
  #define CA_ITER_SRC_NONE 0
207
- #define CA_ITER_SRC_CASTRIDE 1 /* entity / CAStride family (step 1-4).
208
- CAUnboundRepeat is also classified
209
- here via ca_ubrep_func = ca_stride_func. */
203
+ #define CA_ITER_SRC_CASTRIDE 1 /* entity / CAStride family (step 1-4) */
210
204
  #define CA_ITER_SRC_DESCRIPTOR 2 /* CSA / CAGrid / CASelect / CAMapping / CAWindow / CAShift (step 5+) */
211
205
  #define CA_ITER_SRC_ATTACH 3 /* CAFake / CAByteSwap / CABitfield /
212
206
  CABitarray / CAReduce — view's own
@@ -1100,8 +1094,7 @@ void ca_iter_state_finish (ca_iter_state *st);
1100
1094
  *
1101
1095
  * PoC (2026-06-12): N=1M f64 sum 906 us -> 116 us (= 7.8x, 68.8 GB/s).
1102
1096
  * Parity: ε-close (relative error < 2e-16 for f64 sum), bit-exact
1103
- * not guaranteed (= reassoc license, documented in CLAUDE.md
1104
- * ε-close policy section, SL.1.5).
1097
+ * not guaranteed (= the SIMD license; see guides/devel/11_kernel_iterator.md).
1105
1098
  *
1106
1099
  * Other state vars referenced inside REDUCE (induction counters
1107
1100
  * like `cnt`, position counters like `idx`) are auto-vectorised
@@ -1217,7 +1210,7 @@ void ca_iter_state_finish (ca_iter_state *st);
1217
1210
  * EXPR(x) is a function-like macro producing the per-element contribution
1218
1211
  * (Pass 1: (double)(x); Pass 2: ((double)(x)-mean)*((double)(x)-mean)).
1219
1212
  * Reassoc across the 8 lanes is the same ε-close license as _PLUS_EX
1220
- * (bit-exact not guaranteed; CLAUDE.md ε-close policy, SL.1.5).
1213
+ * (bit-exact not guaranteed; see guides/devel/11_kernel_iterator.md).
1221
1214
  *
1222
1215
  * Only the no-mask + contig branch is 8-way; masked / non-contig
1223
1216
  * branches stay single-accumulator (not the hot path). Position