carray 3.0.1 → 3.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +824 -0
  3. data/README.md +5 -6
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_dispatch.c +258 -4
  6. data/ext/ca_axis_group.c +368 -184
  7. data/ext/ca_binop_dispatch.c +54 -32
  8. data/ext/ca_binop_dispatch.h +1 -0
  9. data/ext/ca_categorical_iterator.c +90 -41
  10. data/ext/ca_compare.c +8 -2
  11. data/ext/ca_for_buffer.h +21 -68
  12. data/ext/ca_group_iter.c +27 -11
  13. data/ext/ca_kernel_iterator.c +960 -245
  14. data/ext/ca_kernel_iterator.h +352 -181
  15. data/ext/ca_monop_dispatch.c +4 -4
  16. data/ext/ca_monop_dispatch.h +4 -0
  17. data/ext/ca_obj_array.c +194 -42
  18. data/ext/ca_obj_bincmp.c +152 -121
  19. data/ext/ca_obj_binop.c +107 -210
  20. data/ext/ca_obj_bitarray.c +37 -24
  21. data/ext/ca_obj_bitfield.c +95 -62
  22. data/ext/ca_obj_block.c +5 -4
  23. data/ext/ca_obj_byte_swap.c +40 -40
  24. data/ext/ca_obj_const_string.c +180 -50
  25. data/ext/ca_obj_face.c +428 -62
  26. data/ext/ca_obj_face.h +107 -47
  27. data/ext/ca_obj_fake.c +21 -50
  28. data/ext/ca_obj_farray.c +1 -0
  29. data/ext/ca_obj_field.c +16 -3
  30. data/ext/ca_obj_fixlen_string.c +39 -26
  31. data/ext/ca_obj_grid.c +181 -124
  32. data/ext/ca_obj_meld.c +235 -55
  33. data/ext/ca_obj_moncmp.c +61 -61
  34. data/ext/ca_obj_monop.c +125 -107
  35. data/ext/ca_obj_object.c +551 -233
  36. data/ext/ca_obj_record.c +51 -16
  37. data/ext/ca_obj_reduce.c +5 -8
  38. data/ext/ca_obj_refer.c +117 -33
  39. data/ext/ca_obj_remap.c +57 -17
  40. data/ext/ca_obj_repeat.c +21 -5
  41. data/ext/ca_obj_roll.c +115 -11
  42. data/ext/ca_obj_select.c +88 -75
  43. data/ext/ca_obj_select_axis.c +58 -60
  44. data/ext/ca_obj_shift.c +22 -102
  45. data/ext/ca_obj_stack.c +223 -123
  46. data/ext/ca_obj_stride.c +267 -35
  47. data/ext/ca_obj_string.c +25 -22
  48. data/ext/ca_obj_tile.c +32 -16
  49. data/ext/ca_obj_time.c +23 -25
  50. data/ext/ca_obj_timedelta.c +21 -23
  51. data/ext/ca_obj_transpose.c +12 -3
  52. data/ext/ca_obj_triop.c +110 -117
  53. data/ext/ca_obj_window.c +157 -29
  54. data/ext/ca_op_ipower.c +25 -13
  55. data/ext/ca_op_powi.h +57 -30
  56. data/ext/ca_pow5_table.h +656 -0
  57. data/ext/ca_rng_normal.h +42 -0
  58. data/ext/ca_rng_xoshiro256pp.h +105 -0
  59. data/ext/ca_sort_kernels.h +70 -0
  60. data/ext/ca_sweep_engine.c +385 -198
  61. data/ext/ca_sweep_engine.h +31 -23
  62. data/ext/ca_triop_dispatch.c +4 -1
  63. data/ext/ca_triop_dispatch.h +5 -2
  64. data/ext/caframe_csv_reader.c +463 -0
  65. data/ext/carray.h +90 -28
  66. data/ext/carray_access.c +487 -231
  67. data/ext/carray_address_basis.c +731 -0
  68. data/ext/carray_attribute.c +45 -3
  69. data/ext/carray_bincount.c +32 -2
  70. data/ext/carray_broadcast.c +49 -17
  71. data/ext/carray_call_cfunc.c +667 -511
  72. data/ext/carray_cast.c +1556 -247
  73. data/ext/carray_cast_func.rb +34 -21
  74. data/ext/carray_conversion.c +197 -73
  75. data/ext/carray_copy.c +84 -38
  76. data/ext/carray_core.c +591 -119
  77. data/ext/carray_count.c +148 -96
  78. data/ext/carray_element.c +18 -27
  79. data/ext/carray_factorize.c +604 -291
  80. data/ext/carray_generate.c +76 -40
  81. data/ext/carray_histogram.c +152 -94
  82. data/ext/carray_hold.c +2 -1
  83. data/ext/carray_index_classifier.c +63 -96
  84. data/ext/carray_internal.h +400 -1
  85. data/ext/carray_lazy.c +834 -79
  86. data/ext/carray_loop.c +88 -66
  87. data/ext/carray_mask.c +705 -324
  88. data/ext/carray_math_kernel.h +1 -0
  89. data/ext/carray_median_percentile.c +263 -63
  90. data/ext/carray_memory_view.c +254 -248
  91. data/ext/carray_operator.c +1093 -363
  92. data/ext/carray_order.c +133 -69
  93. data/ext/carray_partition.c +102 -56
  94. data/ext/carray_random.c +582 -204
  95. data/ext/carray_scatter.c +308 -65
  96. data/ext/carray_segment.c +257 -0
  97. data/ext/carray_slab.c +183 -70
  98. data/ext/carray_sort.c +235 -240
  99. data/ext/carray_sort_kernel.c +8 -1
  100. data/ext/carray_struct.c +99 -21
  101. data/ext/carray_test.c +167 -64
  102. data/ext/carray_time_text.c +1182 -0
  103. data/ext/carray_utils.c +363 -98
  104. data/ext/extconf.rb +27 -2
  105. data/ext/mk_call_cfunc.rb +103 -118
  106. data/ext/mk_pow5_table.rb +35 -0
  107. data/ext/mkkernel.rb +1349 -851
  108. data/ext/ruby_carray.c +17 -2
  109. data/ext/version.h +4 -4
  110. data/ext/version.rb +4 -2
  111. data/lib/carray/attribute.rb +39 -0
  112. data/lib/carray/autoload_carray.rb +24 -3
  113. data/lib/carray/autoload_method_extension.rb +18 -0
  114. data/lib/carray/axis_group.rb +245 -20
  115. data/lib/carray/basics.rb +95 -25
  116. data/lib/carray/bincount_nd.rb +6 -0
  117. data/lib/carray/block_iterator.rb +200 -54
  118. data/lib/carray/boolean_reduce.rb +21 -29
  119. data/lib/carray/categorical.rb +244 -59
  120. data/lib/carray/categorical_iterator.rb +391 -757
  121. data/lib/carray/complex.rb +36 -29
  122. data/lib/carray/conditional.rb +94 -7
  123. data/lib/carray/const_string.rb +177 -27
  124. data/lib/carray/construct.rb +40 -0
  125. data/lib/carray/data_type_extension.rb +3 -0
  126. data/lib/carray/data_type_limits.rb +91 -0
  127. data/lib/carray/fixlen_string.rb +1 -1
  128. data/lib/carray/frame/concat.rb +157 -6
  129. data/lib/carray/frame/convert.rb +13 -7
  130. data/lib/carray/frame/csv_parser.rb +590 -76
  131. data/lib/carray/frame/describe.rb +79 -0
  132. data/lib/carray/frame/frame.rb +205 -35
  133. data/lib/carray/frame/group.rb +199 -40
  134. data/lib/carray/frame/io.rb +452 -60
  135. data/lib/carray/frame/join.rb +134 -46
  136. data/lib/carray/frame/records.rb +54 -19
  137. data/lib/carray/frame/reshape.rb +448 -0
  138. data/lib/carray/frame/sort.rb +26 -2
  139. data/lib/carray/frame/verbs.rb +726 -56
  140. data/lib/carray/frame.rb +2 -0
  141. data/lib/carray/fuse_source.rb +111 -2
  142. data/lib/carray/fusion.rb +229 -38
  143. data/lib/carray/inspect.rb +71 -20
  144. data/lib/carray/iterator.rb +157 -2
  145. data/lib/carray/lazy.rb +54 -67
  146. data/lib/carray/mask_gap_fill.rb +66 -50
  147. data/lib/carray/math.rb +32 -19
  148. data/lib/carray/meld_reduce.rb +40 -7
  149. data/lib/carray/methods/align_addr.rb +8 -4
  150. data/lib/carray/methods/bincount.rb +2 -2
  151. data/lib/carray/methods/bit_string.rb +14 -3
  152. data/lib/carray/methods/composition.rb +16 -10
  153. data/lib/carray/methods/discovery_along.rb +74 -0
  154. data/lib/carray/methods/factorize.rb +50 -0
  155. data/lib/carray/methods/gather_nd.rb +13 -2
  156. data/lib/carray/methods/insert_block.rb +9 -14
  157. data/lib/carray/methods/is_in.rb +130 -6
  158. data/lib/carray/methods/join.rb +1 -1
  159. data/lib/carray/methods/locate_addr.rb +235 -21
  160. data/lib/carray/methods/mask_duplicates.rb +35 -1
  161. data/lib/carray/methods/mode.rb +18 -1
  162. data/lib/carray/methods/nunique.rb +22 -1
  163. data/lib/carray/methods/repeat.rb +91 -0
  164. data/lib/carray/methods/resize.rb +21 -23
  165. data/lib/carray/methods/snap.rb +34 -12
  166. data/lib/carray/methods/unique.rb +52 -3
  167. data/lib/carray/record_order.rb +208 -0
  168. data/lib/carray/rng.rb +86 -0
  169. data/lib/carray/runtime.rb +14 -11
  170. data/lib/carray/segment_iterator.rb +990 -0
  171. data/lib/carray/serialize.rb +197 -27
  172. data/lib/carray/slab_iterator.rb +109 -40
  173. data/lib/carray/stack.rb +10 -4
  174. data/lib/carray/string_operation_extension.rb +9 -4
  175. data/lib/carray/struct.rb +142 -51
  176. data/lib/carray/struct_builder.rb +110 -14
  177. data/lib/carray/table.rb +2 -2
  178. data/lib/carray/time.rb +535 -105
  179. data/lib/carray/window_iterator.rb +259 -82
  180. data/lib/carray.rb +4 -2
  181. data/yard-stubs/ca_obj_array.rb +4 -4
  182. data/yard-stubs/ca_obj_bitarray.rb +7 -0
  183. data/yard-stubs/ca_obj_bitfield.rb +23 -12
  184. data/yard-stubs/ca_obj_block.rb +2 -7
  185. data/yard-stubs/ca_obj_fake.rb +1 -1
  186. data/yard-stubs/ca_obj_field.rb +1 -1
  187. data/yard-stubs/ca_obj_refer.rb +3 -3
  188. data/yard-stubs/ca_obj_roll.rb +0 -1
  189. data/yard-stubs/ca_obj_stride.rb +10 -7
  190. data/yard-stubs/ca_obj_transpose.rb +1 -1
  191. data/yard-stubs/ca_obj_window.rb +11 -4
  192. data/yard-stubs/carray_access.rb +6 -5
  193. data/yard-stubs/carray_broadcast.rb +2 -2
  194. data/yard-stubs/carray_cast.rb +78 -43
  195. data/yard-stubs/carray_conversion.rb +13 -1
  196. data/yard-stubs/carray_copy.rb +8 -3
  197. data/yard-stubs/carray_core.rb +3 -83
  198. data/yard-stubs/carray_count.rb +16 -2
  199. data/yard-stubs/carray_lazy.rb +210 -1
  200. data/yard-stubs/carray_loop.rb +4 -1
  201. data/yard-stubs/carray_mask.rb +34 -10
  202. data/yard-stubs/carray_math.rb +1499 -4
  203. data/yard-stubs/carray_median_percentile.rb +26 -8
  204. data/yard-stubs/carray_order.rb +43 -82
  205. data/yard-stubs/carray_scatter.rb +10 -3
  206. data/yard-stubs/carray_segment.rb +72 -0
  207. data/yard-stubs/carray_slab.rb +11 -8
  208. data/yard-stubs/carray_sort.rb +50 -8
  209. data/yard-stubs/carray_test.rb +10 -0
  210. data/yard-stubs/carray_utils.rb +0 -26
  211. metadata +19 -19
  212. data/ext/ca_for_each_element.h +0 -239
  213. data/ext/carray_build_flags.h +0 -3
  214. data/ext/carray_config.h +0 -28
  215. data/ext/carray_kernels_bincmp.c +0 -4446
  216. data/ext/carray_kernels_binop.c +0 -11001
  217. data/ext/carray_kernels_init.c +0 -1131
  218. data/ext/carray_kernels_map.c +0 -3467
  219. data/ext/carray_kernels_moncmp.c +0 -2097
  220. data/ext/carray_kernels_monop.c +0 -18313
  221. data/ext/carray_kernels_reduce_aggregate.c +0 -25837
  222. data/ext/carray_kernels_reduce_boolean.c +0 -330
  223. data/ext/carray_kernels_reduce_cumulative.c +0 -14593
  224. data/ext/carray_kernels_reduce_extreme.c +0 -16948
  225. data/ext/carray_kernels_reduce_variance.c +0 -3910
  226. data/ext/carray_kernels_scan.c +0 -3693
  227. data/ext/carray_kernels_search.c +0 -32138
  228. data/ext/carray_kernels_sort.c +0 -10626
  229. data/ext/carray_kernels_triop.c +0 -1392
data/ext/ruby_carray.c CHANGED
@@ -44,6 +44,7 @@ void Init_carray_access ();
44
44
  void Init_carray_index_classifier ();
45
45
  void Init_carray_element ();
46
46
  void Init_carray_scatter (); /* scatter_*! family */
47
+ void Init_carray_segment (); /* segment lengths / offsets / index */
47
48
  void Init_ca_axis_group (); /* devel/MEMO_AXIS_GROUP.md — axis-group reduction kernel */
48
49
  void Init_ca_group_iter (); /* devel/MEMO_AXIS_GROUP.md — axis-group [] surface */
49
50
  void Init_carray_bincount (); /* bincount dedicated kernel */
@@ -89,6 +90,8 @@ void Init_ca_obj_byte_swap ();
89
90
  void Init_ca_face (); /* CAFace abstract base; must precede CATime / CATimedelta / CARecord / CAConstString */
90
91
  void Init_ca_source (); /* CASource abstract base; concrete sources live in C extensions */
91
92
  void Init_ca_obj_time (); /* first numeric Face */
93
+ void Init_carray_time_text ();
94
+ void Init_caframe_csv_reader ();
92
95
  void Init_ca_obj_timedelta (); /* CATime sibling */
93
96
  void Init_ca_obj_record (); /* composite Face for CAStruct backing */
94
97
  void Init_ca_obj_const_string ();/* read-only variable-length string Face */
@@ -121,6 +124,13 @@ void Init_ca_kernel_iterator ();
121
124
 
122
125
  void Init_carray_slab ();
123
126
 
127
+ void Init_carray_address_basis (); /* CArray::AddressBasis: pointer + byte
128
+ strides lent for the length of a block,
129
+ for code that addresses cells itself.
130
+ A runtime facility at the ca_attach
131
+ layer, not a user surface; see
132
+ guides/devel/21_address_basis.md */
133
+
124
134
  void
125
135
  Init_carray_ext (void)
126
136
  {
@@ -162,7 +172,7 @@ Init_carray_ext (void)
162
172
  /* -- Exception class -- */
163
173
 
164
174
  rb_eCADataTypeError =
165
- rb_define_class_under(rb_cCArray, "DataTypeError", rb_eStandardError);
175
+ rb_define_class_under(rb_cCArray, "DataTypeError", rb_eTypeError);
166
176
 
167
177
  /* -- CA module -- namespace for misc utilities related to CArray */
168
178
 
@@ -279,6 +289,7 @@ Init_carray_ext (void)
279
289
  Init_carray_index_classifier();
280
290
  Init_carray_element();
281
291
  Init_carray_scatter();
292
+ Init_carray_segment();
282
293
  Init_ca_axis_group(); /* devel/MEMO_AXIS_GROUP.md — axis-group reduction kernel */
283
294
  Init_carray_bincount();
284
295
  Init_carray_factorize();
@@ -310,7 +321,7 @@ Init_carray_ext (void)
310
321
  Init_ca_obj_refer();
311
322
  Init_ca_obj_stride(); /* CAREFUL: must precede CAStride subclasses
312
323
  (farray / block / transpose / repeat /
313
- unbound_repeat / field). */
324
+ field). */
314
325
  Init_ca_obj_farray();
315
326
  Init_ca_obj_block();
316
327
  Init_ca_obj_select();
@@ -336,6 +347,8 @@ Init_carray_ext (void)
336
347
  (datetime / timedelta / record /
337
348
  const_string). */
338
349
  Init_ca_obj_time();
350
+ Init_carray_time_text();
351
+ Init_caframe_csv_reader();
339
352
  Init_ca_obj_timedelta();
340
353
  Init_ca_obj_record();
341
354
  Init_ca_obj_const_string();
@@ -373,5 +386,7 @@ Init_carray_ext (void)
373
386
  Init_ca_kernel_iterator();
374
387
 
375
388
  Init_carray_slab();
389
+
390
+ Init_carray_address_basis();
376
391
  }
377
392
 
data/ext/version.h CHANGED
@@ -1,6 +1,6 @@
1
- #define CA_VERSION "3.0.1"
2
- #define CA_VERSION_CODE 301
1
+ #define CA_VERSION "3.0.3"
2
+ #define CA_VERSION_CODE 30003
3
3
  #define CA_VERSION_MAJOR 3
4
4
  #define CA_VERSION_MINOR 0
5
- #define CA_VERSION_TEENY 1
6
- #define CA_VERSION_DATE "2026/09/07"
5
+ #define CA_VERSION_TEENY 3
6
+ #define CA_VERSION_DATE "2026/10/13"
data/ext/version.rb CHANGED
@@ -21,8 +21,10 @@ def carray_version
21
21
 
22
22
  ca_version2 = format("%i.%i.%i",
23
23
  ca_version_major, ca_version_minor, ca_version_teeny)
24
- ca_version_code2 =
25
- 100 * ca_version_major + 10*ca_version_minor + ca_version_teeny
24
+ # Two decimal digits for each of minor and teeny, so 3.0.10 (30010)
25
+ # comes after 3.0.9 (30009) and before 3.1.0 (30100).
26
+ ca_version_code2 =
27
+ 10000 * ca_version_major + 100 * ca_version_minor + ca_version_teeny
26
28
 
27
29
  # Allow pre-release suffix like "3.0.0.dev" / "3.0.0.alpha1" while still
28
30
  # validating the numeric MAJOR.MINOR.TEENY prefix.
@@ -4,6 +4,7 @@
4
4
  # Surface:
5
5
  # ca.attr(:unit) # => getter; nil if absent
6
6
  # ca.set_attr(:unit, "m/s") # => setter; writes to self's @attr
7
+ # ca.set_attrs(other.attrs) # => sets every key of a Hash at once
7
8
  # ca.attrs # => frozen shallow clone of the full Hash
8
9
  # ca.has_attr? # => any attribute present?
9
10
  # ca.has_attr?(:unit) # => specific key present?
@@ -27,6 +28,14 @@
27
28
  # `attrs` returns the effective merged Hash (deeper writes
28
29
  # override shallower ones), frozen.
29
30
  # Setter side always writes to self's @attr.
31
+ # The walk stops at a lazy operation view (CAMonOp / CABinOp /
32
+ # CATriOp / CAMonCmp / CABinCmp): its cells are computed, not the
33
+ # parent's cells, so the parent's attributes do not describe them.
34
+ # `a.lazy` itself (CALazyMarker) is a view of `a` and walks through.
35
+ #
36
+ # Copies: `copy` and `to_type` give the new array the attributes the
37
+ # source shows, as its own Hash. Arithmetic and reductions do not
38
+ # carry them -- the result is a different quantity.
30
39
 
31
40
  class CArray
32
41
 
@@ -61,6 +70,30 @@ class CArray
61
70
  (@attr ||= {})[attr_normalize_key(key)] = attr_coerce_value(value)
62
71
  end
63
72
 
73
+ # @overload set_attrs(hash)
74
+ # Sets every key of `hash` on `self`, as {#set_attr} does one key.
75
+ # Keys already on `self` and not in `hash` are kept. Every value is
76
+ # validated before any is written, so a rejected value leaves `self`
77
+ # unchanged. `b.set_attrs(a.attrs)` gives `b` the attributes `a`
78
+ # shows.
79
+ # @param hash [Hash{Symbol, String => Object}] attributes to set.
80
+ # @return [CArray] self.
81
+ # @raise [TypeError] when a key or a value is not accepted.
82
+ def set_attrs (hash)
83
+ unless hash.is_a?(Hash)
84
+ raise TypeError, "set_attrs takes a Hash (got #{hash.class})"
85
+ end
86
+ entries = hash.map do |key, value|
87
+ attr_validate_value(value)
88
+ [attr_normalize_key(key), attr_coerce_value(value)]
89
+ end
90
+ unless entries.empty?
91
+ h = (@attr ||= {})
92
+ entries.each { |k, v| h[k] = v }
93
+ end
94
+ self
95
+ end
96
+
64
97
  # @overload attrs
65
98
  # Returns a frozen shallow Hash of all attributes visible on
66
99
  # `self`, merged along the parent chain (deeper writes shadow
@@ -114,6 +147,7 @@ class CArray
114
147
  h = ca.instance_variable_get(:@attr)
115
148
  yield(h) if h
116
149
  end
150
+ return nil if attr_computing_view?(ca)
117
151
  parent = ca.respond_to?(:parent) ? ca.parent : nil
118
152
  return nil if parent.nil? || parent.equal?(ca)
119
153
  ca = parent
@@ -121,6 +155,11 @@ class CArray
121
155
  nil
122
156
  end
123
157
 
158
+ def attr_computing_view? (ca)
159
+ ca.is_a?(CAMonOp) || ca.is_a?(CABinOp) || ca.is_a?(CATriOp) ||
160
+ ca.is_a?(CAMonCmp) || ca.is_a?(CABinCmp)
161
+ end
162
+
124
163
  def attr_normalize_key (key)
125
164
  case key
126
165
  when String then key
@@ -30,8 +30,9 @@
30
30
 
31
31
  class CArray
32
32
  inspect_rb = "carray/inspect"
33
- autoload_method "inspect", inspect_rb
34
- autoload_method "source_code", inspect_rb
33
+ autoload_method "inspect", inspect_rb
34
+ autoload_method "inspect_full", inspect_rb
35
+ autoload_method "source_code", inspect_rb
35
36
  end
36
37
 
37
38
  # ---- Serializer / Marshal / dump / load ------------------------------------
@@ -42,11 +43,27 @@ class CArray
42
43
  autoload_method "self.save", serialize_rb
43
44
  autoload_method "self.load", serialize_rb
44
45
  autoload_method "self.dump", serialize_rb
45
- autoload_method "self.load_from_file", serialize_rb
46
46
  autoload_method "marshal_dump", serialize_rb
47
47
  autoload_method "marshal_load", serialize_rb
48
48
  end
49
49
 
50
+ # A view, a wrap and a scalar are marshalled through _dump / self._load,
51
+ # which return a new array: with marshal_dump Marshal records the class and
52
+ # loads into an empty instance of it, which the plain array the payload
53
+ # holds cannot become. A record array keeps marshal_dump, under which it
54
+ # already round-trips with its class.
55
+ [CAView, CAWrap, CScalar].each do |klass|
56
+ klass.class_eval do
57
+ undef_method :marshal_dump
58
+ autoload_method "_dump", "carray/serialize"
59
+ autoload_method "self._load", "carray/serialize"
60
+ end
61
+ end
62
+
63
+ class CARecord
64
+ autoload_method "marshal_dump", "carray/serialize"
65
+ end
66
+
50
67
  # ---- Arrow tensor IPC (interop) --------------------------------------------
51
68
 
52
69
  class CArray
@@ -126,10 +143,12 @@ class CArray
126
143
  categorical_iterator_rb = "carray/categorical_iterator"
127
144
  autoload_method "group_by_category", categorical_iterator_rb
128
145
  autoload_method "group_by_run", categorical_iterator_rb
146
+ autoload_method "segments", "carray/segment_iterator"
129
147
  end
130
148
 
131
149
  autoload :CACategorical, "carray/categorical"
132
150
  autoload :CACategoricalIterator, "carray/categorical_iterator"
151
+ autoload :CASegmentIterator, "carray/segment_iterator"
133
152
  autoload :AxisGroup, "carray/axis_group"
134
153
  autoload :GroupLabels, "carray/axis_group"
135
154
 
@@ -201,11 +220,13 @@ class CArray
201
220
  autoload_method "self.meshgrid", "carray/methods/meshgrid"
202
221
 
203
222
  autoload_method "bincount", "carray/methods/bincount"
223
+ autoload_method "repeat", "carray/methods/repeat"
204
224
  autoload_method "self.broadcast", "carray/methods/broadcast"
205
225
  autoload_method "gather_nd", "carray/methods/gather_nd"
206
226
  autoload_method "put_nd", "carray/methods/gather_nd"
207
227
  autoload_method "mask_duplicates", "carray/methods/mask_duplicates"
208
228
  autoload_method "unique", "carray/methods/unique"
229
+ autoload_method "factorize", "carray/methods/factorize"
209
230
  autoload_method "is_in", "carray/methods/is_in"
210
231
  autoload_method "intersection", "carray/methods/is_in"
211
232
  autoload_method "difference", "carray/methods/is_in"
@@ -32,6 +32,7 @@ module AutoloadMethodExtension
32
32
  private
33
33
 
34
34
  def autoload_define (target, name, library, original_spec)
35
+ stub = nil
35
36
  target.define_method(name) do |*args, **kwargs, &block|
36
37
  begin
37
38
  require library
@@ -39,7 +40,24 @@ module AutoloadMethodExtension
39
40
  raise "error in autoloading '#{library}' hooked by method " \
40
41
  "'#{original_spec}', check gem installation."
41
42
  end
43
+ # The require is supposed to have replaced this stub with the real
44
+ # definition. If it has not, the method does not exist anywhere, and
45
+ # forwarding would land straight back here and keep doing so until the
46
+ # stack gave out -- naming neither the method nor the library. Say
47
+ # which, once.
48
+ if target.instance_method(name) == stub
49
+ raise NoMethodError,
50
+ "'#{original_spec}' is registered for autoload from " \
51
+ "'#{library}', but that library defines no such method"
52
+ end
42
53
  send(name, *args, **kwargs, &block)
43
54
  end
55
+ stub = target.instance_method(name)
56
+ # The library replaces the stub with a plain `def`, which `ruby -w`
57
+ # reports as a redefinition. Replacing is what the stub is for, so mark
58
+ # it as aliased: Ruby does not warn when the old definition has an alias.
59
+ tmp = :"__autoload_stub_#{name}__"
60
+ target.send(:alias_method, tmp, name)
61
+ target.send(:remove_method, tmp)
44
62
  end
45
63
  end
@@ -213,6 +213,57 @@ class AxisGroup
213
213
  end
214
214
  end
215
215
 
216
+ # The keywords a group reduction takes besides axis:. Checked before the
217
+ # work, so an unknown keyword is refused rather than dropped.
218
+ REDUCE_KEYWORDS = [:axis, :min_count, :fill_value].freeze
219
+
220
+ # all / any take axis: only, as the core ones do.
221
+ LOGICAL_REDUCE_KEYWORDS = [:axis].freeze
222
+
223
+ def self.__check_reduce_keywords__ (op, kw)
224
+ allowed = [:all, :any].include?(op.to_sym) ? LOGICAL_REDUCE_KEYWORDS : REDUCE_KEYWORDS
225
+ unknown = kw.keys - allowed
226
+ unless unknown.empty?
227
+ raise ArgumentError,
228
+ "#{op}: unknown keyword#{unknown.size > 1 ? "s" : ""}: " +
229
+ unknown.map(&:inspect).join(", ")
230
+ end
231
+ CArray.__min_count_need__(op, kw[:min_count], 0)
232
+ nil
233
+ end
234
+
235
+ # A boolean array carrying value's mask: what a count reads, for a value
236
+ # whose payload the group kernel cannot read.
237
+ def self.__count_stand_in__ (value)
238
+ return value if value.boolean? || value.integer? || value.float?
239
+ stand_in = CArray.boolean(*value.shape)
240
+ stand_in.mask = value.mask if value.has_mask?
241
+ stand_in
242
+ end
243
+
244
+ # A count of the mask reads every cell of the group, as the core's does.
245
+ MASK_COUNTS = [:count, :count_not_masked, :count_masked].freeze
246
+
247
+ # min_count: masks a group with fewer contributing cells (present cells,
248
+ # or every cell for a count of the mask); fill_value: then fills the masked
249
+ # cells -- as a core reduction does with each.
250
+ def self.__finish_reduction__ (iter, op, kw, result)
251
+ need = CArray.__min_count_need__(op, kw[:min_count], 0)
252
+ if need > 0
253
+ counts = if MASK_COUNTS.include?(op.to_sym)
254
+ iter.elements(axis: kw[:axis])
255
+ else
256
+ iter.count_not_masked(axis: kw[:axis])
257
+ end
258
+ if result.is_a?(CArray)
259
+ result[counts.lt(need)] = UNDEF
260
+ elsif counts < need
261
+ result = UNDEF
262
+ end
263
+ end
264
+ CArray.__reduce_fill__(result, kw[:fill_value])
265
+ end
266
+
216
267
  # Parse an axis: spec into [has_group, fused_band_slot_positions].
217
268
  def self.parse_axis (axis)
218
269
  has_group = false
@@ -308,7 +359,70 @@ end
308
359
  # / min / max / variance / stddev / variancep / stddevp / count /
309
360
  # count_not_masked / all / any) bind in C to one driver; the rest of the common
310
361
  # iterator surface that composes cheaply from those is added here.
362
+ # CAGroupIterator subclasses CAIterator, but the class itself is created in
363
+ # C, so the reopening below carries no superclass for YARD to read.
364
+ # @!parse class CAGroupIterator < CAIterator; end
311
365
  class CAGroupIterator
366
+ # The shape one value per piece comes back in: a group slot contributes its
367
+ # category count, a band slot its length, in slot order. The base declares
368
+ # these and every other member answers them; this one inherited the readers
369
+ # without anything ever setting the ivars, so it answered nil -- which the
370
+ # base's own rule calls out as the one thing a member must not do ("a clean
371
+ # NotImplementedError, never a wrong answer").
372
+ def shape
373
+ spec.slot_meta.map { |m| m[:kind] == :group ? m[:k] : m[:len] }
374
+ end
375
+ alias dim shape
376
+
377
+ def ndim
378
+ spec.nslots
379
+ end
380
+
381
+ # The C dispatcher every reduction shares takes keywords only, so the
382
+ # value-equality and masked forms the base declares -- count(v), count(UNDEF)
383
+ # -- arrived as "wrong number of arguments", which reads as a method that
384
+ # does not take one rather than one whose engine has no such fold. Both are
385
+ # composed here instead, out of folds the engine does have.
386
+ alias __count_present__ count
387
+
388
+ # @overload count
389
+ # Per-group count of present (non-masked) cells.
390
+ # @overload count(UNDEF)
391
+ # Per-group count of masked cells, as {#count_masked}.
392
+ # @overload count(v)
393
+ # Per-group count of cells equal to `v`.
394
+ # @return [CArray]
395
+ def count (*args, **kw)
396
+ return __count_present__(**kw) if args.empty?
397
+ if args.size > 1
398
+ raise ArgumentError, "wrong number of arguments (given #{args.size}, expected 0..1)"
399
+ end
400
+ v = args.first
401
+ return count_masked(**kw) if v.equal?(UNDEF)
402
+ # eq leaves a masked cell UNDEF, so the sum leaves it out and its
403
+ # min_count: counts the present cells, as the core count(v) does
404
+ self.class.__build__(value.eq(v).int64, spec).sum(**kw).int64
405
+ end
406
+
407
+ alias __all_folded__ all
408
+ alias __any_folded__ any
409
+
410
+ # @overload all
411
+ # Per-group `all` over a boolean payload.
412
+ # @return [CArray]
413
+ def all (**kw)
414
+ boolean_payload!(:all)
415
+ __all_folded__(**kw)
416
+ end
417
+
418
+ # @overload any
419
+ # Per-group `any` over a boolean payload.
420
+ # @return [CArray]
421
+ def any (**kw)
422
+ boolean_payload!(:any)
423
+ __any_folded__(**kw)
424
+ end
425
+
312
426
  # Per-group classified cell count (mask-independent) = count on the
313
427
  # mask-stripped value, so every classified cell is counted regardless of the
314
428
  # value mask (unlike count / count_not_masked, which count present cells).
@@ -318,7 +432,30 @@ class CAGroupIterator
318
432
 
319
433
  # Per-group count of value-masked cells = elements - count_not_masked.
320
434
  def count_masked (**kw)
321
- elements(**kw) - count_not_masked(**kw)
435
+ has_group, = AxisGroup.parse_axis(kw[:axis])
436
+ return value.count_masked(**kw) unless has_group
437
+ AxisGroup.__check_reduce_keywords__(:count_masked, kw)
438
+ plain = { axis: kw[:axis] }
439
+ AxisGroup.__finish_reduction__(self, :count_masked, kw,
440
+ elements(**plain) - count_not_masked(**plain))
441
+ end
442
+
443
+ # The scatter kernel reads an integer or float payload, and answers there
444
+ # as the core does. Any other payload -- boolean, complex, object, a Face --
445
+ # is reduced group by group by the core, through the composite categorical,
446
+ # so the data type, the Face and a refusal are the core's: a boolean's sum
447
+ # is uint64, a time column's minimum a time, its sum refused.
448
+ [:sum, :accumulate, :prod, :mean, :min, :max,
449
+ :variance, :stddev, :variancep, :stddevp].each do |op|
450
+ alias_method :"__#{op}_by_kernel__", op
451
+ define_method(op) do |**kw|
452
+ has_group, = AxisGroup.parse_axis(kw[:axis])
453
+ if ! has_group || kernel_reads_value?
454
+ __send__(:"__#{op}_by_kernel__", **kw)
455
+ else
456
+ reduce_by_core(op, kw)
457
+ end
458
+ end
322
459
  end
323
460
 
324
461
  # Per-group [min, max] pair (matching CArray#minmax).
@@ -362,13 +499,21 @@ class CAGroupIterator
362
499
  # (no present pair) is UNDEF (matching CArray#wmean); a present group whose
363
500
  # weights sum to zero yields NaN/Inf (core's 0/0).
364
501
  def wmean (weights, **kw)
502
+ has_group, = AxisGroup.parse_axis(kw[:axis])
503
+ return value.wmean(weights, **kw) unless has_group
504
+ AxisGroup.__check_reduce_keywords__(:wmean, kw)
505
+ plain = { axis: kw[:axis] }
365
506
  prod = value * weights
366
- num = self.class.__build__(prod, spec).sum(**kw)
367
- den = self.class.__build__(value * 0 + weights, spec).sum(**kw)
368
- cnt = self.class.__build__(prod, spec).count_not_masked(**kw)
369
- out = num / den
370
- out[cnt.eq(0)] = UNDEF # no present pair -> masked (empty group)
371
- out
507
+ num = self.class.__build__(prod, spec).sum(**plain)
508
+ den = self.class.__build__(value * 0 + weights, spec).sum(**plain)
509
+ cnt = self.class.__build__(prod, spec).count_not_masked(**plain)
510
+ # No present pair is UNDEF (an empty group), set on the denominator so
511
+ # the division never meets an empty group's 0 / 0.
512
+ den = den.copy
513
+ den[cnt.eq(0)] = UNDEF
514
+ # min_count: counts the present pairs, as the core wmean does.
515
+ pairs = self.class.__build__(prod, spec)
516
+ AxisGroup.__finish_reduction__(pairs, :wmean, kw, num / den)
372
517
  end
373
518
 
374
519
  # ---- segment scan: within-group running statistics ---------------------
@@ -504,12 +649,13 @@ class CAGroupIterator
504
649
  raise LocalJumpError, "no block given (yield)" unless block
505
650
  ccat, _kd, gslots, bslots, gaxes = composite_layout
506
651
  return value.group_by_category(ccat).map(data_type: data_type, &block) if bslots.empty?
507
- dt = data_type || value.data_type
508
- out = CArray.new(dt, value.shape)
652
+ out = nil
509
653
  each_band_block(ccat, gslots, bslots, gaxes) do |val_idx, _oi, _co, gi|
510
- out[*val_idx] = gi.map(data_type: dt, &block)
654
+ result = gi.map(data_type: data_type, &block)
655
+ out ||= blank_of(result, value.shape)
656
+ out[*val_idx] = result
511
657
  end
512
- out
658
+ out || (data_type ? CArray.new(data_type, value.shape) : blank_of(value, value.shape))
513
659
  end
514
660
 
515
661
  # @overload reduce { |members| ... }
@@ -549,6 +695,18 @@ class CAGroupIterator
549
695
  # (the shape a length-K result reshapes to). This is the classification the C
550
696
  # scatter kernel computes on the fly, materialised once so the order
551
697
  # statistics / iterate / sort_addr can hold each group's members together.
698
+ # Whether a payload folds with all / any is the core's call, not this
699
+ # engine's. The engine counts any non-zero numeric cell as true, which made
700
+ # the group the one member of the family that answered where CArray#all and
701
+ # every sibling refuse -- so `data.all` and `data[g].all(axis: :group)` on
702
+ # the same float array disagreed about whether the question was even
703
+ # askable. Asking a one-cell array of the same data type lets the core's own
704
+ # refusal through, unworded by us.
705
+ def boolean_payload! (op)
706
+ CArray.new(value.data_type, [1], bytes: value.bytes).public_send(op)
707
+ nil
708
+ end
709
+
552
710
  def composite_layout
553
711
  gslots = spec.slot_meta.select { |m| m[:kind] == :group }
554
712
  bslots = spec.slot_meta.select { |m| m[:kind] == :band }
@@ -619,10 +777,71 @@ class CAGroupIterator
619
777
  "axis_group scan: folding a band into a scan (axis: [:group, k]) " \
620
778
  "is not supported; a scan preserves the source shape"
621
779
  end
780
+ return scan_by_core(op) unless kernel_scans?(op)
622
781
  group_axes, bundles, = spec.reduce_plan([])
623
782
  value.__axis_group_scan__(group_axes, bundles, op)
624
783
  end
625
784
 
785
+ # Whether the scatter kernels read this payload: an integer or a float,
786
+ # not a Face. They answer there in the core's data types.
787
+ def kernel_reads_value?
788
+ ! value.face? && (value.integer? || value.float?)
789
+ end
790
+
791
+ # Whether the scan kernel gives the core's answer: it reads an integer,
792
+ # float, boolean or object payload (not a complex, not a Face), and answers cumsum / cumprod in
793
+ # float64 (an object in object), cummax / cummin in the payload's data type
794
+ # and cumcount in int64 -- not the core's answer for a boolean's running sum.
795
+ def kernel_scans? (op)
796
+ return false if value.face? || value.complex?
797
+ return true if value.object? || op == :cumcount
798
+ core = CArray.new(value.data_type, [1], bytes: value.bytes).public_send(op).data_type
799
+ kernel = [:cumsum, :cumprod].include?(op) ? CA_FLOAT64 : value.data_type
800
+ core == kernel
801
+ end
802
+
803
+ # A scan the kernel does not answer as the core does: each group block is
804
+ # grouped through the composite categorical, whose scans are the core's.
805
+ def scan_by_core (op)
806
+ ccat, _kd, gslots, bslots, gaxes = composite_layout
807
+ return value.group_by_category(ccat).public_send(op) if bslots.empty?
808
+ out = nil
809
+ each_band_block(ccat, gslots, bslots, gaxes) do |val_idx, _oi, _co, gi|
810
+ result = gi.public_send(op)
811
+ out ||= blank_of(result, value.shape)
812
+ out[*val_idx] = result
813
+ end
814
+ out || blank_of(value, value.shape)
815
+ end
816
+
817
+ # A reduction of a payload the scatter kernel does not read, group by
818
+ # group through the composite categorical. Folding a band into the
819
+ # statistic needs the kernel.
820
+ def reduce_by_core (op, kw)
821
+ _, fused = AxisGroup.parse_axis(kw[:axis])
822
+ unless fused.empty?
823
+ raise ArgumentError,
824
+ "axis_group #{op}: folding a band into the statistic " \
825
+ "(axis: [:group, k]) is not supported for a #{value.class} of " \
826
+ "data type #{value.data_type_name}"
827
+ end
828
+ AxisGroup.__check_reduce_keywords__(op, kw)
829
+ AxisGroup.__finish_reduction__(self, op, kw, composite_order(op, []))
830
+ end
831
+
832
+ # An array of +shape+ of the same kind as +model+ -- its data type, or its
833
+ # Face -- with every cell UNDEF. A read-only Face cannot be written cell by
834
+ # cell, so its values are held in an object array.
835
+ def blank_of (model, shape)
836
+ out = if model.face? && model.read_only?
837
+ CArray.new(CA_OBJECT, shape)
838
+ else
839
+ model.__send__(:__blank_like__, shape)
840
+ end
841
+ out[] = UNDEF
842
+ out
843
+ end
844
+
626
845
  # Dispatch an order statistic (median / percentile / quantile). No :group ->
627
846
  # plain value reduction; :group -> per-group order statistics via the composite
628
847
  # materialize. Folding a band into an order statistic (axis: [:group, k])
@@ -630,13 +849,19 @@ class CAGroupIterator
630
849
  # operation, and remains a follow-up.
631
850
  def order_stat (op, args, kw)
632
851
  has_group, fused = AxisGroup.parse_axis(kw[:axis])
633
- return value.public_send(op, *args) unless has_group
852
+ return value.public_send(op, *args, **kw) unless has_group
634
853
  unless fused.empty?
635
854
  raise NotImplementedError,
636
855
  "axis_group: folding a band into an order statistic (axis: [:group, k]) " \
637
856
  "is not supported; order statistics gather every member of a group"
638
857
  end
639
- composite_order(op, args)
858
+ AxisGroup.__check_reduce_keywords__(op, kw)
859
+ res = composite_order(op, args)
860
+ if op == :quantile
861
+ res.map { |r| AxisGroup.__finish_reduction__(self, op, kw, r) }
862
+ else
863
+ AxisGroup.__finish_reduction__(self, op, kw, res)
864
+ end
640
865
  end
641
866
 
642
867
  # Per-group order statistic over the composite grouping. A flat grouping (no
@@ -655,15 +880,15 @@ class CAGroupIterator
655
880
  return nout == 1 ? res.reshape(*kdims) : res.map { |r| r.reshape(*kdims) }
656
881
  end
657
882
  out_shape = spec.slot_meta.map { |m| m[:kind] == :group ? m[:k] : m[:len] }
658
- outs = Array.new(nout) { CArray.float64(*out_shape) }
883
+ outs = nil
659
884
  each_band_block(ccat, gslots, bslots, gaxes) do |_vi, out_idx, _co, gi|
660
- result = gi.public_send(op, *args)
661
- if nout == 1
662
- outs[0][*out_idx] = result.reshape(*kdims)
663
- else
664
- result.each_with_index { |r, q| outs[q][*out_idx] = r.reshape(*kdims) }
665
- end
885
+ results = gi.public_send(op, *args)
886
+ results = [results] unless results.is_a?(Array) # quantile answers five
887
+ # each output takes the kind the categorical answered in
888
+ outs ||= results.map { |r| blank_of(r, out_shape) }
889
+ results.each_with_index { |r, q| outs[q][*out_idx] = r.reshape(*kdims) }
666
890
  end
891
+ outs ||= Array.new(nout) { blank_of(value, out_shape) }
667
892
  nout == 1 ? outs[0] : outs
668
893
  end
669
894