carray 3.0.0 → 3.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. checksums.yaml +4 -4
  2. data/.yardopts +2 -1
  3. data/CHANGELOG.md +845 -3
  4. data/{NEWS.md → CHANGELOG.v1.md} +1 -1
  5. data/README.md +33 -33
  6. data/carray.gemspec +12 -8
  7. data/ext/ca_axis_dispatch.c +33 -4
  8. data/ext/ca_axis_group.c +256 -114
  9. data/ext/ca_binop_dispatch.c +5 -6
  10. data/ext/ca_binop_dispatch.h +0 -7
  11. data/ext/ca_categorical_iterator.c +121 -67
  12. data/ext/ca_for_each_element.h +6 -8
  13. data/ext/ca_group_iter.c +1 -1
  14. data/ext/ca_kernel_iterator.c +386 -74
  15. data/ext/ca_kernel_iterator.h +145 -45
  16. data/ext/ca_obj_array.c +62 -20
  17. data/ext/ca_obj_bincmp.c +57 -27
  18. data/ext/ca_obj_binop.c +86 -30
  19. data/ext/ca_obj_block.c +4 -4
  20. data/ext/ca_obj_const_string.c +88 -27
  21. data/ext/ca_obj_face.c +104 -0
  22. data/ext/ca_obj_face.h +48 -1
  23. data/ext/ca_obj_fake.c +11 -0
  24. data/ext/ca_obj_fixlen_string.c +21 -6
  25. data/ext/ca_obj_grid.c +11 -3
  26. data/ext/ca_obj_meld.c +125 -22
  27. data/ext/ca_obj_moncmp.c +33 -14
  28. data/ext/ca_obj_monop.c +19 -7
  29. data/ext/ca_obj_object.c +107 -4
  30. data/ext/ca_obj_record.c +3 -1
  31. data/ext/ca_obj_refer.c +30 -6
  32. data/ext/ca_obj_roll.c +13 -5
  33. data/ext/ca_obj_select.c +49 -34
  34. data/ext/ca_obj_select_axis.c +16 -6
  35. data/ext/ca_obj_shift.c +3 -3
  36. data/ext/ca_obj_stack.c +3 -8
  37. data/ext/ca_obj_stride.c +178 -24
  38. data/ext/ca_obj_string.c +11 -5
  39. data/ext/ca_obj_tile.c +13 -5
  40. data/ext/ca_obj_time.c +3 -1
  41. data/ext/ca_obj_timedelta.c +3 -1
  42. data/ext/ca_obj_transpose.c +1 -1
  43. data/ext/ca_obj_triop.c +62 -23
  44. data/ext/ca_obj_window.c +94 -19
  45. data/ext/ca_op_cmplx64.h +123 -0
  46. data/ext/ca_op_ipower.c +1 -5
  47. data/ext/ca_rng_normal.h +42 -0
  48. data/ext/ca_rng_xoshiro256pp.h +105 -0
  49. data/ext/ca_sort_kernels.h +5 -5
  50. data/ext/ca_sweep_engine.c +362 -155
  51. data/ext/ca_sweep_engine.h +39 -9
  52. data/ext/ca_transform_common.c +7 -0
  53. data/ext/carray.h +148 -132
  54. data/ext/carray_access.c +83 -49
  55. data/ext/carray_address_basis.c +590 -0
  56. data/ext/carray_bincount.c +8 -8
  57. data/ext/carray_broadcast.c +102 -9
  58. data/ext/carray_build_flags.h +3 -0
  59. data/ext/carray_call_cfunc.c +2010 -335
  60. data/ext/carray_call_cfunc.h +153 -0
  61. data/ext/carray_cast.c +184 -72
  62. data/ext/carray_conversion.c +22 -22
  63. data/ext/carray_copy.c +67 -32
  64. data/ext/carray_core.c +140 -13
  65. data/ext/carray_count.c +10 -11
  66. data/ext/carray_element.c +7 -7
  67. data/ext/carray_factorize.c +72 -51
  68. data/ext/carray_hold.c +1 -1
  69. data/ext/carray_index_classifier.c +7 -21
  70. data/ext/carray_internal.h +36 -3
  71. data/ext/carray_kernels_bincmp.c +1 -0
  72. data/ext/carray_kernels_binop.c +320 -298
  73. data/ext/carray_kernels_init.c +1095 -0
  74. data/ext/carray_kernels_map.c +1 -0
  75. data/ext/carray_kernels_moncmp.c +1 -0
  76. data/ext/carray_kernels_monop.c +412 -411
  77. data/ext/carray_kernels_reduce_aggregate.c +256 -87
  78. data/ext/carray_kernels_reduce_boolean.c +1 -0
  79. data/ext/carray_kernels_reduce_cumulative.c +324 -54
  80. data/ext/carray_kernels_reduce_extreme.c +605 -58
  81. data/ext/carray_kernels_reduce_variance.c +1 -0
  82. data/ext/carray_kernels_scan.c +5 -4
  83. data/ext/carray_kernels_search.c +187 -106
  84. data/ext/carray_kernels_sort.c +1 -0
  85. data/ext/carray_kernels_triop.c +1 -0
  86. data/ext/carray_lazy.c +175 -5
  87. data/ext/carray_loop.c +7 -1
  88. data/ext/carray_mask.c +29 -13
  89. data/ext/carray_median_percentile.c +56 -1
  90. data/ext/carray_memory_view.c +46 -21
  91. data/ext/carray_operator.c +50 -53
  92. data/ext/carray_order.c +1 -1
  93. data/ext/carray_partition.c +4 -4
  94. data/ext/carray_random.c +391 -47
  95. data/ext/carray_scatter.c +1 -1
  96. data/ext/carray_slab.c +13 -0
  97. data/ext/carray_sort.c +22 -24
  98. data/ext/carray_sort_kernel.c +10 -10
  99. data/ext/carray_test.c +1 -1
  100. data/ext/extconf.rb +18 -0
  101. data/ext/mk_call_cfunc.rb +323 -145
  102. data/ext/mkkernel.rb +651 -129
  103. data/ext/ruby_carray.c +28 -17
  104. data/ext/version.h +4 -4
  105. data/lib/carray/autoload_carray.rb +24 -3
  106. data/lib/carray/autoload_method_extension.rb +14 -1
  107. data/lib/carray/axis_group.rb +84 -7
  108. data/lib/carray/basics.rb +63 -59
  109. data/lib/carray/bincount_nd.rb +12 -12
  110. data/lib/carray/block_iterator.rb +111 -25
  111. data/lib/carray/categorical.rb +154 -37
  112. data/lib/carray/categorical_iterator.rb +300 -151
  113. data/lib/carray/conditional.rb +14 -14
  114. data/lib/carray/const_string.rb +131 -27
  115. data/lib/carray/construct.rb +61 -0
  116. data/lib/carray/core_extensions.rb +62 -5
  117. data/lib/carray/data_type_extension.rb +21 -1
  118. data/lib/carray/data_type_limits.rb +91 -0
  119. data/lib/carray/fixlen_string.rb +1 -1
  120. data/lib/carray/frame/concat.rb +3 -3
  121. data/lib/carray/frame/convert.rb +1 -1
  122. data/lib/carray/frame/csv_parser.rb +11 -4
  123. data/lib/carray/frame/frame.rb +85 -21
  124. data/lib/carray/frame/group.rb +36 -3
  125. data/lib/carray/frame/io.rb +224 -15
  126. data/lib/carray/frame/records.rb +18 -4
  127. data/lib/carray/frame/sort.rb +1 -1
  128. data/lib/carray/frame/verbs.rb +32 -12
  129. data/lib/carray/fuse_source.rb +123 -0
  130. data/lib/carray/fusion.rb +218 -0
  131. data/lib/carray/histogram.rb +16 -16
  132. data/lib/carray/inspect.rb +43 -16
  133. data/lib/carray/iterator.rb +147 -3
  134. data/lib/carray/lazy.rb +88 -73
  135. data/lib/carray/mask_gap_fill.rb +3 -1
  136. data/lib/carray/meld_reduce.rb +2 -2
  137. data/lib/carray/methods/align_addr.rb +1 -1
  138. data/lib/carray/methods/composition.rb +1 -1
  139. data/lib/carray/methods/discovery_along.rb +74 -0
  140. data/lib/carray/methods/factorize.rb +50 -0
  141. data/lib/carray/methods/is_in.rb +25 -14
  142. data/lib/carray/methods/locate_addr.rb +81 -3
  143. data/lib/carray/methods/mask_duplicates.rb +36 -2
  144. data/lib/carray/methods/meshgrid.rb +4 -5
  145. data/lib/carray/methods/mode.rb +2 -2
  146. data/lib/carray/methods/nunique.rb +23 -2
  147. data/lib/carray/methods/repeat.rb +110 -0
  148. data/lib/carray/methods/snap.rb +7 -2
  149. data/lib/carray/methods/unique.rb +44 -4
  150. data/lib/carray/methods/value_counts.rb +2 -2
  151. data/lib/carray/rng.rb +86 -0
  152. data/lib/carray/runtime.rb +0 -19
  153. data/lib/carray/slab_iterator.rb +77 -19
  154. data/lib/carray/string_operation_extension.rb +10 -6
  155. data/lib/carray/time.rb +906 -491
  156. data/lib/carray/window_iterator.rb +421 -27
  157. data/lib/carray.rb +7 -5
  158. data/yard-stubs/ca_obj_array.rb +385 -0
  159. data/yard-stubs/ca_obj_bitarray.rb +38 -0
  160. data/yard-stubs/ca_obj_bitfield.rb +43 -0
  161. data/yard-stubs/ca_obj_block.rb +68 -0
  162. data/yard-stubs/ca_obj_byte_swap.rb +56 -0
  163. data/yard-stubs/ca_obj_fake.rb +31 -0
  164. data/yard-stubs/ca_obj_farray.rb +32 -0
  165. data/yard-stubs/ca_obj_field.rb +45 -0
  166. data/yard-stubs/ca_obj_grid.rb +35 -0
  167. data/yard-stubs/ca_obj_refer.rb +72 -0
  168. data/yard-stubs/ca_obj_roll.rb +45 -0
  169. data/yard-stubs/ca_obj_shift.rb +43 -0
  170. data/yard-stubs/ca_obj_stride.rb +181 -0
  171. data/yard-stubs/ca_obj_tile.rb +29 -0
  172. data/yard-stubs/ca_obj_transpose.rb +40 -0
  173. data/yard-stubs/ca_obj_window.rb +57 -0
  174. data/yard-stubs/carray_access.rb +131 -0
  175. data/yard-stubs/carray_attribute.rb +246 -0
  176. data/yard-stubs/carray_broadcast.rb +37 -0
  177. data/yard-stubs/carray_cast.rb +489 -0
  178. data/yard-stubs/carray_class.rb +65 -0
  179. data/yard-stubs/carray_conversion.rb +76 -0
  180. data/yard-stubs/carray_copy.rb +79 -0
  181. data/yard-stubs/carray_core.rb +34 -0
  182. data/yard-stubs/carray_count.rb +84 -0
  183. data/yard-stubs/carray_element.rb +108 -0
  184. data/yard-stubs/carray_generate.rb +66 -0
  185. data/yard-stubs/carray_lazy.rb +228 -0
  186. data/yard-stubs/carray_loop.rb +140 -0
  187. data/yard-stubs/carray_mask.rb +259 -0
  188. data/yard-stubs/carray_math.rb +1615 -0
  189. data/yard-stubs/carray_mathfunc.rb +45 -0
  190. data/yard-stubs/carray_median_percentile.rb +103 -0
  191. data/yard-stubs/carray_memory_view.rb +163 -0
  192. data/yard-stubs/carray_order.rb +252 -0
  193. data/yard-stubs/carray_random.rb +89 -0
  194. data/yard-stubs/carray_scatter.rb +106 -0
  195. data/yard-stubs/carray_slab.rb +59 -0
  196. data/yard-stubs/carray_sort.rb +165 -0
  197. data/yard-stubs/carray_test.rb +85 -0
  198. data/yard-stubs/carray_undef.rb +64 -0
  199. data/yard-stubs/carray_utils.rb +97 -0
  200. data/yard-stubs/ruby_carray.rb +193 -0
  201. metadata +67 -10
  202. data/ext/ca_obj_unbound_repeat.c +0 -496
@@ -0,0 +1,218 @@
1
+ # ---------------------------------------------------------------------------
2
+ # Turning a lazy expression into a plan a compiler can read.
3
+ #
4
+ # A lazy view is already a typed, closed expression graph, and the kernels
5
+ # already carry the C that computes each operation (CArray.__kernel_body__).
6
+ # What is missing between them is the reading: which operation each node is,
7
+ # what its mask does, and where the leaves are. That is what a plan holds.
8
+ #
9
+ # Nothing here compiles anything. A plan is plain data, and CArray itself
10
+ # never needs one -- it can always walk the view. What a plan is for is a
11
+ # second evaluator: hand it to one, and the answer must be the same.
12
+ # ---------------------------------------------------------------------------
13
+
14
+ class CArray
15
+
16
+ module Fusion
17
+
18
+ # One value per node, in evaluation order; the last is the result.
19
+ #
20
+ # Op an operation, reading the nodes named in `args`
21
+ # Leaf an array, the `index`-th of plan.leaves
22
+ # Const a scalar written into the expression
23
+ Op = Struct.new(:kind, :name, :data_type, :args, :body, :mask, :trapping)
24
+ Leaf = Struct.new(:index, :data_type, :masked)
25
+ Const = Struct.new(:value, :data_type)
26
+
27
+ Plan = Struct.new(:nodes, :leaves, :data_type, :dim, :masked, :signature)
28
+
29
+ LAZY_CLASSES = [CAMonOp, CABinOp, CATriOp, CAMonCmp, CABinCmp, CALazyMarker]
30
+
31
+ # A lazy node names its operation by an id and the kernels name it by a
32
+ # symbol. These are the same operations, spelled the way each side
33
+ # spells them; the rest are spelled alike.
34
+ BINOP_NAMES = {
35
+ :+ => :add, :- => :sub, :* => :mul,
36
+ :/ => :div, :** => :power, :% => :mod,
37
+ :& => :bit_and_i, :| => :bit_or_i, :^ => :bit_xor_i,
38
+ :<< => :bit_lshift, :>> => :bit_rshift,
39
+ }.freeze
40
+ TRIOP_NAMES = { :__clip_ki__ => :clip }.freeze
41
+
42
+ MONOP_BY_ID = CArray::LAZY_MONOP_OP_IDS.invert.freeze
43
+ BINOP_BY_ID = CArray::LAZY_BINOP_OP_IDS.invert.freeze
44
+ TRIOP_BY_ID = CArray::LAZY_TRIOP_OP_IDS.invert.freeze
45
+
46
+ # Integer division and its relatives raise on a zero divisor, so a cell
47
+ # the mask excludes must not be computed at all -- the divisor there is
48
+ # nobody's business (ca_binop_dispatch.c).
49
+ TRAPPING = %i[div mod quo_i fmod].freeze
50
+ INTEGERS = %i[int8 int16 int32 int64 uint8 uint16 uint32 uint64].freeze
51
+
52
+ class Refused < StandardError; end
53
+
54
+ # ---- who computes a plan --------------------------------------------
55
+ #
56
+ # CArray can always walk the expression, so nothing has to be registered
57
+ # and nothing changes when nothing is. What a registered evaluator adds
58
+ # is a second way to arrive at the same answer; it is asked, and it may
59
+ # decline. The dispatch point stays on CArray's side, which is what
60
+ # keeps the threshold below a decision about CArray's own walk rather
61
+ # than one that moves with whatever is installed.
62
+ #
63
+ # The evaluator itself is held by CArray (see carray/lazy.rb), so that
64
+ # materialising an expression need not reach for this file at all until
65
+ # something has been registered.
66
+
67
+ # Reaching a compiled kernel costs about the same whatever the array's
68
+ # size, and what it buys is the passes the walk would make. Below this
69
+ # the walk is the faster answer. The crossing moves with how wide the
70
+ # expression is -- measured, thirty thousand cells at one operation, six
71
+ # thousand at six -- and this brackets those: a one-operation expression
72
+ # loses a couple of microseconds here, a six-operation one wins ten.
73
+ THRESHOLD = 10_000
74
+
75
+ # Returns the array, or nil where nothing computed it.
76
+ def self.evaluate (view)
77
+ return nil unless askable?(view)
78
+ out = CArray.__alloc_uninit__(view.data_type, view.dim)
79
+ evaluate_into(view, out) ? out : nil
80
+ end
81
+
82
+ # Fills an array the caller already has. Called from the store as well,
83
+ # where making one and copying it over would be most of the work.
84
+ # Returns true when something computed it.
85
+ def self.evaluate_into (view, out)
86
+ evaluator = CArray.expression_evaluator or return false
87
+ return false unless askable?(view)
88
+ plan = plan(view) or return false
89
+ # A marker over an array, or anything else with nothing to compute,
90
+ # is not worth handing over.
91
+ return false unless plan.nodes.any? { |n| n.is_a?(Op) }
92
+ out.mask = 0 if plan.masked && ! out.has_mask?
93
+ evaluator.call(plan, out) ? true : false
94
+ rescue StandardError => error
95
+ CArray.expression_evaluator = nil
96
+ warn "CArray: the registered expression evaluator raised " \
97
+ "(#{error.class}: #{error.message}); expressions will be walked " \
98
+ "from here on"
99
+ false
100
+ end
101
+
102
+ def self.askable? (view)
103
+ ! CArray.expression_evaluator.nil? && view.elements >= THRESHOLD
104
+ end
105
+
106
+ # Returns a Plan, or nil where the expression holds something a plan
107
+ # cannot describe. Refusing is ordinary: the caller walks instead.
108
+ def self.plan (view)
109
+ build(view)
110
+ rescue Refused
111
+ nil
112
+ end
113
+
114
+ def self.build (view)
115
+ raise Refused, "not a lazy expression" unless lazy?(view)
116
+ w = Walk.new
117
+ w.visit(view)
118
+ Plan.new(w.nodes, w.leaves, view.data_type, view.dim,
119
+ w.leaves.any? { |a| a.has_mask? }, w.signature)
120
+ end
121
+
122
+ def self.lazy? (x)
123
+ LAZY_CLASSES.any? { |k| x.is_a?(k) }
124
+ end
125
+
126
+ # ---- the walk -------------------------------------------------------
127
+
128
+ class Walk
129
+ attr_reader :nodes, :leaves, :signature
130
+
131
+ def initialize
132
+ @nodes = []
133
+ @leaves = []
134
+ @seen = {}
135
+ @signature = +""
136
+ end
137
+
138
+ def visit (n)
139
+ @seen[n.object_id] ||= build(n)
140
+ end
141
+
142
+ private
143
+
144
+ def build (n)
145
+ case n
146
+ when CALazyMarker then visit(n.parent)
147
+ when CAMonOp then unary(n)
148
+ when CABinOp then binary(n)
149
+ when CATriOp then ternary(n)
150
+ when CScalar then constant(n)
151
+ when CArray then leaf(n)
152
+ else raise Refused, "#{n.class} in an expression"
153
+ end
154
+ end
155
+
156
+ def unary (n)
157
+ name = spell(MONOP_BY_ID, n.__op_id__, {})
158
+ args = [visit(n.parent)]
159
+ # A view over one array is masked exactly where that array is
160
+ # (ca_obj_monop.c).
161
+ op(:monop, name, n.data_type, args, :pass)
162
+ end
163
+
164
+ def binary (n)
165
+ name = spell(BINOP_BY_ID, n.__op_id__, BINOP_NAMES)
166
+ args = [visit(n.parent), visit(n.__binop_right__)]
167
+ # Boolean `&` and `|` are three-valued: a masked cell whose known
168
+ # side settles the answer comes back unmasked (ca_obj_binop.c).
169
+ rule = if n.data_type == :boolean && name == :bit_or_i then :kleene_or
170
+ elsif n.data_type == :boolean && name == :bit_and_i then :kleene_and
171
+ else :union
172
+ end
173
+ op(:binop, name, n.data_type, args, rule)
174
+ end
175
+
176
+ def ternary (n)
177
+ name = spell(TRIOP_BY_ID, n.__op_id__, TRIOP_NAMES)
178
+ args = [visit(n.parent), visit(n.__triop_op2__), visit(n.__triop_op3__)]
179
+ op(:triop, name, n.data_type, args, :union)
180
+ end
181
+
182
+ def op (kind, name, type, args, mask)
183
+ body = CArray.__kernel_body__(kind, name, type) or
184
+ raise Refused, "#{kind} #{name} has no body at #{type}"
185
+ note(kind.to_s[0], name, type)
186
+ push Op.new(kind, name, type, args, body, mask,
187
+ TRAPPING.include?(name) && INTEGERS.include?(type))
188
+ end
189
+
190
+ def leaf (n)
191
+ note("a", n.data_type, n.has_mask? ? 1 : 0)
192
+ @leaves << n
193
+ push Leaf.new(@leaves.size - 1, n.data_type, n.has_mask?)
194
+ end
195
+
196
+ def constant (n)
197
+ note("k", n[0], n.data_type)
198
+ push Const.new(n[0], n.data_type)
199
+ end
200
+
201
+ def push (node)
202
+ @nodes << node
203
+ @nodes.size - 1
204
+ end
205
+
206
+ def spell (table, id, renames)
207
+ ruby = table[id] or raise Refused, "operation id #{id}"
208
+ renames.fetch(ruby, ruby)
209
+ end
210
+
211
+ # Two expressions of the same shape compute alike, whatever arrays
212
+ # they are over, so a consumer can keep one compiled kernel for both.
213
+ def note (*parts)
214
+ @signature << parts.join(":") << ";"
215
+ end
216
+ end
217
+ end
218
+ end
@@ -94,8 +94,8 @@
94
94
  # Each sample contributes `w[i]` instead of 1 to its target cell.
95
95
  #
96
96
  # * **weights.shape** = chunk.shape minus the channel axis (= fiber + (A,)).
97
- # * **dtype** is locked at construction (= the entry method's `weights:` kwarg
98
- # fixes weighted vs unweighted; subsequent adds must match). Counts dtype:
97
+ # * **The data type** is locked at construction (= the entry method's `weights:` kwarg
98
+ # fixes weighted vs unweighted; subsequent adds must match). Counts type:
99
99
  # int64 unweighted, float64 weighted. Weighted counts are always float64:
100
100
  # the fused scatter kernel requires float64 weights, so integer weights are
101
101
  # taken as float64 (= integer weighted counts are not supported).
@@ -118,7 +118,7 @@
118
118
  # ### Composition
119
119
  #
120
120
  # `h1 + h2` returns a new Histogram with cells summed elementwise. Both
121
- # operands must agree on edges / fiber_shape / include_max / weighted dtype
121
+ # operands must agree on edges / fiber_shape / include_max / weighted data type
122
122
  # (= the structure-level semantic guard); cells themselves are just
123
123
  # integer / float tallies. See the `+` method.
124
124
  #
@@ -135,7 +135,7 @@ class CArray
135
135
  # {BincountND} instead.
136
136
  class Histogram
137
137
 
138
- # @overload initialize(edges:, fiber_shape: [], include_max: false, weights_dtype: nil)
138
+ # @overload initialize(edges:, fiber_shape: [], include_max: false, weights_data_type: nil)
139
139
  # Allocates a new histogram accumulator.
140
140
  # @param edges [Array<CArray, Array<Numeric>>] one edges array
141
141
  # per histogram dimension; each must be 1-D sorted ascending
@@ -145,11 +145,11 @@ class CArray
145
145
  # @param include_max [Boolean, Array<Boolean>] whether values
146
146
  # equal to the last edge fold into the last bin; a scalar
147
147
  # broadcasts across dimensions.
148
- # @param weights_dtype [Symbol, nil] `data_type` of the
148
+ # @param weights_data_type [Symbol, nil] `data_type` of the
149
149
  # accumulator when weighted; `nil` for a count-only
150
150
  # accumulator (int64 counts).
151
151
  # @return [Histogram]
152
- def initialize (edges:, fiber_shape: [], include_max: false, weights_dtype: nil)
152
+ def initialize (edges:, fiber_shape: [], include_max: false, weights_data_type: nil)
153
153
  @edges_list = edges.map { |e| CArray.wrap_readonly(e, :float64) }
154
154
  raise ArgumentError, "edges must be a non-empty list" if @edges_list.empty?
155
155
  @edges_list.each_with_index do |e, k|
@@ -166,11 +166,11 @@ class CArray
166
166
  else
167
167
  [!!include_max] * @m
168
168
  end
169
- @weighted = !weights_dtype.nil?
170
- @counts_dtype = @weighted ? weights_dtype : :int64
169
+ @weighted = !weights_data_type.nil?
170
+ @counts_data_type = @weighted ? weights_data_type : :int64
171
171
  ext_dims = @n_list.map { |n| n + 2 }
172
172
  ext_shape = @fiber_shape + ext_dims
173
- @full_counts = CArray.public_send(@counts_dtype, *ext_shape).fill(0)
173
+ @full_counts = CArray.public_send(@counts_data_type, *ext_shape).fill(0)
174
174
  @sample_axis = nil
175
175
  @channel_axis = nil
176
176
  end
@@ -321,7 +321,7 @@ class CArray
321
321
 
322
322
  if weights
323
323
  raise ArgumentError, "weights given but accumulator is unweighted" unless @weighted
324
- weights = CArray.wrap_readonly(weights, @counts_dtype)
324
+ weights = CArray.wrap_readonly(weights, @counts_data_type)
325
325
  expected_w_shape = chunk.shape.dup
326
326
  expected_w_shape.delete_at(channel_ax)
327
327
  unless weights.shape == expected_w_shape
@@ -379,7 +379,7 @@ class CArray
379
379
  edges: @edges_list,
380
380
  fiber_shape: @fiber_shape,
381
381
  include_max: @include_max,
382
- weights_dtype: @weighted ? @counts_dtype : nil)
382
+ weights_data_type: @weighted ? @counts_data_type : nil)
383
383
  rf = result.instance_variable_get(:@full_counts)
384
384
  rf[] = @full_counts + other.full_counts
385
385
  result.instance_variable_set(:@sample_axis, @sample_axis)
@@ -401,7 +401,7 @@ class CArray
401
401
  # arr.shape = fiber_shape + (last M bin axes).
402
402
  # Reduce along the last M axes, returns shape fiber_shape (or scalar).
403
403
  #
404
- # `accumulate` preserves dtype (= int64 stays int64, float64 stays float64),
404
+ # `accumulate` preserves the data type (= int64 stays int64, float64 stays float64),
405
405
  # unlike `sum` which always lifts to float64. Caveat: int64 overflows at
406
406
  # ~9.2e18 (silent wrap); weighted float64 loses precision past 2^53 but
407
407
  # does not overflow. Realistic histograms do not hit these limits.
@@ -497,15 +497,15 @@ class CArray
497
497
  [sample_ax, channel_ax].sort.reverse.each { |p| fiber_shape.delete_at(p) }
498
498
 
499
499
  # Weighted counts are float64-only (the fused scatter kernel requires
500
- # float64 weights and float64 counts), so the dtype is fixed here rather
501
- # than derived from the weights' own dtype.
502
- weights_dtype = (:float64 if weights)
500
+ # float64 weights and float64 counts), so the type is fixed here rather
501
+ # than derived from the weights' own type.
502
+ weights_data_type = (:float64 if weights)
503
503
 
504
504
  h = Histogram.send(:new,
505
505
  edges: edges,
506
506
  fiber_shape: fiber_shape,
507
507
  include_max: include_max,
508
- weights_dtype: weights_dtype)
508
+ weights_data_type: weights_data_type)
509
509
  h.add(arr, axis: axis, weights: weights)
510
510
  h
511
511
  end
@@ -7,7 +7,12 @@ class CArray::Inspector # :nodoc:
7
7
  end
8
8
 
9
9
  # @!visibility private
10
- def inspect_string
10
+ #
11
+ # `abbrev` false renders every element instead of eliding with `...`.
12
+ # It is the one difference between #inspect and #inspect_full: the
13
+ # header and the layout are the same, so there is one renderer rather
14
+ # than two that could drift apart.
15
+ def inspect_string (abbrev: true)
11
16
  if @carray.ndim == 0
12
17
  raise "can't inspect CArray of ndim == 0"
13
18
  end
@@ -15,7 +20,7 @@ class CArray::Inspector # :nodoc:
15
20
  class_name = get_class_name()
16
21
  type_name = get_type_name()
17
22
  shape = get_shape()
18
- data_spec = get_data_spec(0, Array.new(@carray.ndim){0}, formatter)
23
+ data_spec = get_data_spec(0, Array.new(@carray.ndim){0}, formatter, abbrev)
19
24
  info_list = get_info_list()
20
25
  output = ["<",
21
26
  format("%s.%s(%s)", class_name, type_name, shape.join(",")),
@@ -49,13 +54,7 @@ class CArray::Inspector # :nodoc:
49
54
  end
50
55
 
51
56
  def get_shape
52
- case @carray.obj_type
53
- when CA_OBJ_UNBOUND_REPEAT
54
- dim = @carray.spec
55
- else
56
- dim = @carray.shape
57
- end
58
- return dim
57
+ return @carray.shape
59
58
  end
60
59
 
61
60
  def get_info_list
@@ -152,7 +151,7 @@ class CArray::Inspector # :nodoc:
152
151
  end
153
152
  end
154
153
 
155
- def get_data_spec (level, idx, formatter)
154
+ def get_data_spec (level, idx, formatter, abbrev = true)
156
155
  io = +"[ " # mutable buffer; `<<` below appends into it
157
156
  ndim = @carray.ndim
158
157
  dim = @carray.shape
@@ -169,7 +168,7 @@ class CArray::Inspector # :nodoc:
169
168
  if i != dim[level] - 1
170
169
  io << ", "
171
170
  end
172
- if io.length > 48 - 2*level
171
+ if abbrev and io.length > 48 - 2*level
173
172
  if i < dim[level] - 1
174
173
  io << "..."
175
174
  over = true
@@ -188,24 +187,24 @@ class CArray::Inspector # :nodoc:
188
187
  end
189
188
  else
190
189
  over = false
191
- show = [dim[level], 5].min
190
+ show = abbrev ? [dim[level], 5].min : dim[level]
192
191
  show.times do |i|
193
192
  idx[level] = i
194
- io << get_data_spec(level+1, idx, formatter)
193
+ io << get_data_spec(level+1, idx, formatter, abbrev)
195
194
  if i < show - 1
196
195
  io << ",\n" + " " * (level+1)
197
196
  end
198
- if i >= 2 and dim[level] > 5
197
+ if abbrev and i >= 2 and dim[level] > 5
199
198
  break
200
199
  end
201
200
  end
202
- if dim[level] > 5
201
+ if abbrev and dim[level] > 5
203
202
  io << "... ... ..."
204
203
  over = true
205
204
  end
206
205
  if over
207
206
  idx[level] = dim[level] - 1
208
- io << "\n"+ " " * (level+1) + get_data_spec(level+1, idx, formatter)
207
+ io << "\n"+ " " * (level+1) + get_data_spec(level+1, idx, formatter, abbrev)
209
208
  end
210
209
  end
211
210
  io << " ]"
@@ -225,6 +224,34 @@ class CArray
225
224
  return CArray::Inspector.new(self).inspect_string
226
225
  end
227
226
 
227
+ # @overload inspect_full
228
+ # The same description as {#inspect}, with every element rendered
229
+ # instead of the `...` preview.
230
+ #
231
+ # `inspect` abbreviates on purpose -- it is what `p`, `irb` and an
232
+ # error message call, and a million-cell array has to stay readable
233
+ # there. `inspect_full` is for the other moment, when the whole
234
+ # array is the thing you came to look at:
235
+ #
236
+ # puts a.inspect_full
237
+ #
238
+ # The header, the layout and the `_` for a masked cell are
239
+ # `inspect`'s; only the eliding is dropped, so for an array small
240
+ # enough that `inspect` was not abbreviating anything the two give
241
+ # the same string.
242
+ #
243
+ # Note that neither of these is `to_s`, which returns the **raw
244
+ # bytes** of the storage rather than anything printable.
245
+ #
246
+ # The result is one String holding every element, so it is as large
247
+ # as the array is: nothing here is streamed, and a line is as long
248
+ # as the last axis makes it.
249
+ #
250
+ # @return [String]
251
+ def inspect_full
252
+ return CArray::Inspector.new(self).inspect_string(abbrev: false)
253
+ end
254
+
228
255
  private
229
256
 
230
257
  def desc
@@ -14,8 +14,8 @@
14
14
  # those are NOT part of the common contract because some members legitimately
15
15
  # omit them (a window has no map, a group has no within-piece min_index).
16
16
  #
17
- # The retired 2.0 generic dispatch (calculate / filter / evaluate over a
18
- # kernel_at_addr slot) is preserved at `samples/caiterator/iterator.{rb,c}`.
17
+ # The 2.0 generic dispatch (calculate / filter / evaluate over a
18
+ # kernel_at_addr slot) was retired in 3.0.
19
19
  #
20
20
  # ----------------------------------------------------------------------------
21
21
 
@@ -36,8 +36,117 @@ class CAIterator
36
36
  # rather than reading as "no such method" -- that gap is the member's to close.
37
37
  # A member that genuinely cannot provide one overrides it to raise with its
38
38
  # own reason.
39
+ # The methods below are declared here, so that the family surface is
40
+ # documented in one place; each member generates its own implementation and
41
+ # a member for which one is ill-defined overrides it to raise with a reason.
42
+ # "Piece" is the unit a member iterates over: a slab, a window, a block, a
43
+ # category, or a group.
44
+ #
45
+ # @!method sum
46
+ # Returns the sum of each piece.
47
+ # @return [CArray] one value per piece, shaped {#shape}.
48
+ # @!method accumulate
49
+ # Returns the sum of each piece kept in the source's own `data_type`,
50
+ # wrapping at its width, where {#sum} answers in the type the core
51
+ # promotes to (`:float64` for integers).
52
+ # @return [CArray] one value per piece.
53
+ # @!method prod
54
+ # Returns the product of each piece.
55
+ # @return [CArray] one value per piece.
56
+ # @!method mean
57
+ # Returns the arithmetic mean of each piece.
58
+ # @return [CArray] one value per piece.
59
+ # @!method min
60
+ # Returns the smallest value in each piece.
61
+ # @return [CArray] one value per piece.
62
+ # @!method max
63
+ # Returns the largest value in each piece.
64
+ # @return [CArray] one value per piece.
65
+ # @!method minmax
66
+ # Returns the smallest and largest value of each piece, found in one pass.
67
+ # @return [Array<CArray>] the pair `[min, max]`.
68
+ # @!method variance
69
+ # Returns the sample variance (divisor `n - 1`) of each piece.
70
+ # @return [CArray] one value per piece.
71
+ # @!method variancep
72
+ # Returns the population variance (divisor `n`) of each piece.
73
+ # @return [CArray] one value per piece.
74
+ # @!method stddev
75
+ # Returns the sample standard deviation (divisor `n - 1`) of each piece.
76
+ # @return [CArray] one value per piece.
77
+ # @!method stddevp
78
+ # Returns the population standard deviation (divisor `n`) of each piece.
79
+ # @return [CArray] one value per piece.
80
+ # @!method all
81
+ # Returns whether every cell of each piece is true.
82
+ # @return [CArray] `:boolean`, one value per piece.
83
+ # @!method any
84
+ # Returns whether any cell of each piece is true.
85
+ # @return [CArray] `:boolean`, one value per piece.
86
+ # @!method min_index
87
+ # Returns the position of the smallest value **within** each piece.
88
+ # @return [CArray] one index per piece.
89
+ # @!method max_index
90
+ # Returns the position of the largest value within each piece.
91
+ # @return [CArray] one index per piece.
92
+ # @!method min_addr
93
+ # Returns the flat address **in the source** of the smallest value of each
94
+ # piece -- which source cell holds it, rather than where it sits inside the
95
+ # piece. Use it to read the same cell out of another source-shaped array.
96
+ # @return [CArray] one flat address per piece.
97
+ # @!method max_addr
98
+ # Returns the flat address in the source of the largest value of each piece.
99
+ # @return [CArray] one flat address per piece.
100
+ # @!method wsum(weights)
101
+ # Returns the weighted sum of each piece.
102
+ # @param weights [CArray] one weight per source cell, shaped like the source.
103
+ # @return [CArray] one value per piece.
104
+ # @!method wmean(weights)
105
+ # Returns the weighted mean of each piece.
106
+ # @param weights [CArray] one weight per source cell, shaped like the source.
107
+ # @return [CArray] one value per piece.
108
+ # @!method median
109
+ # Returns the median of each piece.
110
+ # @return [CArray] one value per piece.
111
+ # @!method percentile(*pers)
112
+ # Returns the requested percentile(s) of each piece.
113
+ # @param pers [Array<Numeric>] percentile positions in `0..100`.
114
+ # @return [CArray, Array<CArray>] one array per requested position; a
115
+ # single position returns that array directly.
116
+ # @!method quantile
117
+ # Returns the five-number summary of each piece,
118
+ # `[min, Q1, median, Q3, max]`.
119
+ # @return [Array<CArray>] five arrays, one per position.
120
+ # @!method count(value = nil)
121
+ # Returns a count per piece: with no argument the cells that are not
122
+ # masked, with `UNDEF` the masked cells, and with any other value the
123
+ # cells equal to it.
124
+ # @param value [Object] value to match, or `UNDEF`.
125
+ # @return [CArray] one count per piece.
126
+ # @!method count_not_masked
127
+ # Returns the number of cells of each piece that are not masked.
128
+ # @return [CArray] one count per piece.
129
+ # @!method count_masked
130
+ # Returns the number of masked cells of each piece.
131
+ # @return [CArray] one count per piece.
132
+ # @!method elements
133
+ # Returns the total number of cells in each piece, masked or not.
134
+ # @return [CArray] one count per piece.
135
+ # @!method each
136
+ # Yields each piece in turn as a CArray.
137
+ # @yieldparam piece [CArray]
138
+ # @return [Enumerator, self] an Enumerator when no block is given.
139
+ # @!method reduce(init = nil)
140
+ # Folds the pieces with a block, for a reduction the family does not name.
141
+ # Without `init` the first piece seeds the accumulator.
142
+ # @param init [Object] initial accumulator value.
143
+ # @yieldparam acc [Object] running accumulator.
144
+ # @yieldparam piece [CArray] next piece.
145
+ # @yieldreturn [Object] updated accumulator.
146
+ # @return [Object] the final accumulator.
39
147
  [
40
- :sum, :prod, :mean, :min, :max, :variance, :stddev, :all, :any, # tier 1
148
+ :sum, :accumulate, :prod, :mean, :min, :max, # tier 1
149
+ :variance, :stddev, :all, :any,
41
150
  :variancep, :stddevp, :minmax, # tier 2
42
151
  :min_index, :max_index, :min_addr, :max_addr, # position
43
152
  :wsum, :wmean, # weighted
@@ -65,6 +174,41 @@ class CAIterator
65
174
  # sort_addr. Un-overridden each is simply unavailable, not a contract
66
175
  # violation. min_addr / max_addr stay required: a single winner address is
67
176
  # well-defined even for an overlapping window.
177
+ # @!method map
178
+ # Returns a source-shaped array built by applying the block to each piece
179
+ # and scattering the result back into that piece's cells.
180
+ # @yieldparam piece [CArray]
181
+ # @yieldreturn [CArray, Numeric] replacement values for the piece.
182
+ # @return [CArray] shaped like the source.
183
+ # @raise [NotImplementedError] for a member whose pieces overlap, where a
184
+ # cell would receive more than one value.
185
+ # @!method sort_addr
186
+ # Returns a source-shaped array whose cells, read piece by piece, give the
187
+ # flat source addresses that put that piece in ascending order.
188
+ # @return [CArray] `:int64`, shaped like the source.
189
+ # @raise [NotImplementedError] for a member whose pieces overlap.
190
+ # @!method cumsum
191
+ # Returns a source-shaped array of the running sum within each piece.
192
+ # @return [CArray] shaped like the source.
193
+ # @raise [NotImplementedError] for a member whose pieces overlap, where a
194
+ # cell has no single running value.
195
+ # @!method cumprod
196
+ # Returns a source-shaped array of the running product within each piece.
197
+ # @return [CArray] shaped like the source.
198
+ # @raise [NotImplementedError] for a member whose pieces overlap.
199
+ # @!method cummax
200
+ # Returns a source-shaped array of the running maximum within each piece.
201
+ # @return [CArray] shaped like the source.
202
+ # @raise [NotImplementedError] for a member whose pieces overlap.
203
+ # @!method cummin
204
+ # Returns a source-shaped array of the running minimum within each piece.
205
+ # @return [CArray] shaped like the source.
206
+ # @raise [NotImplementedError] for a member whose pieces overlap.
207
+ # @!method cumcount
208
+ # Returns a source-shaped array of the running count of cells that are not
209
+ # masked within each piece.
210
+ # @return [CArray] shaped like the source.
211
+ # @raise [NotImplementedError] for a member whose pieces overlap.
68
212
  [:map, :sort_addr,
69
213
  :cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |name|
70
214
  define_method(name) do |*, **, &_blk|