carray 3.0.0 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.yardopts +2 -1
- data/CHANGELOG.md +845 -3
- data/{NEWS.md → CHANGELOG.v1.md} +1 -1
- data/README.md +33 -33
- data/carray.gemspec +12 -8
- data/ext/ca_axis_dispatch.c +33 -4
- data/ext/ca_axis_group.c +256 -114
- data/ext/ca_binop_dispatch.c +5 -6
- data/ext/ca_binop_dispatch.h +0 -7
- data/ext/ca_categorical_iterator.c +121 -67
- data/ext/ca_for_each_element.h +6 -8
- data/ext/ca_group_iter.c +1 -1
- data/ext/ca_kernel_iterator.c +386 -74
- data/ext/ca_kernel_iterator.h +145 -45
- data/ext/ca_obj_array.c +62 -20
- data/ext/ca_obj_bincmp.c +57 -27
- data/ext/ca_obj_binop.c +86 -30
- data/ext/ca_obj_block.c +4 -4
- data/ext/ca_obj_const_string.c +88 -27
- data/ext/ca_obj_face.c +104 -0
- data/ext/ca_obj_face.h +48 -1
- data/ext/ca_obj_fake.c +11 -0
- data/ext/ca_obj_fixlen_string.c +21 -6
- data/ext/ca_obj_grid.c +11 -3
- data/ext/ca_obj_meld.c +125 -22
- data/ext/ca_obj_moncmp.c +33 -14
- data/ext/ca_obj_monop.c +19 -7
- data/ext/ca_obj_object.c +107 -4
- data/ext/ca_obj_record.c +3 -1
- data/ext/ca_obj_refer.c +30 -6
- data/ext/ca_obj_roll.c +13 -5
- data/ext/ca_obj_select.c +49 -34
- data/ext/ca_obj_select_axis.c +16 -6
- data/ext/ca_obj_shift.c +3 -3
- data/ext/ca_obj_stack.c +3 -8
- data/ext/ca_obj_stride.c +178 -24
- data/ext/ca_obj_string.c +11 -5
- data/ext/ca_obj_tile.c +13 -5
- data/ext/ca_obj_time.c +3 -1
- data/ext/ca_obj_timedelta.c +3 -1
- data/ext/ca_obj_transpose.c +1 -1
- data/ext/ca_obj_triop.c +62 -23
- data/ext/ca_obj_window.c +94 -19
- data/ext/ca_op_cmplx64.h +123 -0
- data/ext/ca_op_ipower.c +1 -5
- data/ext/ca_rng_normal.h +42 -0
- data/ext/ca_rng_xoshiro256pp.h +105 -0
- data/ext/ca_sort_kernels.h +5 -5
- data/ext/ca_sweep_engine.c +362 -155
- data/ext/ca_sweep_engine.h +39 -9
- data/ext/ca_transform_common.c +7 -0
- data/ext/carray.h +148 -132
- data/ext/carray_access.c +83 -49
- data/ext/carray_address_basis.c +590 -0
- data/ext/carray_bincount.c +8 -8
- data/ext/carray_broadcast.c +102 -9
- data/ext/carray_build_flags.h +3 -0
- data/ext/carray_call_cfunc.c +2010 -335
- data/ext/carray_call_cfunc.h +153 -0
- data/ext/carray_cast.c +184 -72
- data/ext/carray_conversion.c +22 -22
- data/ext/carray_copy.c +67 -32
- data/ext/carray_core.c +140 -13
- data/ext/carray_count.c +10 -11
- data/ext/carray_element.c +7 -7
- data/ext/carray_factorize.c +72 -51
- data/ext/carray_hold.c +1 -1
- data/ext/carray_index_classifier.c +7 -21
- data/ext/carray_internal.h +36 -3
- data/ext/carray_kernels_bincmp.c +1 -0
- data/ext/carray_kernels_binop.c +320 -298
- data/ext/carray_kernels_init.c +1095 -0
- data/ext/carray_kernels_map.c +1 -0
- data/ext/carray_kernels_moncmp.c +1 -0
- data/ext/carray_kernels_monop.c +412 -411
- data/ext/carray_kernels_reduce_aggregate.c +256 -87
- data/ext/carray_kernels_reduce_boolean.c +1 -0
- data/ext/carray_kernels_reduce_cumulative.c +324 -54
- data/ext/carray_kernels_reduce_extreme.c +605 -58
- data/ext/carray_kernels_reduce_variance.c +1 -0
- data/ext/carray_kernels_scan.c +5 -4
- data/ext/carray_kernels_search.c +187 -106
- data/ext/carray_kernels_sort.c +1 -0
- data/ext/carray_kernels_triop.c +1 -0
- data/ext/carray_lazy.c +175 -5
- data/ext/carray_loop.c +7 -1
- data/ext/carray_mask.c +29 -13
- data/ext/carray_median_percentile.c +56 -1
- data/ext/carray_memory_view.c +46 -21
- data/ext/carray_operator.c +50 -53
- data/ext/carray_order.c +1 -1
- data/ext/carray_partition.c +4 -4
- data/ext/carray_random.c +391 -47
- data/ext/carray_scatter.c +1 -1
- data/ext/carray_slab.c +13 -0
- data/ext/carray_sort.c +22 -24
- data/ext/carray_sort_kernel.c +10 -10
- data/ext/carray_test.c +1 -1
- data/ext/extconf.rb +18 -0
- data/ext/mk_call_cfunc.rb +323 -145
- data/ext/mkkernel.rb +651 -129
- data/ext/ruby_carray.c +28 -17
- data/ext/version.h +4 -4
- data/lib/carray/autoload_carray.rb +24 -3
- data/lib/carray/autoload_method_extension.rb +14 -1
- data/lib/carray/axis_group.rb +84 -7
- data/lib/carray/basics.rb +63 -59
- data/lib/carray/bincount_nd.rb +12 -12
- data/lib/carray/block_iterator.rb +111 -25
- data/lib/carray/categorical.rb +154 -37
- data/lib/carray/categorical_iterator.rb +300 -151
- data/lib/carray/conditional.rb +14 -14
- data/lib/carray/const_string.rb +131 -27
- data/lib/carray/construct.rb +61 -0
- data/lib/carray/core_extensions.rb +62 -5
- data/lib/carray/data_type_extension.rb +21 -1
- data/lib/carray/data_type_limits.rb +91 -0
- data/lib/carray/fixlen_string.rb +1 -1
- data/lib/carray/frame/concat.rb +3 -3
- data/lib/carray/frame/convert.rb +1 -1
- data/lib/carray/frame/csv_parser.rb +11 -4
- data/lib/carray/frame/frame.rb +85 -21
- data/lib/carray/frame/group.rb +36 -3
- data/lib/carray/frame/io.rb +224 -15
- data/lib/carray/frame/records.rb +18 -4
- data/lib/carray/frame/sort.rb +1 -1
- data/lib/carray/frame/verbs.rb +32 -12
- data/lib/carray/fuse_source.rb +123 -0
- data/lib/carray/fusion.rb +218 -0
- data/lib/carray/histogram.rb +16 -16
- data/lib/carray/inspect.rb +43 -16
- data/lib/carray/iterator.rb +147 -3
- data/lib/carray/lazy.rb +88 -73
- data/lib/carray/mask_gap_fill.rb +3 -1
- data/lib/carray/meld_reduce.rb +2 -2
- data/lib/carray/methods/align_addr.rb +1 -1
- data/lib/carray/methods/composition.rb +1 -1
- data/lib/carray/methods/discovery_along.rb +74 -0
- data/lib/carray/methods/factorize.rb +50 -0
- data/lib/carray/methods/is_in.rb +25 -14
- data/lib/carray/methods/locate_addr.rb +81 -3
- data/lib/carray/methods/mask_duplicates.rb +36 -2
- data/lib/carray/methods/meshgrid.rb +4 -5
- data/lib/carray/methods/mode.rb +2 -2
- data/lib/carray/methods/nunique.rb +23 -2
- data/lib/carray/methods/repeat.rb +110 -0
- data/lib/carray/methods/snap.rb +7 -2
- data/lib/carray/methods/unique.rb +44 -4
- data/lib/carray/methods/value_counts.rb +2 -2
- data/lib/carray/rng.rb +86 -0
- data/lib/carray/runtime.rb +0 -19
- data/lib/carray/slab_iterator.rb +77 -19
- data/lib/carray/string_operation_extension.rb +10 -6
- data/lib/carray/time.rb +906 -491
- data/lib/carray/window_iterator.rb +421 -27
- data/lib/carray.rb +7 -5
- data/yard-stubs/ca_obj_array.rb +385 -0
- data/yard-stubs/ca_obj_bitarray.rb +38 -0
- data/yard-stubs/ca_obj_bitfield.rb +43 -0
- data/yard-stubs/ca_obj_block.rb +68 -0
- data/yard-stubs/ca_obj_byte_swap.rb +56 -0
- data/yard-stubs/ca_obj_fake.rb +31 -0
- data/yard-stubs/ca_obj_farray.rb +32 -0
- data/yard-stubs/ca_obj_field.rb +45 -0
- data/yard-stubs/ca_obj_grid.rb +35 -0
- data/yard-stubs/ca_obj_refer.rb +72 -0
- data/yard-stubs/ca_obj_roll.rb +45 -0
- data/yard-stubs/ca_obj_shift.rb +43 -0
- data/yard-stubs/ca_obj_stride.rb +181 -0
- data/yard-stubs/ca_obj_tile.rb +29 -0
- data/yard-stubs/ca_obj_transpose.rb +40 -0
- data/yard-stubs/ca_obj_window.rb +57 -0
- data/yard-stubs/carray_access.rb +131 -0
- data/yard-stubs/carray_attribute.rb +246 -0
- data/yard-stubs/carray_broadcast.rb +37 -0
- data/yard-stubs/carray_cast.rb +489 -0
- data/yard-stubs/carray_class.rb +65 -0
- data/yard-stubs/carray_conversion.rb +76 -0
- data/yard-stubs/carray_copy.rb +79 -0
- data/yard-stubs/carray_core.rb +34 -0
- data/yard-stubs/carray_count.rb +84 -0
- data/yard-stubs/carray_element.rb +108 -0
- data/yard-stubs/carray_generate.rb +66 -0
- data/yard-stubs/carray_lazy.rb +228 -0
- data/yard-stubs/carray_loop.rb +140 -0
- data/yard-stubs/carray_mask.rb +259 -0
- data/yard-stubs/carray_math.rb +1615 -0
- data/yard-stubs/carray_mathfunc.rb +45 -0
- data/yard-stubs/carray_median_percentile.rb +103 -0
- data/yard-stubs/carray_memory_view.rb +163 -0
- data/yard-stubs/carray_order.rb +252 -0
- data/yard-stubs/carray_random.rb +89 -0
- data/yard-stubs/carray_scatter.rb +106 -0
- data/yard-stubs/carray_slab.rb +59 -0
- data/yard-stubs/carray_sort.rb +165 -0
- data/yard-stubs/carray_test.rb +85 -0
- data/yard-stubs/carray_undef.rb +64 -0
- data/yard-stubs/carray_utils.rb +97 -0
- data/yard-stubs/ruby_carray.rb +193 -0
- metadata +67 -10
- data/ext/ca_obj_unbound_repeat.c +0 -496
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# ---------------------------------------------------------------------------
|
|
2
|
+
# Turning a lazy expression into a plan a compiler can read.
|
|
3
|
+
#
|
|
4
|
+
# A lazy view is already a typed, closed expression graph, and the kernels
|
|
5
|
+
# already carry the C that computes each operation (CArray.__kernel_body__).
|
|
6
|
+
# What is missing between them is the reading: which operation each node is,
|
|
7
|
+
# what its mask does, and where the leaves are. That is what a plan holds.
|
|
8
|
+
#
|
|
9
|
+
# Nothing here compiles anything. A plan is plain data, and CArray itself
|
|
10
|
+
# never needs one -- it can always walk the view. What a plan is for is a
|
|
11
|
+
# second evaluator: hand it to one, and the answer must be the same.
|
|
12
|
+
# ---------------------------------------------------------------------------
|
|
13
|
+
|
|
14
|
+
class CArray
|
|
15
|
+
|
|
16
|
+
module Fusion
|
|
17
|
+
|
|
18
|
+
# One value per node, in evaluation order; the last is the result.
|
|
19
|
+
#
|
|
20
|
+
# Op an operation, reading the nodes named in `args`
|
|
21
|
+
# Leaf an array, the `index`-th of plan.leaves
|
|
22
|
+
# Const a scalar written into the expression
|
|
23
|
+
Op = Struct.new(:kind, :name, :data_type, :args, :body, :mask, :trapping)
|
|
24
|
+
Leaf = Struct.new(:index, :data_type, :masked)
|
|
25
|
+
Const = Struct.new(:value, :data_type)
|
|
26
|
+
|
|
27
|
+
Plan = Struct.new(:nodes, :leaves, :data_type, :dim, :masked, :signature)
|
|
28
|
+
|
|
29
|
+
LAZY_CLASSES = [CAMonOp, CABinOp, CATriOp, CAMonCmp, CABinCmp, CALazyMarker]
|
|
30
|
+
|
|
31
|
+
# A lazy node names its operation by an id and the kernels name it by a
|
|
32
|
+
# symbol. These are the same operations, spelled the way each side
|
|
33
|
+
# spells them; the rest are spelled alike.
|
|
34
|
+
BINOP_NAMES = {
|
|
35
|
+
:+ => :add, :- => :sub, :* => :mul,
|
|
36
|
+
:/ => :div, :** => :power, :% => :mod,
|
|
37
|
+
:& => :bit_and_i, :| => :bit_or_i, :^ => :bit_xor_i,
|
|
38
|
+
:<< => :bit_lshift, :>> => :bit_rshift,
|
|
39
|
+
}.freeze
|
|
40
|
+
TRIOP_NAMES = { :__clip_ki__ => :clip }.freeze
|
|
41
|
+
|
|
42
|
+
MONOP_BY_ID = CArray::LAZY_MONOP_OP_IDS.invert.freeze
|
|
43
|
+
BINOP_BY_ID = CArray::LAZY_BINOP_OP_IDS.invert.freeze
|
|
44
|
+
TRIOP_BY_ID = CArray::LAZY_TRIOP_OP_IDS.invert.freeze
|
|
45
|
+
|
|
46
|
+
# Integer division and its relatives raise on a zero divisor, so a cell
|
|
47
|
+
# the mask excludes must not be computed at all -- the divisor there is
|
|
48
|
+
# nobody's business (ca_binop_dispatch.c).
|
|
49
|
+
TRAPPING = %i[div mod quo_i fmod].freeze
|
|
50
|
+
INTEGERS = %i[int8 int16 int32 int64 uint8 uint16 uint32 uint64].freeze
|
|
51
|
+
|
|
52
|
+
class Refused < StandardError; end
|
|
53
|
+
|
|
54
|
+
# ---- who computes a plan --------------------------------------------
|
|
55
|
+
#
|
|
56
|
+
# CArray can always walk the expression, so nothing has to be registered
|
|
57
|
+
# and nothing changes when nothing is. What a registered evaluator adds
|
|
58
|
+
# is a second way to arrive at the same answer; it is asked, and it may
|
|
59
|
+
# decline. The dispatch point stays on CArray's side, which is what
|
|
60
|
+
# keeps the threshold below a decision about CArray's own walk rather
|
|
61
|
+
# than one that moves with whatever is installed.
|
|
62
|
+
#
|
|
63
|
+
# The evaluator itself is held by CArray (see carray/lazy.rb), so that
|
|
64
|
+
# materialising an expression need not reach for this file at all until
|
|
65
|
+
# something has been registered.
|
|
66
|
+
|
|
67
|
+
# Reaching a compiled kernel costs about the same whatever the array's
|
|
68
|
+
# size, and what it buys is the passes the walk would make. Below this
|
|
69
|
+
# the walk is the faster answer. The crossing moves with how wide the
|
|
70
|
+
# expression is -- measured, thirty thousand cells at one operation, six
|
|
71
|
+
# thousand at six -- and this brackets those: a one-operation expression
|
|
72
|
+
# loses a couple of microseconds here, a six-operation one wins ten.
|
|
73
|
+
THRESHOLD = 10_000
|
|
74
|
+
|
|
75
|
+
# Returns the array, or nil where nothing computed it.
|
|
76
|
+
def self.evaluate (view)
|
|
77
|
+
return nil unless askable?(view)
|
|
78
|
+
out = CArray.__alloc_uninit__(view.data_type, view.dim)
|
|
79
|
+
evaluate_into(view, out) ? out : nil
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Fills an array the caller already has. Called from the store as well,
|
|
83
|
+
# where making one and copying it over would be most of the work.
|
|
84
|
+
# Returns true when something computed it.
|
|
85
|
+
def self.evaluate_into (view, out)
|
|
86
|
+
evaluator = CArray.expression_evaluator or return false
|
|
87
|
+
return false unless askable?(view)
|
|
88
|
+
plan = plan(view) or return false
|
|
89
|
+
# A marker over an array, or anything else with nothing to compute,
|
|
90
|
+
# is not worth handing over.
|
|
91
|
+
return false unless plan.nodes.any? { |n| n.is_a?(Op) }
|
|
92
|
+
out.mask = 0 if plan.masked && ! out.has_mask?
|
|
93
|
+
evaluator.call(plan, out) ? true : false
|
|
94
|
+
rescue StandardError => error
|
|
95
|
+
CArray.expression_evaluator = nil
|
|
96
|
+
warn "CArray: the registered expression evaluator raised " \
|
|
97
|
+
"(#{error.class}: #{error.message}); expressions will be walked " \
|
|
98
|
+
"from here on"
|
|
99
|
+
false
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def self.askable? (view)
|
|
103
|
+
! CArray.expression_evaluator.nil? && view.elements >= THRESHOLD
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Returns a Plan, or nil where the expression holds something a plan
|
|
107
|
+
# cannot describe. Refusing is ordinary: the caller walks instead.
|
|
108
|
+
def self.plan (view)
|
|
109
|
+
build(view)
|
|
110
|
+
rescue Refused
|
|
111
|
+
nil
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def self.build (view)
|
|
115
|
+
raise Refused, "not a lazy expression" unless lazy?(view)
|
|
116
|
+
w = Walk.new
|
|
117
|
+
w.visit(view)
|
|
118
|
+
Plan.new(w.nodes, w.leaves, view.data_type, view.dim,
|
|
119
|
+
w.leaves.any? { |a| a.has_mask? }, w.signature)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def self.lazy? (x)
|
|
123
|
+
LAZY_CLASSES.any? { |k| x.is_a?(k) }
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# ---- the walk -------------------------------------------------------
|
|
127
|
+
|
|
128
|
+
class Walk
|
|
129
|
+
attr_reader :nodes, :leaves, :signature
|
|
130
|
+
|
|
131
|
+
def initialize
|
|
132
|
+
@nodes = []
|
|
133
|
+
@leaves = []
|
|
134
|
+
@seen = {}
|
|
135
|
+
@signature = +""
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def visit (n)
|
|
139
|
+
@seen[n.object_id] ||= build(n)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
private
|
|
143
|
+
|
|
144
|
+
def build (n)
|
|
145
|
+
case n
|
|
146
|
+
when CALazyMarker then visit(n.parent)
|
|
147
|
+
when CAMonOp then unary(n)
|
|
148
|
+
when CABinOp then binary(n)
|
|
149
|
+
when CATriOp then ternary(n)
|
|
150
|
+
when CScalar then constant(n)
|
|
151
|
+
when CArray then leaf(n)
|
|
152
|
+
else raise Refused, "#{n.class} in an expression"
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def unary (n)
|
|
157
|
+
name = spell(MONOP_BY_ID, n.__op_id__, {})
|
|
158
|
+
args = [visit(n.parent)]
|
|
159
|
+
# A view over one array is masked exactly where that array is
|
|
160
|
+
# (ca_obj_monop.c).
|
|
161
|
+
op(:monop, name, n.data_type, args, :pass)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def binary (n)
|
|
165
|
+
name = spell(BINOP_BY_ID, n.__op_id__, BINOP_NAMES)
|
|
166
|
+
args = [visit(n.parent), visit(n.__binop_right__)]
|
|
167
|
+
# Boolean `&` and `|` are three-valued: a masked cell whose known
|
|
168
|
+
# side settles the answer comes back unmasked (ca_obj_binop.c).
|
|
169
|
+
rule = if n.data_type == :boolean && name == :bit_or_i then :kleene_or
|
|
170
|
+
elsif n.data_type == :boolean && name == :bit_and_i then :kleene_and
|
|
171
|
+
else :union
|
|
172
|
+
end
|
|
173
|
+
op(:binop, name, n.data_type, args, rule)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def ternary (n)
|
|
177
|
+
name = spell(TRIOP_BY_ID, n.__op_id__, TRIOP_NAMES)
|
|
178
|
+
args = [visit(n.parent), visit(n.__triop_op2__), visit(n.__triop_op3__)]
|
|
179
|
+
op(:triop, name, n.data_type, args, :union)
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def op (kind, name, type, args, mask)
|
|
183
|
+
body = CArray.__kernel_body__(kind, name, type) or
|
|
184
|
+
raise Refused, "#{kind} #{name} has no body at #{type}"
|
|
185
|
+
note(kind.to_s[0], name, type)
|
|
186
|
+
push Op.new(kind, name, type, args, body, mask,
|
|
187
|
+
TRAPPING.include?(name) && INTEGERS.include?(type))
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def leaf (n)
|
|
191
|
+
note("a", n.data_type, n.has_mask? ? 1 : 0)
|
|
192
|
+
@leaves << n
|
|
193
|
+
push Leaf.new(@leaves.size - 1, n.data_type, n.has_mask?)
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def constant (n)
|
|
197
|
+
note("k", n[0], n.data_type)
|
|
198
|
+
push Const.new(n[0], n.data_type)
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def push (node)
|
|
202
|
+
@nodes << node
|
|
203
|
+
@nodes.size - 1
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def spell (table, id, renames)
|
|
207
|
+
ruby = table[id] or raise Refused, "operation id #{id}"
|
|
208
|
+
renames.fetch(ruby, ruby)
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Two expressions of the same shape compute alike, whatever arrays
|
|
212
|
+
# they are over, so a consumer can keep one compiled kernel for both.
|
|
213
|
+
def note (*parts)
|
|
214
|
+
@signature << parts.join(":") << ";"
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
end
|
data/lib/carray/histogram.rb
CHANGED
|
@@ -94,8 +94,8 @@
|
|
|
94
94
|
# Each sample contributes `w[i]` instead of 1 to its target cell.
|
|
95
95
|
#
|
|
96
96
|
# * **weights.shape** = chunk.shape minus the channel axis (= fiber + (A,)).
|
|
97
|
-
# * **
|
|
98
|
-
# fixes weighted vs unweighted; subsequent adds must match). Counts
|
|
97
|
+
# * **The data type** is locked at construction (= the entry method's `weights:` kwarg
|
|
98
|
+
# fixes weighted vs unweighted; subsequent adds must match). Counts type:
|
|
99
99
|
# int64 unweighted, float64 weighted. Weighted counts are always float64:
|
|
100
100
|
# the fused scatter kernel requires float64 weights, so integer weights are
|
|
101
101
|
# taken as float64 (= integer weighted counts are not supported).
|
|
@@ -118,7 +118,7 @@
|
|
|
118
118
|
# ### Composition
|
|
119
119
|
#
|
|
120
120
|
# `h1 + h2` returns a new Histogram with cells summed elementwise. Both
|
|
121
|
-
# operands must agree on edges / fiber_shape / include_max / weighted
|
|
121
|
+
# operands must agree on edges / fiber_shape / include_max / weighted data type
|
|
122
122
|
# (= the structure-level semantic guard); cells themselves are just
|
|
123
123
|
# integer / float tallies. See the `+` method.
|
|
124
124
|
#
|
|
@@ -135,7 +135,7 @@ class CArray
|
|
|
135
135
|
# {BincountND} instead.
|
|
136
136
|
class Histogram
|
|
137
137
|
|
|
138
|
-
# @overload initialize(edges:, fiber_shape: [], include_max: false,
|
|
138
|
+
# @overload initialize(edges:, fiber_shape: [], include_max: false, weights_data_type: nil)
|
|
139
139
|
# Allocates a new histogram accumulator.
|
|
140
140
|
# @param edges [Array<CArray, Array<Numeric>>] one edges array
|
|
141
141
|
# per histogram dimension; each must be 1-D sorted ascending
|
|
@@ -145,11 +145,11 @@ class CArray
|
|
|
145
145
|
# @param include_max [Boolean, Array<Boolean>] whether values
|
|
146
146
|
# equal to the last edge fold into the last bin; a scalar
|
|
147
147
|
# broadcasts across dimensions.
|
|
148
|
-
# @param
|
|
148
|
+
# @param weights_data_type [Symbol, nil] `data_type` of the
|
|
149
149
|
# accumulator when weighted; `nil` for a count-only
|
|
150
150
|
# accumulator (int64 counts).
|
|
151
151
|
# @return [Histogram]
|
|
152
|
-
def initialize (edges:, fiber_shape: [], include_max: false,
|
|
152
|
+
def initialize (edges:, fiber_shape: [], include_max: false, weights_data_type: nil)
|
|
153
153
|
@edges_list = edges.map { |e| CArray.wrap_readonly(e, :float64) }
|
|
154
154
|
raise ArgumentError, "edges must be a non-empty list" if @edges_list.empty?
|
|
155
155
|
@edges_list.each_with_index do |e, k|
|
|
@@ -166,11 +166,11 @@ class CArray
|
|
|
166
166
|
else
|
|
167
167
|
[!!include_max] * @m
|
|
168
168
|
end
|
|
169
|
-
@weighted = !
|
|
170
|
-
@
|
|
169
|
+
@weighted = !weights_data_type.nil?
|
|
170
|
+
@counts_data_type = @weighted ? weights_data_type : :int64
|
|
171
171
|
ext_dims = @n_list.map { |n| n + 2 }
|
|
172
172
|
ext_shape = @fiber_shape + ext_dims
|
|
173
|
-
@full_counts = CArray.public_send(@
|
|
173
|
+
@full_counts = CArray.public_send(@counts_data_type, *ext_shape).fill(0)
|
|
174
174
|
@sample_axis = nil
|
|
175
175
|
@channel_axis = nil
|
|
176
176
|
end
|
|
@@ -321,7 +321,7 @@ class CArray
|
|
|
321
321
|
|
|
322
322
|
if weights
|
|
323
323
|
raise ArgumentError, "weights given but accumulator is unweighted" unless @weighted
|
|
324
|
-
weights = CArray.wrap_readonly(weights, @
|
|
324
|
+
weights = CArray.wrap_readonly(weights, @counts_data_type)
|
|
325
325
|
expected_w_shape = chunk.shape.dup
|
|
326
326
|
expected_w_shape.delete_at(channel_ax)
|
|
327
327
|
unless weights.shape == expected_w_shape
|
|
@@ -379,7 +379,7 @@ class CArray
|
|
|
379
379
|
edges: @edges_list,
|
|
380
380
|
fiber_shape: @fiber_shape,
|
|
381
381
|
include_max: @include_max,
|
|
382
|
-
|
|
382
|
+
weights_data_type: @weighted ? @counts_data_type : nil)
|
|
383
383
|
rf = result.instance_variable_get(:@full_counts)
|
|
384
384
|
rf[] = @full_counts + other.full_counts
|
|
385
385
|
result.instance_variable_set(:@sample_axis, @sample_axis)
|
|
@@ -401,7 +401,7 @@ class CArray
|
|
|
401
401
|
# arr.shape = fiber_shape + (last M bin axes).
|
|
402
402
|
# Reduce along the last M axes, returns shape fiber_shape (or scalar).
|
|
403
403
|
#
|
|
404
|
-
# `accumulate` preserves
|
|
404
|
+
# `accumulate` preserves the data type (= int64 stays int64, float64 stays float64),
|
|
405
405
|
# unlike `sum` which always lifts to float64. Caveat: int64 overflows at
|
|
406
406
|
# ~9.2e18 (silent wrap); weighted float64 loses precision past 2^53 but
|
|
407
407
|
# does not overflow. Realistic histograms do not hit these limits.
|
|
@@ -497,15 +497,15 @@ class CArray
|
|
|
497
497
|
[sample_ax, channel_ax].sort.reverse.each { |p| fiber_shape.delete_at(p) }
|
|
498
498
|
|
|
499
499
|
# Weighted counts are float64-only (the fused scatter kernel requires
|
|
500
|
-
# float64 weights and float64 counts), so the
|
|
501
|
-
# than derived from the weights' own
|
|
502
|
-
|
|
500
|
+
# float64 weights and float64 counts), so the type is fixed here rather
|
|
501
|
+
# than derived from the weights' own type.
|
|
502
|
+
weights_data_type = (:float64 if weights)
|
|
503
503
|
|
|
504
504
|
h = Histogram.send(:new,
|
|
505
505
|
edges: edges,
|
|
506
506
|
fiber_shape: fiber_shape,
|
|
507
507
|
include_max: include_max,
|
|
508
|
-
|
|
508
|
+
weights_data_type: weights_data_type)
|
|
509
509
|
h.add(arr, axis: axis, weights: weights)
|
|
510
510
|
h
|
|
511
511
|
end
|
data/lib/carray/inspect.rb
CHANGED
|
@@ -7,7 +7,12 @@ class CArray::Inspector # :nodoc:
|
|
|
7
7
|
end
|
|
8
8
|
|
|
9
9
|
# @!visibility private
|
|
10
|
-
|
|
10
|
+
#
|
|
11
|
+
# `abbrev` false renders every element instead of eliding with `...`.
|
|
12
|
+
# It is the one difference between #inspect and #inspect_full: the
|
|
13
|
+
# header and the layout are the same, so there is one renderer rather
|
|
14
|
+
# than two that could drift apart.
|
|
15
|
+
def inspect_string (abbrev: true)
|
|
11
16
|
if @carray.ndim == 0
|
|
12
17
|
raise "can't inspect CArray of ndim == 0"
|
|
13
18
|
end
|
|
@@ -15,7 +20,7 @@ class CArray::Inspector # :nodoc:
|
|
|
15
20
|
class_name = get_class_name()
|
|
16
21
|
type_name = get_type_name()
|
|
17
22
|
shape = get_shape()
|
|
18
|
-
data_spec = get_data_spec(0, Array.new(@carray.ndim){0}, formatter)
|
|
23
|
+
data_spec = get_data_spec(0, Array.new(@carray.ndim){0}, formatter, abbrev)
|
|
19
24
|
info_list = get_info_list()
|
|
20
25
|
output = ["<",
|
|
21
26
|
format("%s.%s(%s)", class_name, type_name, shape.join(",")),
|
|
@@ -49,13 +54,7 @@ class CArray::Inspector # :nodoc:
|
|
|
49
54
|
end
|
|
50
55
|
|
|
51
56
|
def get_shape
|
|
52
|
-
|
|
53
|
-
when CA_OBJ_UNBOUND_REPEAT
|
|
54
|
-
dim = @carray.spec
|
|
55
|
-
else
|
|
56
|
-
dim = @carray.shape
|
|
57
|
-
end
|
|
58
|
-
return dim
|
|
57
|
+
return @carray.shape
|
|
59
58
|
end
|
|
60
59
|
|
|
61
60
|
def get_info_list
|
|
@@ -152,7 +151,7 @@ class CArray::Inspector # :nodoc:
|
|
|
152
151
|
end
|
|
153
152
|
end
|
|
154
153
|
|
|
155
|
-
def get_data_spec (level, idx, formatter)
|
|
154
|
+
def get_data_spec (level, idx, formatter, abbrev = true)
|
|
156
155
|
io = +"[ " # mutable buffer; `<<` below appends into it
|
|
157
156
|
ndim = @carray.ndim
|
|
158
157
|
dim = @carray.shape
|
|
@@ -169,7 +168,7 @@ class CArray::Inspector # :nodoc:
|
|
|
169
168
|
if i != dim[level] - 1
|
|
170
169
|
io << ", "
|
|
171
170
|
end
|
|
172
|
-
if io.length > 48 - 2*level
|
|
171
|
+
if abbrev and io.length > 48 - 2*level
|
|
173
172
|
if i < dim[level] - 1
|
|
174
173
|
io << "..."
|
|
175
174
|
over = true
|
|
@@ -188,24 +187,24 @@ class CArray::Inspector # :nodoc:
|
|
|
188
187
|
end
|
|
189
188
|
else
|
|
190
189
|
over = false
|
|
191
|
-
show = [dim[level], 5].min
|
|
190
|
+
show = abbrev ? [dim[level], 5].min : dim[level]
|
|
192
191
|
show.times do |i|
|
|
193
192
|
idx[level] = i
|
|
194
|
-
io << get_data_spec(level+1, idx, formatter)
|
|
193
|
+
io << get_data_spec(level+1, idx, formatter, abbrev)
|
|
195
194
|
if i < show - 1
|
|
196
195
|
io << ",\n" + " " * (level+1)
|
|
197
196
|
end
|
|
198
|
-
if i >= 2 and dim[level] > 5
|
|
197
|
+
if abbrev and i >= 2 and dim[level] > 5
|
|
199
198
|
break
|
|
200
199
|
end
|
|
201
200
|
end
|
|
202
|
-
if dim[level] > 5
|
|
201
|
+
if abbrev and dim[level] > 5
|
|
203
202
|
io << "... ... ..."
|
|
204
203
|
over = true
|
|
205
204
|
end
|
|
206
205
|
if over
|
|
207
206
|
idx[level] = dim[level] - 1
|
|
208
|
-
io << "\n"+ " " * (level+1) + get_data_spec(level+1, idx, formatter)
|
|
207
|
+
io << "\n"+ " " * (level+1) + get_data_spec(level+1, idx, formatter, abbrev)
|
|
209
208
|
end
|
|
210
209
|
end
|
|
211
210
|
io << " ]"
|
|
@@ -225,6 +224,34 @@ class CArray
|
|
|
225
224
|
return CArray::Inspector.new(self).inspect_string
|
|
226
225
|
end
|
|
227
226
|
|
|
227
|
+
# @overload inspect_full
|
|
228
|
+
# The same description as {#inspect}, with every element rendered
|
|
229
|
+
# instead of the `...` preview.
|
|
230
|
+
#
|
|
231
|
+
# `inspect` abbreviates on purpose -- it is what `p`, `irb` and an
|
|
232
|
+
# error message call, and a million-cell array has to stay readable
|
|
233
|
+
# there. `inspect_full` is for the other moment, when the whole
|
|
234
|
+
# array is the thing you came to look at:
|
|
235
|
+
#
|
|
236
|
+
# puts a.inspect_full
|
|
237
|
+
#
|
|
238
|
+
# The header, the layout and the `_` for a masked cell are
|
|
239
|
+
# `inspect`'s; only the eliding is dropped, so for an array small
|
|
240
|
+
# enough that `inspect` was not abbreviating anything the two give
|
|
241
|
+
# the same string.
|
|
242
|
+
#
|
|
243
|
+
# Note that neither of these is `to_s`, which returns the **raw
|
|
244
|
+
# bytes** of the storage rather than anything printable.
|
|
245
|
+
#
|
|
246
|
+
# The result is one String holding every element, so it is as large
|
|
247
|
+
# as the array is: nothing here is streamed, and a line is as long
|
|
248
|
+
# as the last axis makes it.
|
|
249
|
+
#
|
|
250
|
+
# @return [String]
|
|
251
|
+
def inspect_full
|
|
252
|
+
return CArray::Inspector.new(self).inspect_string(abbrev: false)
|
|
253
|
+
end
|
|
254
|
+
|
|
228
255
|
private
|
|
229
256
|
|
|
230
257
|
def desc
|
data/lib/carray/iterator.rb
CHANGED
|
@@ -14,8 +14,8 @@
|
|
|
14
14
|
# those are NOT part of the common contract because some members legitimately
|
|
15
15
|
# omit them (a window has no map, a group has no within-piece min_index).
|
|
16
16
|
#
|
|
17
|
-
# The
|
|
18
|
-
# kernel_at_addr slot)
|
|
17
|
+
# The 2.0 generic dispatch (calculate / filter / evaluate over a
|
|
18
|
+
# kernel_at_addr slot) was retired in 3.0.
|
|
19
19
|
#
|
|
20
20
|
# ----------------------------------------------------------------------------
|
|
21
21
|
|
|
@@ -36,8 +36,117 @@ class CAIterator
|
|
|
36
36
|
# rather than reading as "no such method" -- that gap is the member's to close.
|
|
37
37
|
# A member that genuinely cannot provide one overrides it to raise with its
|
|
38
38
|
# own reason.
|
|
39
|
+
# The methods below are declared here, so that the family surface is
|
|
40
|
+
# documented in one place; each member generates its own implementation and
|
|
41
|
+
# a member for which one is ill-defined overrides it to raise with a reason.
|
|
42
|
+
# "Piece" is the unit a member iterates over: a slab, a window, a block, a
|
|
43
|
+
# category, or a group.
|
|
44
|
+
#
|
|
45
|
+
# @!method sum
|
|
46
|
+
# Returns the sum of each piece.
|
|
47
|
+
# @return [CArray] one value per piece, shaped {#shape}.
|
|
48
|
+
# @!method accumulate
|
|
49
|
+
# Returns the sum of each piece kept in the source's own `data_type`,
|
|
50
|
+
# wrapping at its width, where {#sum} answers in the type the core
|
|
51
|
+
# promotes to (`:float64` for integers).
|
|
52
|
+
# @return [CArray] one value per piece.
|
|
53
|
+
# @!method prod
|
|
54
|
+
# Returns the product of each piece.
|
|
55
|
+
# @return [CArray] one value per piece.
|
|
56
|
+
# @!method mean
|
|
57
|
+
# Returns the arithmetic mean of each piece.
|
|
58
|
+
# @return [CArray] one value per piece.
|
|
59
|
+
# @!method min
|
|
60
|
+
# Returns the smallest value in each piece.
|
|
61
|
+
# @return [CArray] one value per piece.
|
|
62
|
+
# @!method max
|
|
63
|
+
# Returns the largest value in each piece.
|
|
64
|
+
# @return [CArray] one value per piece.
|
|
65
|
+
# @!method minmax
|
|
66
|
+
# Returns the smallest and largest value of each piece, found in one pass.
|
|
67
|
+
# @return [Array<CArray>] the pair `[min, max]`.
|
|
68
|
+
# @!method variance
|
|
69
|
+
# Returns the sample variance (divisor `n - 1`) of each piece.
|
|
70
|
+
# @return [CArray] one value per piece.
|
|
71
|
+
# @!method variancep
|
|
72
|
+
# Returns the population variance (divisor `n`) of each piece.
|
|
73
|
+
# @return [CArray] one value per piece.
|
|
74
|
+
# @!method stddev
|
|
75
|
+
# Returns the sample standard deviation (divisor `n - 1`) of each piece.
|
|
76
|
+
# @return [CArray] one value per piece.
|
|
77
|
+
# @!method stddevp
|
|
78
|
+
# Returns the population standard deviation (divisor `n`) of each piece.
|
|
79
|
+
# @return [CArray] one value per piece.
|
|
80
|
+
# @!method all
|
|
81
|
+
# Returns whether every cell of each piece is true.
|
|
82
|
+
# @return [CArray] `:boolean`, one value per piece.
|
|
83
|
+
# @!method any
|
|
84
|
+
# Returns whether any cell of each piece is true.
|
|
85
|
+
# @return [CArray] `:boolean`, one value per piece.
|
|
86
|
+
# @!method min_index
|
|
87
|
+
# Returns the position of the smallest value **within** each piece.
|
|
88
|
+
# @return [CArray] one index per piece.
|
|
89
|
+
# @!method max_index
|
|
90
|
+
# Returns the position of the largest value within each piece.
|
|
91
|
+
# @return [CArray] one index per piece.
|
|
92
|
+
# @!method min_addr
|
|
93
|
+
# Returns the flat address **in the source** of the smallest value of each
|
|
94
|
+
# piece -- which source cell holds it, rather than where it sits inside the
|
|
95
|
+
# piece. Use it to read the same cell out of another source-shaped array.
|
|
96
|
+
# @return [CArray] one flat address per piece.
|
|
97
|
+
# @!method max_addr
|
|
98
|
+
# Returns the flat address in the source of the largest value of each piece.
|
|
99
|
+
# @return [CArray] one flat address per piece.
|
|
100
|
+
# @!method wsum(weights)
|
|
101
|
+
# Returns the weighted sum of each piece.
|
|
102
|
+
# @param weights [CArray] one weight per source cell, shaped like the source.
|
|
103
|
+
# @return [CArray] one value per piece.
|
|
104
|
+
# @!method wmean(weights)
|
|
105
|
+
# Returns the weighted mean of each piece.
|
|
106
|
+
# @param weights [CArray] one weight per source cell, shaped like the source.
|
|
107
|
+
# @return [CArray] one value per piece.
|
|
108
|
+
# @!method median
|
|
109
|
+
# Returns the median of each piece.
|
|
110
|
+
# @return [CArray] one value per piece.
|
|
111
|
+
# @!method percentile(*pers)
|
|
112
|
+
# Returns the requested percentile(s) of each piece.
|
|
113
|
+
# @param pers [Array<Numeric>] percentile positions in `0..100`.
|
|
114
|
+
# @return [CArray, Array<CArray>] one array per requested position; a
|
|
115
|
+
# single position returns that array directly.
|
|
116
|
+
# @!method quantile
|
|
117
|
+
# Returns the five-number summary of each piece,
|
|
118
|
+
# `[min, Q1, median, Q3, max]`.
|
|
119
|
+
# @return [Array<CArray>] five arrays, one per position.
|
|
120
|
+
# @!method count(value = nil)
|
|
121
|
+
# Returns a count per piece: with no argument the cells that are not
|
|
122
|
+
# masked, with `UNDEF` the masked cells, and with any other value the
|
|
123
|
+
# cells equal to it.
|
|
124
|
+
# @param value [Object] value to match, or `UNDEF`.
|
|
125
|
+
# @return [CArray] one count per piece.
|
|
126
|
+
# @!method count_not_masked
|
|
127
|
+
# Returns the number of cells of each piece that are not masked.
|
|
128
|
+
# @return [CArray] one count per piece.
|
|
129
|
+
# @!method count_masked
|
|
130
|
+
# Returns the number of masked cells of each piece.
|
|
131
|
+
# @return [CArray] one count per piece.
|
|
132
|
+
# @!method elements
|
|
133
|
+
# Returns the total number of cells in each piece, masked or not.
|
|
134
|
+
# @return [CArray] one count per piece.
|
|
135
|
+
# @!method each
|
|
136
|
+
# Yields each piece in turn as a CArray.
|
|
137
|
+
# @yieldparam piece [CArray]
|
|
138
|
+
# @return [Enumerator, self] an Enumerator when no block is given.
|
|
139
|
+
# @!method reduce(init = nil)
|
|
140
|
+
# Folds the pieces with a block, for a reduction the family does not name.
|
|
141
|
+
# Without `init` the first piece seeds the accumulator.
|
|
142
|
+
# @param init [Object] initial accumulator value.
|
|
143
|
+
# @yieldparam acc [Object] running accumulator.
|
|
144
|
+
# @yieldparam piece [CArray] next piece.
|
|
145
|
+
# @yieldreturn [Object] updated accumulator.
|
|
146
|
+
# @return [Object] the final accumulator.
|
|
39
147
|
[
|
|
40
|
-
:sum, :prod, :mean, :min, :max,
|
|
148
|
+
:sum, :accumulate, :prod, :mean, :min, :max, # tier 1
|
|
149
|
+
:variance, :stddev, :all, :any,
|
|
41
150
|
:variancep, :stddevp, :minmax, # tier 2
|
|
42
151
|
:min_index, :max_index, :min_addr, :max_addr, # position
|
|
43
152
|
:wsum, :wmean, # weighted
|
|
@@ -65,6 +174,41 @@ class CAIterator
|
|
|
65
174
|
# sort_addr. Un-overridden each is simply unavailable, not a contract
|
|
66
175
|
# violation. min_addr / max_addr stay required: a single winner address is
|
|
67
176
|
# well-defined even for an overlapping window.
|
|
177
|
+
# @!method map
|
|
178
|
+
# Returns a source-shaped array built by applying the block to each piece
|
|
179
|
+
# and scattering the result back into that piece's cells.
|
|
180
|
+
# @yieldparam piece [CArray]
|
|
181
|
+
# @yieldreturn [CArray, Numeric] replacement values for the piece.
|
|
182
|
+
# @return [CArray] shaped like the source.
|
|
183
|
+
# @raise [NotImplementedError] for a member whose pieces overlap, where a
|
|
184
|
+
# cell would receive more than one value.
|
|
185
|
+
# @!method sort_addr
|
|
186
|
+
# Returns a source-shaped array whose cells, read piece by piece, give the
|
|
187
|
+
# flat source addresses that put that piece in ascending order.
|
|
188
|
+
# @return [CArray] `:int64`, shaped like the source.
|
|
189
|
+
# @raise [NotImplementedError] for a member whose pieces overlap.
|
|
190
|
+
# @!method cumsum
|
|
191
|
+
# Returns a source-shaped array of the running sum within each piece.
|
|
192
|
+
# @return [CArray] shaped like the source.
|
|
193
|
+
# @raise [NotImplementedError] for a member whose pieces overlap, where a
|
|
194
|
+
# cell has no single running value.
|
|
195
|
+
# @!method cumprod
|
|
196
|
+
# Returns a source-shaped array of the running product within each piece.
|
|
197
|
+
# @return [CArray] shaped like the source.
|
|
198
|
+
# @raise [NotImplementedError] for a member whose pieces overlap.
|
|
199
|
+
# @!method cummax
|
|
200
|
+
# Returns a source-shaped array of the running maximum within each piece.
|
|
201
|
+
# @return [CArray] shaped like the source.
|
|
202
|
+
# @raise [NotImplementedError] for a member whose pieces overlap.
|
|
203
|
+
# @!method cummin
|
|
204
|
+
# Returns a source-shaped array of the running minimum within each piece.
|
|
205
|
+
# @return [CArray] shaped like the source.
|
|
206
|
+
# @raise [NotImplementedError] for a member whose pieces overlap.
|
|
207
|
+
# @!method cumcount
|
|
208
|
+
# Returns a source-shaped array of the running count of cells that are not
|
|
209
|
+
# masked within each piece.
|
|
210
|
+
# @return [CArray] shaped like the source.
|
|
211
|
+
# @raise [NotImplementedError] for a member whose pieces overlap.
|
|
68
212
|
[:map, :sort_addr,
|
|
69
213
|
:cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |name|
|
|
70
214
|
define_method(name) do |*, **, &_blk|
|