carray 3.0.5 → 3.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +233 -0
- data/README.md +1 -1
- data/carray.gemspec +3 -2
- data/ext/ca_axis_dispatch.c +6 -0
- data/ext/ca_axis_group.c +1 -1
- data/ext/ca_categorical_iterator.c +7 -7
- data/ext/ca_kernel_iterator.c +116 -231
- data/ext/ca_kernel_iterator.h +410 -254
- data/ext/ca_obj_bincmp.c +9 -1
- data/ext/ca_obj_binop.c +10 -1
- data/ext/ca_obj_bitfield.c +2 -0
- data/ext/ca_obj_byte_swap.c +2 -0
- data/ext/ca_obj_face.c +2 -0
- data/ext/ca_obj_fake.c +3 -1
- data/ext/ca_obj_grid.c +167 -12
- data/ext/ca_obj_meld.c +69 -113
- data/ext/ca_obj_moncmp.c +9 -1
- data/ext/ca_obj_monop.c +40 -2
- data/ext/ca_obj_object.c +54 -20
- data/ext/ca_obj_reduce.c +15 -1
- data/ext/ca_obj_refer.c +65 -20
- data/ext/ca_obj_remap.c +69 -10
- data/ext/ca_obj_roll.c +1 -0
- data/ext/ca_obj_select.c +4 -1
- data/ext/ca_obj_select_axis.c +1 -0
- data/ext/ca_obj_shift.c +1 -0
- data/ext/ca_obj_stack.c +310 -111
- data/ext/ca_obj_stride.c +131 -45
- data/ext/ca_obj_string.c +18 -2
- data/ext/ca_obj_tile.c +1 -0
- data/ext/ca_obj_triop.c +9 -1
- data/ext/ca_obj_window.c +1 -0
- data/ext/ca_op_ipower.c +8 -8
- data/ext/caframe_csv_reader.c +1 -1
- data/ext/carray.h +2 -1
- data/ext/carray_access.c +27 -64
- data/ext/carray_address_basis.c +134 -0
- data/ext/carray_attribute.c +8 -0
- data/ext/carray_broadcast.c +3 -4
- data/ext/carray_build_flags.h +1 -1
- data/ext/carray_cast.c +8 -11
- data/ext/carray_conversion.c +9 -5
- data/ext/carray_core.c +811 -72
- data/ext/carray_element.c +13 -9
- data/ext/carray_histogram.c +25 -3
- data/ext/carray_internal.h +147 -12
- data/ext/carray_kernels_bincmp.c +2007 -3
- data/ext/carray_kernels_binop.c +25365 -5199
- data/ext/carray_kernels_init.c +8 -8
- data/ext/carray_kernels_monop.c +10696 -3678
- data/ext/carray_kernels_reduce_aggregate.c +4429 -702
- data/ext/carray_kernels_reduce_boolean.c +1533 -15
- data/ext/carray_kernels_reduce_cumulative.c +1808 -248
- data/ext/carray_kernels_reduce_extreme.c +4508 -511
- data/ext/carray_kernels_reduce_variance.c +2284 -152
- data/ext/carray_kernels_scan.c +240 -240
- data/ext/carray_kernels_search.c +4211 -264
- data/ext/carray_kernels_sort.c +367 -450
- data/ext/carray_lazy.c +47 -0
- data/ext/carray_loop.c +1 -1
- data/ext/carray_mask.c +130 -139
- data/ext/carray_median_percentile.c +4 -4
- data/ext/carray_memory_view.c +13 -7
- data/ext/carray_operator.c +147 -44
- data/ext/carray_order.c +3 -2
- data/ext/carray_random.c +4 -1
- data/ext/carray_slab.c +97 -21
- data/ext/carray_sort.c +1 -1
- data/ext/carray_test.c +74 -19
- data/ext/carray_time_text.c +2 -2
- data/ext/extconf.rb +11 -0
- data/ext/mkkernel.rb +346 -94
- data/ext/ruby_carray.c +4 -1
- data/ext/version.h +4 -4
- data/lib/carray/attribute.rb +25 -1
- data/lib/carray/autoload_carray.rb +3 -3
- data/lib/carray/autoload_method_extension.rb +5 -3
- data/lib/carray/axis_group.rb +13 -7
- data/lib/carray/basics.rb +1 -1
- data/lib/carray/block_iterator.rb +17 -3
- data/lib/carray/categorical.rb +6 -8
- data/lib/carray/categorical_iterator.rb +2 -2
- data/lib/carray/complex.rb +2 -2
- data/lib/carray/conditional.rb +3 -2
- data/lib/carray/frame/column_verbs.rb +45 -14
- data/lib/carray/frame/convert.rb +19 -0
- data/lib/carray/frame/csv_parser.rb +32 -32
- data/lib/carray/frame/describe.rb +4 -2
- data/lib/carray/frame/frame.rb +399 -78
- data/lib/carray/frame/group.rb +679 -53
- data/lib/carray/frame/io.rb +209 -71
- data/lib/carray/frame/join.rb +146 -37
- data/lib/carray/frame/records.rb +3 -3
- data/lib/carray/frame/reshape.rb +89 -11
- data/lib/carray/frame/sort.rb +57 -9
- data/lib/carray/frame/verbs.rb +384 -49
- data/lib/carray/frame/windows.rb +44 -6
- data/lib/carray/inspect.rb +7 -2
- data/lib/carray/iterator.rb +32 -0
- data/lib/carray/lazy.rb +17 -8
- data/lib/carray/mask_gap_fill.rb +37 -17
- data/lib/carray/meld_reduce.rb +1 -1
- data/lib/carray/methods/choose.rb +3 -2
- data/lib/carray/methods/meshgrid.rb +3 -1
- data/lib/carray/runtime.rb +47 -8
- data/lib/carray/segment_iterator.rb +23 -8
- data/lib/carray/serialize.rb +34 -17
- data/lib/carray/stack.rb +18 -6
- data/lib/carray/time.rb +5 -0
- data/lib/carray/window_iterator.rb +16 -4
- data/yard-stubs/ca_obj_array.rb +26 -1
- data/yard-stubs/carray_cast.rb +8 -5
- data/yard-stubs/carray_conversion.rb +8 -2
- data/yard-stubs/carray_mask.rb +20 -1
- data/yard-stubs/carray_math.rb +31 -16
- data/yard-stubs/carray_memory_view.rb +7 -0
- data/yard-stubs/carray_test.rb +7 -4
- metadata +18 -4
data/lib/carray/frame/windows.rb
CHANGED
|
@@ -79,8 +79,12 @@ class WindowedFrame
|
|
|
79
79
|
|
|
80
80
|
# The windowed reductions. Without names, a frame of every column whose
|
|
81
81
|
# type has the reduction (the on: and by: columns left out), on the
|
|
82
|
-
# frame's rows and index; given names, the CArray (one)
|
|
83
|
-
#
|
|
82
|
+
# frame's rows and index; given names, the CArray (one), as +df[name]+
|
|
83
|
+
# returns a column, or a frame of those columns (several) on the frame's
|
|
84
|
+
# rows and index. With +at:+ the answers
|
|
85
|
+
# are one per point, not per row, so names or none, the result is a frame
|
|
86
|
+
# with the points as its index, as a group reduction's result is labelled
|
|
87
|
+
# by the groups.
|
|
84
88
|
CArray.send(:define_named_methods, self,
|
|
85
89
|
[:sum, :mean, :min, :max, :prod, :accumulate, :median,
|
|
86
90
|
:variance, :stddev, :variancep, :stddevp], :window_reduction)
|
|
@@ -88,6 +92,38 @@ class WindowedFrame
|
|
|
88
92
|
CArray.send(:define_named_methods, self,
|
|
89
93
|
[:first, :last, :nunique, :count, :all, :any], :window_value)
|
|
90
94
|
|
|
95
|
+
# Each window's +p+-th percentile (0 to 100), as +median+ gives the 50th.
|
|
96
|
+
# The percentage comes first, then the names.
|
|
97
|
+
#
|
|
98
|
+
# w.percentile(90, "rain")
|
|
99
|
+
#
|
|
100
|
+
# @param p [Numeric] the percentage.
|
|
101
|
+
# @param names [Array<String>] the columns; default every column that has
|
|
102
|
+
# a percentile.
|
|
103
|
+
# @return [CAFrame, CArray] as the other windowed reductions return.
|
|
104
|
+
def percentile (p, *names, min_count: nil, fill_value: nil)
|
|
105
|
+
CAFrame.__send__(:check_percentage, p)
|
|
106
|
+
window_reduce(:percentile, names, [p], min_count: min_count, fill_value: fill_value)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Each window's five quartile percentiles, as +CAFrame#quantile+ gives
|
|
110
|
+
# them: a column "x" gives "x_p0", "x_p25", "x_p50", "x_p75" and "x_p100",
|
|
111
|
+
# a frame on the frame's rows and index, or with +at:+ on the points.
|
|
112
|
+
#
|
|
113
|
+
# @param names [Array<String>] the columns; default every column that has
|
|
114
|
+
# a quantile.
|
|
115
|
+
# @return [CAFrame]
|
|
116
|
+
def quantile (*names)
|
|
117
|
+
named = {}
|
|
118
|
+
over_named_columns(:quantile, names, @key_names) do |col, name|
|
|
119
|
+
named[name] = @plan.for_value(col).quantile
|
|
120
|
+
end
|
|
121
|
+
cols = CAFrame.__send__(:quantile_columns, named)
|
|
122
|
+
return CAFrame.new(cols, axis_name: @frame.axis_name, index: @frame.index) unless @at
|
|
123
|
+
key = @frame.__send__(:key_column, @key_names.first, "windows")
|
|
124
|
+
CAFrame.new(cols, axis_name: @key_names.first, index: CAFrame.__send__(:key_labels, key, @at.copy))
|
|
125
|
+
end
|
|
126
|
+
|
|
91
127
|
private
|
|
92
128
|
|
|
93
129
|
def window_reduction (reduction, *names, min_count: nil, fill_value: nil)
|
|
@@ -100,12 +136,14 @@ class WindowedFrame
|
|
|
100
136
|
|
|
101
137
|
# An N-D column (each row a profile) is reduced along the rows, each
|
|
102
138
|
# window keeping the trailing shape, as a group reduces it.
|
|
103
|
-
def window_reduce (reduction, names, **keywords)
|
|
139
|
+
def window_reduce (reduction, names, args = [], **keywords)
|
|
104
140
|
keywords = keywords.compact
|
|
105
|
-
|
|
106
|
-
|
|
141
|
+
named = {}
|
|
142
|
+
out = over_named_columns(reduction, names, @key_names, args) do |col, name|
|
|
143
|
+
named[name] = @plan.for_value(col).public_send(reduction, *args, **keywords)
|
|
107
144
|
end
|
|
108
|
-
|
|
145
|
+
out = named if @at && !names.empty?
|
|
146
|
+
return out if out.is_a?(CArray) && !@at
|
|
109
147
|
if @at
|
|
110
148
|
# Each result has an index of its own, as a group's result has.
|
|
111
149
|
key = @frame.__send__(:key_column, @key_names.first, "windows")
|
data/lib/carray/inspect.rb
CHANGED
|
@@ -262,7 +262,11 @@ class CArray
|
|
|
262
262
|
# count, and a truncated data preview.
|
|
263
263
|
# @return [String]
|
|
264
264
|
def inspect
|
|
265
|
-
|
|
265
|
+
# A cell of an object array can hold an expression over the array, whose
|
|
266
|
+
# inspect shows the array again: that is refused as cyclic, as reading a
|
|
267
|
+
# cell that holds the array is.
|
|
268
|
+
return CArray::Inspector.new(self).inspect_string unless object?
|
|
269
|
+
refuse_reentry("inspect") { CArray::Inspector.new(self).inspect_string }
|
|
266
270
|
end
|
|
267
271
|
|
|
268
272
|
# @overload inspect_full
|
|
@@ -290,7 +294,8 @@ class CArray
|
|
|
290
294
|
#
|
|
291
295
|
# @return [String]
|
|
292
296
|
def inspect_full
|
|
293
|
-
return CArray::Inspector.new(self).inspect_string(abbrev: false)
|
|
297
|
+
return CArray::Inspector.new(self).inspect_string(abbrev: false) unless object?
|
|
298
|
+
refuse_reentry("inspect_full") { CArray::Inspector.new(self).inspect_string(abbrev: false) }
|
|
294
299
|
end
|
|
295
300
|
|
|
296
301
|
private
|
data/lib/carray/iterator.rb
CHANGED
|
@@ -71,6 +71,38 @@ class CAIterator
|
|
|
71
71
|
end
|
|
72
72
|
end
|
|
73
73
|
|
|
74
|
+
# A member that keeps what it built from its source's values or mask (a
|
|
75
|
+
# padded copy, the views over it, counts of the cells present) names those
|
|
76
|
+
# instance variables here. They are dropped when a public method of the
|
|
77
|
+
# member is entered from outside, so every call reads the source as it is
|
|
78
|
+
# then; within one call the helpers it runs share them. What follows from
|
|
79
|
+
# the source's shape alone is kept for good.
|
|
80
|
+
def self.read_source_per_call (*state)
|
|
81
|
+
names = public_instance_methods(false)
|
|
82
|
+
prepend(Module.new do
|
|
83
|
+
module_eval(<<~RUBY, __FILE__, __LINE__ + 1)
|
|
84
|
+
private def forget_source_state
|
|
85
|
+
#{state.inspect}.each { |v| remove_instance_variable(v) if instance_variable_defined?(v) }
|
|
86
|
+
end
|
|
87
|
+
RUBY
|
|
88
|
+
names.each do |name|
|
|
89
|
+
module_eval(<<~RUBY, __FILE__, __LINE__ + 1)
|
|
90
|
+
def #{name} (...)
|
|
91
|
+
return super if @reading_source
|
|
92
|
+
forget_source_state
|
|
93
|
+
@reading_source = true
|
|
94
|
+
begin
|
|
95
|
+
super
|
|
96
|
+
ensure
|
|
97
|
+
@reading_source = false
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
RUBY
|
|
101
|
+
end
|
|
102
|
+
end)
|
|
103
|
+
end
|
|
104
|
+
private_class_method :read_source_per_call
|
|
105
|
+
|
|
74
106
|
def self.reduce_keywords (op, kw) # :nodoc:
|
|
75
107
|
if kw.key?(:axis)
|
|
76
108
|
raise ArgumentError, "#{op}: unknown keyword: :axis (the iterator sets the axes)"
|
data/lib/carray/lazy.rb
CHANGED
|
@@ -384,12 +384,6 @@ class CArray
|
|
|
384
384
|
alias_method word, op
|
|
385
385
|
end
|
|
386
386
|
|
|
387
|
-
# `pow` is the canonical C method (defined by rb_define_method in
|
|
388
|
-
# ext/ca_op_ipower.c) that `**` is rb_define_alias'd to. The `**`
|
|
389
|
-
# dispatch in this file does not reach `pow` through the C alias, so
|
|
390
|
-
# `a.lazy.pow(2)` would fall to eager; `pow` is aliased again here.
|
|
391
|
-
alias_method :pow, :**
|
|
392
|
-
|
|
393
387
|
# `fmax` / `fmin` are rb_define_alias'd to `pmax` / `pmin` at Init
|
|
394
388
|
# time (the same C alias); they are aliased again beside them, in
|
|
395
389
|
# LazyDispatch.
|
|
@@ -474,6 +468,14 @@ class CArray
|
|
|
474
468
|
end
|
|
475
469
|
end
|
|
476
470
|
|
|
471
|
+
# `pow` is the canonical C method (defined by rb_define_method in
|
|
472
|
+
# ext/ca_op_ipower.c) that `**` is rb_define_alias'd to. The `**`
|
|
473
|
+
# dispatch in this file does not reach `pow` through the C alias, so
|
|
474
|
+
# `a.lazy.pow(2)` would fall to eager; `pow` is aliased again here.
|
|
475
|
+
# An alias copies the method as it is at that moment, so this follows
|
|
476
|
+
# the integer-exponent `**` above, not the template one.
|
|
477
|
+
alias_method :pow, :**
|
|
478
|
+
|
|
477
479
|
# coerce: when self is a lazy view and the scalar appears on the LEFT
|
|
478
480
|
# (e.g. `2 * a.lazy`), Ruby's Numeric#* calls a.lazy.coerce(2). The
|
|
479
481
|
# default coerce would unwrap the lazy-ness via eager scalar promotion;
|
|
@@ -495,6 +497,9 @@ class CArray
|
|
|
495
497
|
# equality returning bool, not element-wise. Element-wise equality is
|
|
496
498
|
# `eq` / `feq`.
|
|
497
499
|
#
|
|
500
|
+
# A comparison with UNDEF goes to the eager method, which answers `eq` /
|
|
501
|
+
# `ne` from the mask and refuses the others.
|
|
502
|
+
#
|
|
498
503
|
# `feq` is arity 1 in eager (compile-time FLT_EPSILON / DBL_EPSILON);
|
|
499
504
|
# we mirror that here. Runtime eps is a future extension (struct field
|
|
500
505
|
# already reserved).
|
|
@@ -518,7 +523,9 @@ class CArray
|
|
|
518
523
|
|
|
519
524
|
CArray.send(:define_named_methods, LazyDispatch, LAZY_BINCMP_OP_IDS, template: <<~'RUBY')
|
|
520
525
|
def __NAME__(other)
|
|
521
|
-
if
|
|
526
|
+
if UNDEF.equal?(other)
|
|
527
|
+
super(other)
|
|
528
|
+
elsif lazy_view? || (other.is_a?(CArray) && other.lazy_view?)
|
|
522
529
|
if face_among_operands?(self, other)
|
|
523
530
|
return computed_operand(self).__send__(__method__, computed_operand(other))
|
|
524
531
|
end
|
|
@@ -540,7 +547,9 @@ class CArray
|
|
|
540
547
|
|
|
541
548
|
CArray.send(:define_named_methods, LazyDispatch, LAZY_BINCMP_TOL_OP_IDS, template: <<~'RUBY')
|
|
542
549
|
def __NAME__(other, tol)
|
|
543
|
-
if
|
|
550
|
+
if UNDEF.equal?(other)
|
|
551
|
+
super(other, tol)
|
|
552
|
+
elsif lazy_view? || (other.is_a?(CArray) && other.lazy_view?)
|
|
544
553
|
if face_among_operands?(self, other)
|
|
545
554
|
return computed_operand(self).__send__(__method__, computed_operand(other), tol)
|
|
546
555
|
end
|
data/lib/carray/mask_gap_fill.rb
CHANGED
|
@@ -45,14 +45,14 @@ class CArray
|
|
|
45
45
|
"unmask: pass either a constant fill value or method:, not both"
|
|
46
46
|
end
|
|
47
47
|
held = gap_fill(method, axis, "unmask")
|
|
48
|
-
#
|
|
49
|
-
# as it is).
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
48
|
+
# Write only the cells the fill gave a value (a Face takes a Face of
|
|
49
|
+
# its own kind as it is). Writing every cell would let a view that
|
|
50
|
+
# reaches one cell twice send that cell's unfilled copy back last,
|
|
51
|
+
# masking it again.
|
|
52
|
+
return self unless has_mask?
|
|
53
|
+
filled = is_masked
|
|
54
|
+
filled = filled.and(held.is_not_masked) if held.has_mask?
|
|
55
|
+
self[filled] = held[filled] if filled.any
|
|
56
56
|
return self
|
|
57
57
|
end
|
|
58
58
|
fill.equal?(MASK_FILL_UNSET) ? super() : super(fill)
|
|
@@ -161,7 +161,12 @@ class CArray
|
|
|
161
161
|
end
|
|
162
162
|
|
|
163
163
|
# Fill the masked cells of this 1-D array by linear interpolation against
|
|
164
|
-
# the coordinate `x` (same length
|
|
164
|
+
# the coordinate `x` (same length). `x` need not be in order: the present
|
|
165
|
+
# cells are read as one series in the order of their `x`, so a frame whose
|
|
166
|
+
# index restarts for each station interpolates across the stations by
|
|
167
|
+
# time (interpolate one station at a time to keep them apart). Present
|
|
168
|
+
# cells sharing an `x` keep their row order. A cell whose `x` is masked,
|
|
169
|
+
# NaN or infinite has no place on the line: it is not used and not filled.
|
|
165
170
|
# The one rule both the core (x = cell position) and CAFrame (x = index)
|
|
166
171
|
# use: present cells come back exactly as they are, in this array's own
|
|
167
172
|
# type; a masked cell inside the span of the present ones takes the
|
|
@@ -170,17 +175,34 @@ class CArray
|
|
|
170
175
|
# NaN or Inf stored in a present cell is left alone. A Face goes through
|
|
171
176
|
# its own linear_fetch, which keeps its unit and rounds to its grid.
|
|
172
177
|
def linear_fill_along (x)
|
|
173
|
-
|
|
178
|
+
# A cell whose coordinate is masked, NaN or infinite has no place on the
|
|
179
|
+
# line: it is no point to interpolate between, and is not filled.
|
|
180
|
+
placed = x.is_not_masked
|
|
181
|
+
placed &= x.is_finite if !x.face? && x.float?
|
|
182
|
+
placed = placed.strip_mask(false)
|
|
183
|
+
present = is_not_masked & placed
|
|
174
184
|
# Fewer than two valid points -> nothing to interpolate between; leave
|
|
175
185
|
# every masked cell masked.
|
|
176
186
|
return copy if present.count(true) < 2
|
|
177
|
-
|
|
187
|
+
grid = x[present]
|
|
188
|
+
order = nil
|
|
189
|
+
if ( grid[1..-1] < grid[0..-2] ).any
|
|
190
|
+
order = grid.sort_addr # stable: ties keep their row order
|
|
191
|
+
grid = grid[order]
|
|
192
|
+
end
|
|
193
|
+
addr = grid.linear_section(x) # valid positions -> monotonic grid
|
|
194
|
+
fill = (is_masked & placed & addr.is_finite).strip_mask(false)
|
|
178
195
|
if face?
|
|
179
|
-
# The Face's linear_fetch
|
|
180
|
-
#
|
|
181
|
-
|
|
196
|
+
# The Face's linear_fetch lands on its own grid; only the cells to fill
|
|
197
|
+
# take it, so a present cell comes back as it is.
|
|
198
|
+
known = self[present]
|
|
199
|
+
known = known[order] if order
|
|
200
|
+
out = copy
|
|
201
|
+
out[fill] = known.linear_fetch(addr)[fill] if fill.any
|
|
202
|
+
return out
|
|
182
203
|
end
|
|
183
204
|
v = value[present]
|
|
205
|
+
v = v[order] if order
|
|
184
206
|
# A complex value interpolates its real and imaginary parts.
|
|
185
207
|
interp = v.complex? ? v.linear_fetch(addr) : v.float64.linear_fetch(addr)
|
|
186
208
|
# An integer array takes the nearest integer, half away from zero (as a
|
|
@@ -189,9 +211,7 @@ class CArray
|
|
|
189
211
|
interp = interp.round if integer?
|
|
190
212
|
# Built from the values alone, so a fully filled result carries no mask.
|
|
191
213
|
out = value.copy
|
|
192
|
-
|
|
193
|
-
fill = masked & addr.is_finite
|
|
194
|
-
exterior = masked & fill.not
|
|
214
|
+
exterior = is_masked & fill.not
|
|
195
215
|
out[fill] = interp[fill] if fill.any
|
|
196
216
|
out[exterior] = UNDEF if exterior.any
|
|
197
217
|
out
|
data/lib/carray/meld_reduce.rb
CHANGED
|
@@ -171,7 +171,7 @@ class CAMeld
|
|
|
171
171
|
# more elaborate (min_count / fill_value / keep_axis / multi-axis /
|
|
172
172
|
# mask propagation) punts to super.
|
|
173
173
|
def meld_reduce_fast_path_ok?(kw)
|
|
174
|
-
return false unless CArray.__fast_paths_enabled__
|
|
174
|
+
return false unless CArray.__fast_paths_enabled__(:meld_reduce)
|
|
175
175
|
# The mask of a meld is a meld built in C, with no parents list.
|
|
176
176
|
return false unless parents
|
|
177
177
|
return false unless (kw.keys - [:axis]).empty?
|
|
@@ -41,11 +41,12 @@ class CArray
|
|
|
41
41
|
data_type = ca.empty? ? CA_OBJECT : CArray.result_type(*ca)
|
|
42
42
|
end
|
|
43
43
|
out = template(data_type)
|
|
44
|
+
dst = out.scalar? ? out.flatten : out # a CScalar takes no index
|
|
44
45
|
choices.each_with_index do |v, i|
|
|
45
46
|
s = self.eq(i)
|
|
46
|
-
|
|
47
|
+
dst[s] = !v.is_a?(CArray) ? v : v.scalar? ? v.flatten[s] : v[s]
|
|
47
48
|
end
|
|
48
|
-
|
|
49
|
+
dst[self.is_masked] = UNDEF if has_mask?
|
|
49
50
|
text ? CAString.wrap(out) : out
|
|
50
51
|
end
|
|
51
52
|
|
|
@@ -58,7 +58,9 @@ class CArray
|
|
|
58
58
|
raise ArgumentError,
|
|
59
59
|
"coordinate vector #{k} should be 1-D (got #{a.ndim}-D)"
|
|
60
60
|
end
|
|
61
|
-
a
|
|
61
|
+
# A CScalar takes no index, so a scalar axis is read as the
|
|
62
|
+
# one-cell array it stands for.
|
|
63
|
+
a.is_a?(CScalar) ? a.reshape(1) : a
|
|
62
64
|
end
|
|
63
65
|
|
|
64
66
|
ndim = axes.size
|
data/lib/carray/runtime.rb
CHANGED
|
@@ -7,10 +7,11 @@ class CArray
|
|
|
7
7
|
|
|
8
8
|
# A fast path answers by a different algorithm what a general path also
|
|
9
9
|
# answers. A development build defines this in C, false when run with
|
|
10
|
-
# CARRAY_GENERAL_PATH=1 so the test suite takes the general paths
|
|
10
|
+
# CARRAY_GENERAL_PATH=1 so the test suite takes the general paths, and
|
|
11
|
+
# false for a name listed in CARRAY_SHORTCUTS_OFF (see
|
|
11
12
|
# ext/carray_internal.h); a release build has no switch.
|
|
12
13
|
unless respond_to?(:__fast_paths_enabled__)
|
|
13
|
-
def self.__fast_paths_enabled__ # :nodoc:
|
|
14
|
+
def self.__fast_paths_enabled__ (name = nil) # :nodoc:
|
|
14
15
|
true
|
|
15
16
|
end
|
|
16
17
|
end
|
|
@@ -57,13 +58,14 @@ class CArray
|
|
|
57
58
|
# is more naturally (and exactly, for Rational / BigDecimal weights) a
|
|
58
59
|
# +(self * weights).sum+ composition than a dedicated :object kernel branch.
|
|
59
60
|
#
|
|
60
|
-
# +weights+
|
|
61
|
-
#
|
|
62
|
-
#
|
|
63
|
-
#
|
|
64
|
-
#
|
|
61
|
+
# +weights+ takes the forms the numeric kernel takes: a scalar (or a
|
|
62
|
+
# one-element CArray), an array of the shape of the reduced axes (taken in
|
|
63
|
+
# ascending axis order), or an array of self.shape. +opts+ is the trailing
|
|
64
|
+
# options Hash (axis:/keep_axis:/min_count:/fill_value:) forwarded verbatim
|
|
65
|
+
# by the C escape; it goes straight to +sum+.
|
|
65
66
|
def wsum_object (weights, opts = {}) # :nodoc:
|
|
66
|
-
|
|
67
|
+
weights = weights_over_self_shape(weights, opts[:axis], "wsum")
|
|
68
|
+
return (self * weights).sum(**opts)
|
|
67
69
|
end
|
|
68
70
|
private :wsum_object
|
|
69
71
|
|
|
@@ -76,6 +78,7 @@ class CArray
|
|
|
76
78
|
def wmean_object (weights, opts = {}) # :nodoc:
|
|
77
79
|
axis = opts[:axis]
|
|
78
80
|
keep_axis = opts[:keep_axis]
|
|
81
|
+
weights = weights_over_self_shape(weights, axis, "wmean")
|
|
79
82
|
prod = self * weights
|
|
80
83
|
wv = (weights.is_a?(CArray) ? weights : self * 0 + weights).copy
|
|
81
84
|
wv[prod.is_masked] = UNDEF if prod.has_mask?
|
|
@@ -97,6 +100,42 @@ class CArray
|
|
|
97
100
|
end
|
|
98
101
|
private :wmean_object
|
|
99
102
|
|
|
103
|
+
# The weights of wsum / wmean as a scalar or an array of self.shape. A
|
|
104
|
+
# CScalar is a scalar; any other array, one with a single element included,
|
|
105
|
+
# is held to the shapes. An array of the shape of the reduced axes (in ascending axis order) is
|
|
106
|
+
# repeated along the kept axes; any other shape is refused, with the same
|
|
107
|
+
# message as the numeric kernel.
|
|
108
|
+
def weights_over_self_shape (weights, axis, name) # :nodoc:
|
|
109
|
+
unless weights.is_a?(CArray)
|
|
110
|
+
return weights
|
|
111
|
+
end
|
|
112
|
+
if weights.scalar?
|
|
113
|
+
w = weights[0]
|
|
114
|
+
# A boolean cell reads as true / false; as a weight it is 1 / 0.
|
|
115
|
+
w = (w ? 1 : 0) if weights.data_type == CA_BOOLEAN && !UNDEF.equal?(w)
|
|
116
|
+
return w
|
|
117
|
+
end
|
|
118
|
+
if weights.shape == self.shape
|
|
119
|
+
return weights
|
|
120
|
+
end
|
|
121
|
+
axes = case axis
|
|
122
|
+
when nil then (0...self.ndim).to_a
|
|
123
|
+
when Integer then [axis]
|
|
124
|
+
else axis
|
|
125
|
+
end
|
|
126
|
+
axes = axes.map { |a| ( a < 0 ) ? a + self.ndim : a }.sort
|
|
127
|
+
reduced_shape = axes.map { |a| self.shape[a] }
|
|
128
|
+
unless weights.shape == reduced_shape
|
|
129
|
+
raise ArgumentError,
|
|
130
|
+
"#{name}: weights of shape #{weights.shape.inspect} not accepted " \
|
|
131
|
+
"(expected a scalar, the shape of the reduced axes " \
|
|
132
|
+
"#{reduced_shape.inspect}, or self.shape #{self.shape.inspect})"
|
|
133
|
+
end
|
|
134
|
+
spread = self.shape.each_index.map { |i| axes.include?(i) ? self.shape[i] : 1 }
|
|
135
|
+
return weights.reshape(*spread).broadcast_to(*self.shape)
|
|
136
|
+
end
|
|
137
|
+
private :weights_over_self_shape
|
|
138
|
+
|
|
100
139
|
# Define on +mod+ one method per name in +names+, each a `def` that passes
|
|
101
140
|
# its own name and its arguments to the private method +to+:
|
|
102
141
|
#
|
|
@@ -337,9 +337,17 @@ class CASegmentIterator < CAIterator
|
|
|
337
337
|
# with the distinctness of `CArray#nunique` (every NaN is one value,
|
|
338
338
|
# -0.0 is 0.0). A segment with no present value counts 0.
|
|
339
339
|
# @return [CArray]
|
|
340
|
+
# Coding every cell costs the same for every cell, and the segment-by-
|
|
341
|
+
# segment way the same for every segment, so long segments are faster one
|
|
342
|
+
# by one: from an average of about 100 cells (1M cells on an M2 Max).
|
|
343
|
+
NUNIQUE_KERNEL_MAX_LENGTH = 100
|
|
344
|
+
|
|
340
345
|
def nunique (axis: nil)
|
|
341
346
|
return axis_by_core(axis, :nunique) if axis
|
|
342
|
-
|
|
347
|
+
if ! CArray.__fast_paths_enabled__(:segment_nunique) ||
|
|
348
|
+
grouped.elements >= NUNIQUE_KERNEL_MAX_LENGTH * @k
|
|
349
|
+
return per_segment(CA_INT64) { |s| s.nunique }
|
|
350
|
+
end
|
|
343
351
|
out = CArray.int64(@k)
|
|
344
352
|
return out if grouped.elements.zero?
|
|
345
353
|
# Each cell's value becomes a code, so a distinct (segment, value) pair is
|
|
@@ -476,9 +484,8 @@ class CASegmentIterator < CAIterator
|
|
|
476
484
|
end
|
|
477
485
|
|
|
478
486
|
# @overload wmean(weights)
|
|
479
|
-
# Per-segment weighted mean as float64, matching `CArray#wmean`.
|
|
480
|
-
# segment
|
|
481
|
-
# (core's 0/0 contract).
|
|
487
|
+
# Per-segment weighted mean as float64, matching `CArray#wmean`. An empty
|
|
488
|
+
# segment, and one whose weights sum to zero, is masked.
|
|
482
489
|
# @param weights [CArray]
|
|
483
490
|
# @return [CArray]
|
|
484
491
|
def wmean (weights, axis: nil)
|
|
@@ -749,7 +756,7 @@ class CASegmentIterator < CAIterator
|
|
|
749
756
|
# cumcount in int64 -- which is not the core's answer for every data type
|
|
750
757
|
# (the core counts a boolean's running sum in uint64).
|
|
751
758
|
def kernel_scans? (op, answer)
|
|
752
|
-
return false unless CArray.__fast_paths_enabled__
|
|
759
|
+
return false unless CArray.__fast_paths_enabled__(:segment_scan_kernel)
|
|
753
760
|
return false if @value.face? || answer.face? || @value.complex?
|
|
754
761
|
kernel_type =
|
|
755
762
|
case op
|
|
@@ -906,9 +913,17 @@ class CASegmentIterator < CAIterator
|
|
|
906
913
|
# segment's first or last slot and kept when it falls inside the segment.
|
|
907
914
|
# A read-only Face cannot take UNDEF cell by cell, so its answers are
|
|
908
915
|
# collected segment by segment, as its other reductions are.
|
|
916
|
+
#
|
|
917
|
+
# The running minimum or maximum costs the same for every cell, and the
|
|
918
|
+
# segment-by-segment way the same for every segment, so long segments
|
|
919
|
+
# are faster one by one: from an average of about 300 cells for :first
|
|
920
|
+
# and 2000 for :last (1M cells on an M2 Max).
|
|
921
|
+
EDGE_KERNEL_MAX_LENGTH = { first: 300, last: 2000 }.freeze
|
|
922
|
+
|
|
909
923
|
def edge_values (op)
|
|
910
924
|
n = grouped.elements
|
|
911
|
-
if n.zero? || (@value.face? && ! face_output?) || ! CArray.__fast_paths_enabled__
|
|
925
|
+
if n.zero? || (@value.face? && ! face_output?) || ! CArray.__fast_paths_enabled__(:segment_edge_kernel) ||
|
|
926
|
+
n >= EDGE_KERNEL_MAX_LENGTH.fetch(op) * @k
|
|
912
927
|
return per_segment(core_reduce_kind(op)) { |s| s.public_send(op) }
|
|
913
928
|
end
|
|
914
929
|
slot = CArray.int64(n).seq!
|
|
@@ -955,7 +970,7 @@ class CASegmentIterator < CAIterator
|
|
|
955
970
|
# CARRAY_GENERAL_PATH set reduces every group by the core instead, which
|
|
956
971
|
# is how the two are compared.
|
|
957
972
|
def kernel_reads? (data_type = grouped.data_type)
|
|
958
|
-
MONOID_TYPES.include?(data_type) && CArray.__fast_paths_enabled__
|
|
973
|
+
MONOID_TYPES.include?(data_type) && CArray.__fast_paths_enabled__(:segment_reduce_kernel)
|
|
959
974
|
end
|
|
960
975
|
|
|
961
976
|
def moments
|
|
@@ -991,7 +1006,7 @@ class CASegmentIterator < CAIterator
|
|
|
991
1006
|
def all_any
|
|
992
1007
|
return @all_any if defined?(@all_any)
|
|
993
1008
|
@all_any =
|
|
994
|
-
if grouped.data_type == CA_BOOLEAN && CArray.__fast_paths_enabled__
|
|
1009
|
+
if grouped.data_type == CA_BOOLEAN && CArray.__fast_paths_enabled__(:segment_all_any_kernel)
|
|
995
1010
|
a = CArray.boolean(@k)
|
|
996
1011
|
o = CArray.boolean(@k)
|
|
997
1012
|
grouped.send(:__reduceat_all_any__, @offsets, a, o)
|
data/lib/carray/serialize.rb
CHANGED
|
@@ -685,11 +685,11 @@ class CArray
|
|
|
685
685
|
|
|
686
686
|
end
|
|
687
687
|
|
|
688
|
-
# Marshal of a view and a scalar: the {CArray#marshal_dump} payload,
|
|
689
|
-
# back by the class method rather than into an empty instance. A
|
|
690
|
-
# back as a plain array (a copy owns its
|
|
691
|
-
#
|
|
692
|
-
[CAView, CAWrap, CScalar].each do |klass|
|
|
688
|
+
# Marshal of a view, a mask and a scalar: the {CArray#marshal_dump} payload,
|
|
689
|
+
# loaded back by the class method rather than into an empty instance. A
|
|
690
|
+
# view, a wrap and a mask come back as a plain array (a copy owns its
|
|
691
|
+
# memory, as #copy gives); a scalar comes back a scalar.
|
|
692
|
+
[CAView, CAWrap, CScalar, CArrayMask, CAObjectMask].each do |klass|
|
|
693
693
|
klass.class_eval do
|
|
694
694
|
# A Face other than a record is refused: the payload would carry its
|
|
695
695
|
# storage without what makes it the Face (a unit, labels), and load
|
|
@@ -703,28 +703,45 @@ end
|
|
|
703
703
|
"dump its values with to_type(:object)" +
|
|
704
704
|
(storage ? ", or its storage, which ##{storage} gives" : "")
|
|
705
705
|
end
|
|
706
|
+
return dump_payload unless data_type == CA_OBJECT
|
|
707
|
+
# An object view dumps its cells by a nested Marshal.dump, which does
|
|
708
|
+
# not know the objects the outer dump has written. A cell holding the
|
|
709
|
+
# view again (directly or through other views) would dump it without
|
|
710
|
+
# end, so dumping the same view from inside its own dump is refused,
|
|
711
|
+
# as reading such a cell is.
|
|
712
|
+
refuse_reentry("Marshal.dump") { dump_payload }
|
|
713
|
+
end
|
|
714
|
+
|
|
715
|
+
private
|
|
716
|
+
|
|
717
|
+
# Marshal writes the instance variables of an object that has _dump
|
|
718
|
+
# after the String it returns, unless the String has instance
|
|
719
|
+
# variables of its own. A view keeps its parent in one, so a view of
|
|
720
|
+
# a large array would carry the whole parent, and a parent holding the
|
|
721
|
+
# view in a cell would be an object cycle. The String is given one of
|
|
722
|
+
# its own, which _load ignores.
|
|
723
|
+
def dump_payload
|
|
706
724
|
Marshal.dump(CArray.instance_method(:marshal_dump).bind_call(self))
|
|
725
|
+
.tap { |s| s.instance_variable_set(:@carray_payload, true) }
|
|
707
726
|
end
|
|
708
727
|
end
|
|
709
728
|
end
|
|
710
729
|
|
|
711
|
-
|
|
730
|
+
[CAView, CArrayMask, CAObjectMask].each do |klass|
|
|
712
731
|
# @return [CArray] the plain array a {#_dump} string describes.
|
|
713
|
-
def
|
|
714
|
-
|
|
732
|
+
# Defined with def, not define_singleton_method: a block's method keeps
|
|
733
|
+
# the block's self and cannot be called from another Ractor.
|
|
734
|
+
klass.singleton_class.class_eval do
|
|
735
|
+
def _load (str)
|
|
736
|
+
CArray.send(:array_from_marshal_payload, Marshal.load(str))
|
|
737
|
+
end
|
|
715
738
|
end
|
|
716
739
|
end
|
|
717
740
|
|
|
718
741
|
class CAWrap
|
|
719
|
-
# A wrap borrows memory, which Marshal cannot carry to another
|
|
720
|
-
#
|
|
721
|
-
#
|
|
722
|
-
def _dump (_level)
|
|
723
|
-
raise TypeError,
|
|
724
|
-
"Marshal cannot carry a #{self.class}: it borrows its memory; " \
|
|
725
|
-
"dump .copy instead"
|
|
726
|
-
end
|
|
727
|
-
|
|
742
|
+
# A wrap borrows its memory, which Marshal cannot carry to another
|
|
743
|
+
# process: {#_dump} above writes its values, and the load gives a plain
|
|
744
|
+
# array that owns a copy of them, as it does for a view of the wrap.
|
|
728
745
|
def self._load (str)
|
|
729
746
|
CArray.send(:array_from_marshal_payload, Marshal.load(str))
|
|
730
747
|
end
|
data/lib/carray/stack.rb
CHANGED
|
@@ -106,11 +106,14 @@ class CArray
|
|
|
106
106
|
# describe a segment sequence, so absorbing their parents keeps chain
|
|
107
107
|
# depth at 1 (avoids 2-level xfer_all / reduce chains through the
|
|
108
108
|
# intermediate CAMeld). A CAMeld with a different meld_axis is left
|
|
109
|
-
# intact — its segment structure is orthogonal.
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
109
|
+
# intact — its segment structure is orthogonal. So is a read-only one
|
|
110
|
+
# (frozen or flagged): writes through it are refused, and absorbing it
|
|
111
|
+
# would drop that. The mask of a meld is a CAMeld with no parents of
|
|
112
|
+
# its own; it is passed on, to be refused as a part.
|
|
113
|
+
absorbed = ->(a) { a.is_a?(CAMeld) && a.meld_axis == axis_norm &&
|
|
114
|
+
!a.read_only? && !a.mask_array? }
|
|
115
|
+
if arrays.any?(&absorbed)
|
|
116
|
+
arrays = arrays.flat_map { |a| absorbed.(a) ? a.parents : [a] }
|
|
114
117
|
end
|
|
115
118
|
CAMeld.new(arrays, axis: axis_norm)
|
|
116
119
|
end
|
|
@@ -291,10 +294,19 @@ class CAStack
|
|
|
291
294
|
# preserving `self.k_axis`. Distinct from `CArray#stack`, which
|
|
292
295
|
# always treats `self` as a single parent of a new K-stack.
|
|
293
296
|
# @param others [Array<CArray>] extra parents to append.
|
|
297
|
+
# A read-only stack hands on read-only views of its parents, so
|
|
298
|
+
# writes into its cells stay refused.
|
|
294
299
|
# @return [CAStack]
|
|
295
300
|
# @raise [ArgumentError] when no `others` are given.
|
|
296
301
|
def append (*others)
|
|
297
302
|
raise ArgumentError, "append: at least one parent required" if others.empty?
|
|
298
|
-
|
|
303
|
+
if mask_array?
|
|
304
|
+
raise ArgumentError, "append: a mask array can have no mask and can not be a part"
|
|
305
|
+
end
|
|
306
|
+
mine = self.parents
|
|
307
|
+
if read_only?
|
|
308
|
+
mine = mine.map { |p| p[].tap(&:set_read_only_flag) }
|
|
309
|
+
end
|
|
310
|
+
CArray.stack(mine + others, axis: self.k_axis)
|
|
299
311
|
end
|
|
300
312
|
end
|
data/lib/carray/time.rb
CHANGED
|
@@ -1122,6 +1122,9 @@ class CATime
|
|
|
1122
1122
|
# @raise [RangeError] when the widened ticks overflow int64.
|
|
1123
1123
|
def to_unit(unit)
|
|
1124
1124
|
to = CATime::Resolution.parse(unit)
|
|
1125
|
+
# The conversion hands back the storage itself when the unit does not
|
|
1126
|
+
# change; the result is a new array whichever unit is asked for.
|
|
1127
|
+
return face_storage.copy.time(unit: to) if to == self.unit
|
|
1125
1128
|
CATimeUnitAlgebra.convert_instant_floor(face_storage, self.unit, to).time(unit: to)
|
|
1126
1129
|
end
|
|
1127
1130
|
|
|
@@ -1830,6 +1833,8 @@ class CATimedelta
|
|
|
1830
1833
|
# @raise [RangeError] when the widened ticks overflow int64.
|
|
1831
1834
|
def to_unit(unit)
|
|
1832
1835
|
to = CATime::Resolution.parse(unit)
|
|
1836
|
+
# As CATime#to_unit: a new array even when the unit does not change.
|
|
1837
|
+
return face_storage.copy.timedelta(unit: to) if to == self.unit
|
|
1833
1838
|
CATimeUnitAlgebra.convert_scale_trunc(face_storage, self.unit, to).timedelta(unit: to)
|
|
1834
1839
|
end
|
|
1835
1840
|
|