carray 3.0.6 → 3.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +110 -0
- data/README.md +24 -3
- data/carray.gemspec +1 -1
- data/ext/ca_axis_group.c +6 -6
- data/ext/ca_binop_dispatch.c +8 -0
- data/ext/ca_binop_dispatch.h +1 -0
- data/ext/ca_categorical_iterator.c +7 -7
- data/ext/ca_kernel_iterator.h +6 -4
- data/ext/ca_obj_array.c +4 -4
- data/ext/ca_obj_bincmp.c +19 -3
- data/ext/ca_obj_binop.c +36 -11
- data/ext/ca_obj_bitarray.c +2 -4
- data/ext/ca_obj_bitfield.c +86 -4
- data/ext/ca_obj_byte_swap.c +2 -4
- data/ext/ca_obj_const_string.c +422 -257
- data/ext/ca_obj_face.c +71 -33
- data/ext/ca_obj_fake.c +112 -13
- data/ext/ca_obj_farray.c +7 -1
- data/ext/ca_obj_fixlen_string.c +51 -17
- data/ext/ca_obj_grid.c +12 -7
- data/ext/ca_obj_meld.c +5 -5
- data/ext/ca_obj_monop.c +65 -39
- data/ext/ca_obj_object.c +9 -9
- data/ext/ca_obj_reduce.c +2 -4
- data/ext/ca_obj_refer.c +104 -10
- data/ext/ca_obj_repeat.c +1 -1
- data/ext/ca_obj_roll.c +2 -4
- data/ext/ca_obj_select.c +8 -6
- data/ext/ca_obj_select_axis.c +8 -6
- data/ext/ca_obj_stack.c +5 -5
- data/ext/ca_obj_stride.c +46 -8
- data/ext/ca_obj_string.c +2 -2
- data/ext/ca_obj_tile.c +2 -4
- data/ext/ca_obj_triop.c +15 -12
- data/ext/ca_obj_window.c +22 -15
- data/ext/ca_op_ipower.c +63 -15
- data/ext/ca_op_powi.h +6 -29
- data/ext/ca_sort_kernels.h +3 -0
- data/ext/ca_sweep_engine.c +90 -23
- data/ext/ca_sweep_engine.h +5 -6
- data/ext/caframe_csv_reader.c +1 -1
- data/ext/carray.h +3 -1
- data/ext/carray_access.c +98 -39
- data/ext/carray_address_basis.c +98 -24
- data/ext/carray_broadcast.c +2 -2
- data/ext/carray_call_cfunc.c +5 -8
- data/ext/carray_cast.c +311 -37
- data/ext/carray_cast_func.rb +37 -5
- data/ext/carray_conversion.c +56 -2
- data/ext/carray_copy.c +24 -0
- data/ext/carray_core.c +69 -6
- data/ext/carray_count.c +28 -9
- data/ext/carray_element.c +9 -3
- data/ext/carray_factorize.c +11 -11
- data/ext/carray_histogram.c +2 -2
- data/ext/carray_index_classifier.c +7 -5
- data/ext/carray_internal.h +94 -0
- data/ext/carray_kernels_binop.c +55 -55
- data/ext/carray_kernels_init.c +14 -14
- data/ext/carray_kernels_monop.c +90 -140
- data/ext/carray_lazy.c +16 -1
- data/ext/carray_loop.c +9 -2
- data/ext/carray_mask.c +118 -11
- data/ext/carray_median_percentile.c +13 -8
- data/ext/carray_memory_view.c +41 -22
- data/ext/carray_operator.c +72 -11
- data/ext/carray_order.c +52 -9
- data/ext/carray_partition.c +1 -1
- data/ext/carray_random.c +1 -1
- data/ext/carray_round.c +24 -1
- data/ext/carray_scatter.c +4 -0
- data/ext/carray_segment.c +1 -1
- data/ext/carray_slab.c +13 -4
- data/ext/carray_sort.c +6 -3
- data/ext/carray_sort_kernel.c +24 -0
- data/ext/carray_struct.c +21 -6
- data/ext/carray_test.c +85 -39
- data/ext/carray_time_text.c +6 -0
- data/ext/carray_utils.c +124 -54
- data/ext/mk_call_cfunc.rb +6 -11
- data/ext/mkkernel.rb +37 -27
- data/ext/version.h +4 -4
- data/lib/carray/axis_group.rb +9 -6
- data/lib/carray/basics.rb +26 -8
- data/lib/carray/bincount_nd.rb +8 -2
- data/lib/carray/block_iterator.rb +48 -21
- data/lib/carray/categorical.rb +21 -8
- data/lib/carray/categorical_iterator.rb +2 -1
- data/lib/carray/conditional.rb +52 -4
- data/lib/carray/const_string.rb +74 -0
- data/lib/carray/construct.rb +2 -2
- data/lib/carray/core_extensions.rb +19 -68
- data/lib/carray/data_type_extension.rb +1 -0
- data/lib/carray/fixlen_string.rb +83 -1
- data/lib/carray/frame/column_verbs.rb +61 -6
- data/lib/carray/frame/concat.rb +50 -6
- data/lib/carray/frame/csv_parser.rb +13 -4
- data/lib/carray/frame/frame.rb +38 -6
- data/lib/carray/frame/group.rb +67 -40
- data/lib/carray/frame/io.rb +10 -5
- data/lib/carray/frame/join.rb +20 -6
- data/lib/carray/frame/records.rb +6 -4
- data/lib/carray/frame/reshape.rb +25 -5
- data/lib/carray/frame/verbs.rb +78 -25
- data/lib/carray/frame/windows.rb +30 -6
- data/lib/carray/fuse_source.rb +4 -2
- data/lib/carray/iterator.rb +8 -0
- data/lib/carray/key_windows.rb +10 -4
- data/lib/carray/lazy.rb +12 -17
- data/lib/carray/methods/bin.rb +3 -0
- data/lib/carray/methods/composition.rb +1 -1
- data/lib/carray/methods/factorize.rb +2 -0
- data/lib/carray/methods/gather_nd.rb +4 -1
- data/lib/carray/methods/is_in.rb +23 -3
- data/lib/carray/methods/locate_addr.rb +63 -12
- data/lib/carray/methods/repeat.rb +11 -1
- data/lib/carray/record_order.rb +35 -6
- data/lib/carray/runtime.rb +10 -0
- data/lib/carray/segment_iterator.rb +19 -16
- data/lib/carray/serialize.rb +22 -0
- data/lib/carray/slab_iterator.rb +1 -1
- data/lib/carray/stack.rb +3 -3
- data/lib/carray/string.rb +6 -6
- data/lib/carray/string_operation_extension.rb +25 -4
- data/lib/carray/struct_builder.rb +4 -1
- data/lib/carray/time.rb +190 -46
- data/lib/carray/window_iterator.rb +48 -12
- data/yard-stubs/ca_obj_bitfield.rb +4 -1
- data/yard-stubs/ca_obj_refer.rb +3 -0
- data/yard-stubs/ca_obj_window.rb +2 -1
- data/yard-stubs/carray_access.rb +4 -2
- data/yard-stubs/carray_cast.rb +6 -0
- data/yard-stubs/carray_count.rb +6 -3
- data/yard-stubs/carray_math.rb +5 -0
- data/yard-stubs/carray_order.rb +6 -0
- data/yard-stubs/carray_scatter.rb +2 -1
- data/yard-stubs/carray_test.rb +18 -5
- metadata +2 -2
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
# Provides:
|
|
10
10
|
# - Postfix math on Float/Integer/Rational (sqrt, exp, log, sin, ...)
|
|
11
11
|
# enabling scalar/CArray polymorphism: the same expression
|
|
12
|
-
# `(0.0415*(t-218.8)).tanh` works whether t is a Float or a CArray
|
|
12
|
+
# `(0.0415*(t-218.8)).tanh` works whether t is a Float or a CArray,
|
|
13
|
+
# and answers as the array does (`(-1.0).sqrt` is NaN, not an exception).
|
|
13
14
|
# - The same postfix math on Complex, so an expression written for a
|
|
14
15
|
# complex CArray still reads for a single cell taken out of it.
|
|
15
16
|
# - Angle normalisation on Numeric (deg_360 / deg_180 / rad_2pi / rad_pi).
|
|
@@ -43,11 +44,18 @@ class CArray
|
|
|
43
44
|
# lexically scoped: nothing changes for code that does not opt in.
|
|
44
45
|
module CoreExtensions
|
|
45
46
|
|
|
47
|
+
# What a real scalar answers through the float64 kernel of the array form.
|
|
48
|
+
# The refinement exists so that one expression reads for a scalar and for
|
|
49
|
+
# an array, so it answers as the array does: NaN outside a function's
|
|
50
|
+
# domain where Ruby's Math raises, the sign of a zero, and the accuracy of
|
|
51
|
+
# expm1 and log1p near zero. The cost is one CScalar per call.
|
|
52
|
+
#
|
|
46
53
|
# @!visibility private
|
|
47
54
|
MATH_METHODS = %i[
|
|
48
55
|
sqrt exp log log10
|
|
49
56
|
sin cos tan sinh cosh tanh
|
|
50
57
|
asin acos atan asinh acosh atanh
|
|
58
|
+
expm1 log1p rsqrt signbit
|
|
51
59
|
].freeze
|
|
52
60
|
|
|
53
61
|
[Float, Integer, Rational].each do |klass|
|
|
@@ -55,7 +63,7 @@ class CArray
|
|
|
55
63
|
MATH_METHODS.each do |m|
|
|
56
64
|
module_eval <<~RUBY, __FILE__, __LINE__ + 1
|
|
57
65
|
def #{m}
|
|
58
|
-
|
|
66
|
+
CA_FLOAT64(self).#{m}[0]
|
|
59
67
|
end
|
|
60
68
|
RUBY
|
|
61
69
|
end
|
|
@@ -68,26 +76,12 @@ class CArray
|
|
|
68
76
|
def distance(other)
|
|
69
77
|
(self.to_f - other.to_f).abs
|
|
70
78
|
end
|
|
71
|
-
# M.1 (PyTorch alignment) scalar polymorphism additions.
|
|
72
79
|
# `trunc` is left to Ruby's built-in semantics (Float#truncate
|
|
73
80
|
# returns Integer when arg=0) — its CArray counterpart preserves
|
|
74
81
|
# data type, so scalar-side use should call .to_f.trunc explicitly.
|
|
75
|
-
# Math.expm1 / Math.log1p don't exist in stdlib; hand-roll.
|
|
76
|
-
def expm1
|
|
77
|
-
Math.exp(self) - 1.0
|
|
78
|
-
end
|
|
79
|
-
def log1p
|
|
80
|
-
Math.log(1.0 + self)
|
|
81
|
-
end
|
|
82
|
-
def rsqrt
|
|
83
|
-
1.0 / Math.sqrt(self)
|
|
84
|
-
end
|
|
85
82
|
def square
|
|
86
83
|
self * self
|
|
87
84
|
end
|
|
88
|
-
def signbit
|
|
89
|
-
self.negative?
|
|
90
|
-
end
|
|
91
85
|
end
|
|
92
86
|
end
|
|
93
87
|
|
|
@@ -109,14 +103,9 @@ class CArray
|
|
|
109
103
|
square rsqrt
|
|
110
104
|
].freeze
|
|
111
105
|
|
|
112
|
-
# Complex
|
|
113
|
-
#
|
|
114
|
-
#
|
|
115
|
-
# need their own implementation.
|
|
116
|
-
#
|
|
117
|
-
# Each one runs the value through a one-element cmplx128 CScalar, so
|
|
118
|
-
# the answer comes from the very kernel the array form would have
|
|
119
|
-
# used. That is the point: the branch cuts of csqrt / clog / casin /
|
|
106
|
+
# Complex goes the same way, through a one-element cmplx128 CScalar, so
|
|
107
|
+
# the answer comes from the very kernel the array form would have used.
|
|
108
|
+
# That is the point: the branch cuts of csqrt / clog / casin /
|
|
120
109
|
# cacosh / catanh, and the sign of a zero on either side of them, are
|
|
121
110
|
# whatever the platform's C99 complex.h says they are, and they are
|
|
122
111
|
# not the same everywhere. On this machine, for instance,
|
|
@@ -143,50 +132,12 @@ class CArray
|
|
|
143
132
|
end
|
|
144
133
|
|
|
145
134
|
refine Numeric do
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
(fa - fa.ceil + 1) * 360.0
|
|
153
|
-
end
|
|
154
|
-
end
|
|
155
|
-
|
|
156
|
-
def deg_180
|
|
157
|
-
a = self.to_f
|
|
158
|
-
fa = (a + 180.0) / 360.0
|
|
159
|
-
b = if a >= 0
|
|
160
|
-
(fa - fa.floor) * 360.0 - 180.0
|
|
161
|
-
else
|
|
162
|
-
(fa - fa.ceil) * 360.0 - 180.0
|
|
163
|
-
end
|
|
164
|
-
b += 360.0 if b <= -180.0
|
|
165
|
-
b
|
|
166
|
-
end
|
|
167
|
-
|
|
168
|
-
def rad_2pi
|
|
169
|
-
a = self.to_f
|
|
170
|
-
two_pi = 2 * Math::PI
|
|
171
|
-
fa = a / two_pi
|
|
172
|
-
if a >= 0
|
|
173
|
-
(fa - fa.floor) * two_pi
|
|
174
|
-
else
|
|
175
|
-
(fa - fa.ceil + 1) * two_pi
|
|
176
|
-
end
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
def rad_pi
|
|
180
|
-
a = self.to_f
|
|
181
|
-
two_pi = 2 * Math::PI
|
|
182
|
-
fa = (a + Math::PI) / two_pi
|
|
183
|
-
b = if a >= 0
|
|
184
|
-
(fa - fa.floor) * two_pi - Math::PI
|
|
185
|
-
else
|
|
186
|
-
(fa - fa.ceil) * two_pi - Math::PI
|
|
187
|
-
end
|
|
188
|
-
b += two_pi if b <= -Math::PI
|
|
189
|
-
b
|
|
135
|
+
%i[deg_360 deg_180 rad_2pi rad_pi].each do |m|
|
|
136
|
+
module_eval <<~RUBY, __FILE__, __LINE__ + 1
|
|
137
|
+
def #{m}
|
|
138
|
+
CA_FLOAT64(self.to_f).#{m}[0]
|
|
139
|
+
end
|
|
140
|
+
RUBY
|
|
190
141
|
end
|
|
191
142
|
|
|
192
143
|
def eq(other)
|
|
@@ -151,6 +151,7 @@ class CArray
|
|
|
151
151
|
# @param n [Integer] number of samples.
|
|
152
152
|
# @return [CArray]
|
|
153
153
|
def linspace (x1, x2, n = 100)
|
|
154
|
+
raise TypeError, "linspace: n must be an Integer (got #{n.class})" unless n.is_a?(Integer)
|
|
154
155
|
data_type = self::DataType
|
|
155
156
|
unless data_type
|
|
156
157
|
guess = guess_data_type_from_values(x1, x2)
|
data/lib/carray/fixlen_string.rb
CHANGED
|
@@ -87,7 +87,7 @@ class CArray
|
|
|
87
87
|
values.each_with_index do |s, i|
|
|
88
88
|
entity[i] = s.nil? ? UNDEF : s.to_s
|
|
89
89
|
end
|
|
90
|
-
CAFixlenString.wrap(entity)
|
|
90
|
+
CAFixlenString.wrap(entity, encoding: CAFixlenString.__send__(:encoding_of_strings, values))
|
|
91
91
|
end
|
|
92
92
|
|
|
93
93
|
end
|
|
@@ -95,6 +95,88 @@ end
|
|
|
95
95
|
# Mutable fixed-width String Face: L1 ops + in-place variants. In-place
|
|
96
96
|
# transforms re-store through the native fixlen path (silent truncate).
|
|
97
97
|
class CAFixlenString
|
|
98
|
+
|
|
99
|
+
# The greater String of each cell and `other`, cell by cell: `other` is a
|
|
100
|
+
# CAFixlenString of the same shape or one String. The result is as wide as
|
|
101
|
+
# the wider of the two.
|
|
102
|
+
# @return [CAFixlenString]
|
|
103
|
+
def pmax (other)
|
|
104
|
+
refuse_unordered_operands(__callee__, other)
|
|
105
|
+
pick_by_order(other) { |a, b| a.lt(b) }
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# The lesser String of each cell and `other`; see {#pmax}.
|
|
109
|
+
# @return [CAFixlenString]
|
|
110
|
+
def pmin (other)
|
|
111
|
+
refuse_unordered_operands(__callee__, other)
|
|
112
|
+
pick_by_order(other) { |a, b| a.gt(b) }
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# A String has no NaN to let win, so these are other names of pmax / pmin.
|
|
116
|
+
alias_method :maximum, :pmax
|
|
117
|
+
alias_method :minimum, :pmin
|
|
118
|
+
alias_method :fmax, :pmax
|
|
119
|
+
alias_method :fmin, :pmin
|
|
120
|
+
|
|
121
|
+
# @overload min_addr(axis: nil)
|
|
122
|
+
# @return [Integer, CArray] the flat address of the smallest String.
|
|
123
|
+
def min_addr (*args, **kw)
|
|
124
|
+
to_string.min_addr(*args, **kw)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# @overload max_addr(axis: nil)
|
|
128
|
+
# @return [Integer, CArray] the flat address of the largest String.
|
|
129
|
+
def max_addr (*args, **kw)
|
|
130
|
+
to_string.max_addr(*args, **kw)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# A value that is no String is in no cell, and the kernels below take only
|
|
134
|
+
# bytes: these ask the Strings, which answer it as any other value.
|
|
135
|
+
CArray.send(:define_named_methods, self,
|
|
136
|
+
[:eq, :ne, :count, :search, :search_addr, :is_in, :union, :difference, :intersection],
|
|
137
|
+
:through_strings_unless_text, super_block: true)
|
|
138
|
+
|
|
139
|
+
private def through_strings_unless_text (op, args, kw)
|
|
140
|
+
v = args[0]
|
|
141
|
+
return yield(*args, **kw) if args.empty? || v.is_a?(String) || v.is_a?(CArray) ||
|
|
142
|
+
v.is_a?(Array) || UNDEF.equal?(v)
|
|
143
|
+
to_string.public_send(op, *args, **kw)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# `other` where the block says self's cell loses to it, else self's cell.
|
|
147
|
+
# Another array of Strings (a CAString, a CAConstString, an object array)
|
|
148
|
+
# is taken as fixed-length strings, so the result is one.
|
|
149
|
+
private def pick_by_order (other)
|
|
150
|
+
return copy.tap { |out| out[] = UNDEF } if UNDEF.equal?(other)
|
|
151
|
+
if other.is_a?(CArray) && !other.is_a?(CAFixlenString)
|
|
152
|
+
other = CArray.string(other) if other.data_type == CA_OBJECT && !other.face?
|
|
153
|
+
other = other.to_fixlen_string if other.respond_to?(:to_fixlen_string)
|
|
154
|
+
end
|
|
155
|
+
yield(self, other).then_else(other, self)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Both branches of a selection at the width of the wider one.
|
|
159
|
+
private def branches_in_common (x, y)
|
|
160
|
+
width = [x, y].map { |v|
|
|
161
|
+
case v
|
|
162
|
+
when CArray then v.bytes
|
|
163
|
+
when String then v.bytesize
|
|
164
|
+
else 1
|
|
165
|
+
end
|
|
166
|
+
}.max
|
|
167
|
+
[x, y].map { |v|
|
|
168
|
+
v.is_a?(CAFixlenString) && !v.scalar? ? v.to_fixlen_string(bytes: width) : v
|
|
169
|
+
}
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# The encoding a column of these strings holds its text in: that of the
|
|
173
|
+
# strings beyond ASCII, UTF-8 if none is, and BINARY if they disagree,
|
|
174
|
+
# since the column cannot read them all back.
|
|
175
|
+
private_class_method def self.encoding_of_strings (values)
|
|
176
|
+
text = values.compact.map(&:to_s).reject(&:ascii_only?).map(&:encoding).uniq
|
|
177
|
+
text.size > 1 ? Encoding::BINARY : (text.first || Encoding::UTF_8)
|
|
178
|
+
end
|
|
179
|
+
|
|
98
180
|
# inspect's name for the array: the class and the cell width, which is the
|
|
99
181
|
# longest string a cell holds.
|
|
100
182
|
private def inspect_label
|
|
@@ -26,13 +26,36 @@ class CAFrame
|
|
|
26
26
|
# rows), so the answer is about the type and not the data: a refusal
|
|
27
27
|
# there means the operation does not apply, while an error in the verb
|
|
28
28
|
# itself is left to raise. An object column is the exception: its type
|
|
29
|
-
# says nothing of what it holds, so one with no present value
|
|
30
|
-
# of a verb run over every column. An N-D column (each row a profile) is
|
|
29
|
+
# says nothing of what it holds, so one with no present value, or whose
|
|
30
|
+
# cells do not answer, is left out of a verb run over every column. An N-D column (each row a profile) is
|
|
31
31
|
# asked, and computed, along the rows.
|
|
32
32
|
module ColumnVerbs
|
|
33
33
|
|
|
34
34
|
private
|
|
35
35
|
|
|
36
|
+
# The row of each cell, UNDEF where the cell is masked: a first or last
|
|
37
|
+
# taken of it picks the row, and the row picks the value of a Face that
|
|
38
|
+
# cannot hold new values, as lag picks it.
|
|
39
|
+
def present_rows (col)
|
|
40
|
+
rows = CArray.int64(col.elements).seq
|
|
41
|
+
rows[col.is_masked] = UNDEF if col.has_mask?
|
|
42
|
+
rows
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# A verb whose answers lie on the frame's rows, with no column to compute:
|
|
46
|
+
# a frame of no column would also have no row.
|
|
47
|
+
def refuse_no_column_on_rows (verb, out, nrow)
|
|
48
|
+
return unless out.is_a?(Hash) && out.empty? && nrow > 0
|
|
49
|
+
raise ArgumentError, "#{verb}: no column to compute (every column is a key or has no #{verb})"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def refuse_unknown_fill_columns (verb, fills)
|
|
53
|
+
return unless fills.is_a?(Hash)
|
|
54
|
+
unknown = fills.keys - @frame.column_names
|
|
55
|
+
return if unknown.empty?
|
|
56
|
+
raise ArgumentError, "#{verb}: fill_value names no column #{unknown.map(&:inspect).join(', ')}"
|
|
57
|
+
end
|
|
58
|
+
|
|
36
59
|
def over_named_columns (verb, names, key_names, args = [], keywords = {})
|
|
37
60
|
if names.empty?
|
|
38
61
|
cols = {}
|
|
@@ -41,7 +64,8 @@ class CAFrame
|
|
|
41
64
|
col = @frame[name]
|
|
42
65
|
next if valueless_object_column?(col, verb, args)
|
|
43
66
|
next unless column_has?(col, verb, args, keywords)
|
|
44
|
-
cols[name] =
|
|
67
|
+
cols[name] = answer_of_every_column(col, name) { |c, n| yield(c, n) }
|
|
68
|
+
cols.delete(name) if cols[name].equal?(NO_ANSWER)
|
|
45
69
|
end
|
|
46
70
|
return cols
|
|
47
71
|
end
|
|
@@ -50,7 +74,7 @@ class CAFrame
|
|
|
50
74
|
names.map { |name| @frame.__send__(:column_key, name, verb) }, verb)
|
|
51
75
|
results = keys.map do |key|
|
|
52
76
|
name = key
|
|
53
|
-
col = @frame
|
|
77
|
+
col = @frame.__send__(:named_column, key, verb)
|
|
54
78
|
unless column_has?(col, verb, args, keywords)
|
|
55
79
|
raise CArray::DataTypeError,
|
|
56
80
|
"#{verb}: column #{name.inspect} " \
|
|
@@ -61,11 +85,42 @@ class CAFrame
|
|
|
61
85
|
results.size == 1 ? results.first : keys.zip(results).to_h
|
|
62
86
|
end
|
|
63
87
|
|
|
88
|
+
NO_ANSWER = Object.new.freeze
|
|
89
|
+
private_constant :NO_ANSWER
|
|
90
|
+
|
|
91
|
+
# What an object column holds shows only in the verb itself: one whose
|
|
92
|
+
# cells do not answer is left out of a verb run over every column, as a
|
|
93
|
+
# column of a type without the operation is, wherever those cells sit.
|
|
94
|
+
# refuse_wrong_reduction_arguments has refused what no column would take.
|
|
95
|
+
def answer_of_every_column (col, name)
|
|
96
|
+
return yield(col, name) if col.face? || col.data_type != CA_OBJECT
|
|
97
|
+
begin
|
|
98
|
+
yield(col, name)
|
|
99
|
+
rescue TypeError, ArgumentError, NoMethodError, RangeError
|
|
100
|
+
NO_ANSWER
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# The arguments of a reduction, asked of the core on a number before any
|
|
105
|
+
# column, so a wrong one is not taken for a column that does not answer.
|
|
106
|
+
# fill_value: is of each column's type, and stays out of this.
|
|
107
|
+
def refuse_wrong_reduction_arguments (verb, args, keywords)
|
|
108
|
+
return if verb == :count
|
|
109
|
+
CArray.float64(1).public_send(verb, *args, **keywords.reject { |k, _| k == :fill_value })
|
|
110
|
+
rescue CArray::DataTypeError, NoMethodError
|
|
111
|
+
nil
|
|
112
|
+
end
|
|
113
|
+
|
|
64
114
|
# An object column with no present cell says nothing of what it holds,
|
|
65
115
|
# so a verb run over every column leaves it out, as select leaves it out
|
|
66
|
-
# of every kind; counting the present cells applies to any column.
|
|
116
|
+
# of every kind; counting the present cells applies to any column. A
|
|
117
|
+
# Face kept in object cells (a CAString) with rows, every one masked, is
|
|
118
|
+
# the same: the type would be asked of masked cells only, which answer
|
|
119
|
+
# with the identity whatever the type. With no rows the type is asked of
|
|
120
|
+
# blank cells of its kind.
|
|
67
121
|
def valueless_object_column? (col, verb, args)
|
|
68
|
-
return false if col.
|
|
122
|
+
return false if col.data_type != CA_OBJECT
|
|
123
|
+
return false if col.face? && col.elements == 0
|
|
69
124
|
return false if %i[count_not_masked count_masked].include?(verb)
|
|
70
125
|
return false if verb == :count && args.empty?
|
|
71
126
|
col.count_not_masked == 0
|
data/lib/carray/frame/concat.rb
CHANGED
|
@@ -52,10 +52,13 @@ class CAFrame
|
|
|
52
52
|
index_pieces(frames, verb: "meld")
|
|
53
53
|
check_trailing_shapes(frames, names, verb: "meld")
|
|
54
54
|
frames = frames_with_rows(frames)
|
|
55
|
+
if first.index
|
|
56
|
+
check_same_column_type(frames.map(&:index), first.axis_name, verb: "meld")
|
|
57
|
+
end
|
|
55
58
|
cols = {}
|
|
56
59
|
names.each do |name|
|
|
57
60
|
check_same_column_type(frames.map { |f| f[name] }, name, verb: "meld")
|
|
58
|
-
cols[name] = CArray.meld(frames.map { |f| f[name] }, axis: 0)
|
|
61
|
+
cols[name] = welding_column(name, "meld") { CArray.meld(frames.map { |f| f[name] }, axis: 0) }
|
|
59
62
|
end
|
|
60
63
|
new(cols, axis_name: first.axis_name, index: meld_index(frames))
|
|
61
64
|
end
|
|
@@ -85,7 +88,9 @@ class CAFrame
|
|
|
85
88
|
frames = frames_with_rows(frames)
|
|
86
89
|
cols = {}
|
|
87
90
|
names.each do |name|
|
|
88
|
-
|
|
91
|
+
pieces = face_pieces_in_one_state(frames.map { |f| f[name] })
|
|
92
|
+
refuse_rounded_pieces(pieces, "concatenate", "column #{name.inspect}")
|
|
93
|
+
cols[name] = welding_column(name, "concatenate") { CArray.concatenate(pieces) }
|
|
89
94
|
end
|
|
90
95
|
new(cols, axis_name: first.axis_name, index: concatenate_index(frames))
|
|
91
96
|
end
|
|
@@ -135,7 +140,8 @@ class CAFrame
|
|
|
135
140
|
"stack: axis #{axis} out of range for column #{name.inspect} " \
|
|
136
141
|
"(a #{parts.first.ndim}-D column takes 1 to #{parts.first.ndim})"
|
|
137
142
|
end
|
|
138
|
-
|
|
143
|
+
refuse_rounded_pieces(parts, "stack", "column #{name.inspect}")
|
|
144
|
+
cols[name] = welding_column(name, "stack") { CArray.stack(parts, axis: axis) }
|
|
139
145
|
end
|
|
140
146
|
new(cols, axis_name: first.axis_name, index: first.index)
|
|
141
147
|
end
|
|
@@ -194,6 +200,43 @@ class CAFrame
|
|
|
194
200
|
|
|
195
201
|
# ---- shared validators -------------------------------------------------
|
|
196
202
|
|
|
203
|
+
# Pieces of one Face in different states brought to one, as an outer join
|
|
204
|
+
# brings its two keys: times and durations to a common unit, categoricals
|
|
205
|
+
# to the labels of all, the first piece's first.
|
|
206
|
+
def self.face_pieces_in_one_state(pieces)
|
|
207
|
+
first = pieces.first
|
|
208
|
+
return pieces unless first.face? && pieces.all? { |p| p.class == first.class }
|
|
209
|
+
# names-check: units, not names
|
|
210
|
+
if first.respond_to?(:in_common_unit_with) && pieces.map(&:unit).uniq.size > 1
|
|
211
|
+
common = pieces.drop(1).reduce(first) { |acc, p| acc.in_common_unit_with(p).first }
|
|
212
|
+
return pieces.map { |p| p.unit == common.unit ? p : p.to_unit(common.unit) }
|
|
213
|
+
end
|
|
214
|
+
# names-check: label lists, not names
|
|
215
|
+
if first.is_a?(CACategorical) && pieces.map(&:labels).uniq.size > 1
|
|
216
|
+
labels = pieces.map(&:labels).reduce(:|)
|
|
217
|
+
return pieces.map { |p| CArray.object(*p.shape) { p.to_a }.categorize(labels: labels) }
|
|
218
|
+
end
|
|
219
|
+
pieces
|
|
220
|
+
end
|
|
221
|
+
private_class_method :face_pieces_in_one_state
|
|
222
|
+
|
|
223
|
+
# Pieces brought to a float: a value it rounds would be another value.
|
|
224
|
+
def self.refuse_rounded_pieces(pieces, verb, what)
|
|
225
|
+
return if pieces.size < 2 || pieces.any?(&:face?) || pieces.any? { |c| c.data_type == CA_FIXLEN }
|
|
226
|
+
refuse_rounded_integers(pieces, CArray.result_type(*pieces), verb, what)
|
|
227
|
+
end
|
|
228
|
+
private_class_method :refuse_rounded_pieces
|
|
229
|
+
|
|
230
|
+
# A refusal of the pieces reads as the verb the frames were given to, with
|
|
231
|
+
# the column it came from.
|
|
232
|
+
def self.welding_column(name, verb)
|
|
233
|
+
yield
|
|
234
|
+
rescue ArgumentError, CArray::DataTypeError => e
|
|
235
|
+
raise unless e.message.start_with?("#{verb}: ", "can't promote", "CAStack parents")
|
|
236
|
+
raise e.class, "#{verb}: column #{name.inspect}: #{e.message.delete_prefix("#{verb}: ")}"
|
|
237
|
+
end
|
|
238
|
+
private_class_method :welding_column
|
|
239
|
+
|
|
197
240
|
def self.check_concat_inputs(frames, verb:)
|
|
198
241
|
raise ArgumentError, "#{verb} requires at least one frame" if frames.empty?
|
|
199
242
|
unless frames.all? { |f| f.is_a?(CAFrame) }
|
|
@@ -267,8 +310,7 @@ class CAFrame
|
|
|
267
310
|
def self.meld_index(frames)
|
|
268
311
|
indexes, axis = index_pieces(frames, verb: "meld")
|
|
269
312
|
return nil unless indexes
|
|
270
|
-
axis
|
|
271
|
-
CArray.meld(indexes, axis: 0)
|
|
313
|
+
welding_column(axis, "meld") { CArray.meld(indexes, axis: 0) }
|
|
272
314
|
end
|
|
273
315
|
private_class_method :meld_index
|
|
274
316
|
|
|
@@ -276,7 +318,9 @@ class CAFrame
|
|
|
276
318
|
def self.concatenate_index(frames)
|
|
277
319
|
indexes, _axis = index_pieces(frames, verb: "concatenate")
|
|
278
320
|
return nil unless indexes
|
|
279
|
-
|
|
321
|
+
indexes = face_pieces_in_one_state(indexes)
|
|
322
|
+
refuse_rounded_pieces(indexes, "concatenate", "the index")
|
|
323
|
+
welding_column(frames.first.axis_name, "concatenate") { CArray.concatenate(indexes) }
|
|
280
324
|
end
|
|
281
325
|
private_class_method :concatenate_index
|
|
282
326
|
|
|
@@ -99,16 +99,24 @@ class CAFrame
|
|
|
99
99
|
class Tokenizer
|
|
100
100
|
attr_reader :sep, :quote, :strip
|
|
101
101
|
|
|
102
|
-
|
|
102
|
+
# What a separator and a quote have to be, for reading and for writing.
|
|
103
|
+
def self.check_delimiters(sep, quote)
|
|
103
104
|
unless sep.is_a?(String) && !sep.empty?
|
|
104
105
|
raise ArgumentError, "sep: must be a non-empty String (got #{sep.inspect})"
|
|
105
106
|
end
|
|
107
|
+
if sep.match?(/[\r\n]/)
|
|
108
|
+
raise ArgumentError, "sep: cannot hold a line end (got #{sep.inspect})"
|
|
109
|
+
end
|
|
106
110
|
unless quote.is_a?(String) && quote.size == 1
|
|
107
111
|
raise ArgumentError, "quote: must be one character (got #{quote.inspect})"
|
|
108
112
|
end
|
|
109
113
|
if sep.include?(quote)
|
|
110
114
|
raise ArgumentError, "sep: #{sep.inspect} cannot contain the quote character"
|
|
111
115
|
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def initialize(sep, quote, strip)
|
|
119
|
+
Tokenizer.check_delimiters(sep, quote)
|
|
112
120
|
@sep = sep
|
|
113
121
|
@quote = quote
|
|
114
122
|
@strip = strip
|
|
@@ -239,7 +247,7 @@ class CAFrame
|
|
|
239
247
|
part = part + @sep + parts[i]
|
|
240
248
|
end
|
|
241
249
|
inner = part[1...-1]
|
|
242
|
-
fields << (inner.include?(@quote) ? inner.gsub(@qq
|
|
250
|
+
fields << (inner.include?(@quote) ? inner.gsub(@qq) { @quote } : inner)
|
|
243
251
|
elsif part.include?(@quote)
|
|
244
252
|
return nil
|
|
245
253
|
else
|
|
@@ -438,7 +446,7 @@ class CAFrame
|
|
|
438
446
|
CAFrame.__send__(:column_key, c, "read_csv", "columns: a column name") if c.is_a?(Symbol)
|
|
439
447
|
next if c.is_a?(Integer) && c >= 0
|
|
440
448
|
next if c.is_a?(Range) && (c.begin.nil? || (c.begin.is_a?(Integer) && c.begin >= 0)) &&
|
|
441
|
-
(c.end.nil? || c.end.is_a?(Integer))
|
|
449
|
+
(c.end.nil? || (c.end.is_a?(Integer) && c.end >= 0))
|
|
442
450
|
raise ArgumentError, "read_csv: columns: takes column names, or indexes from 0 " \
|
|
443
451
|
"and Ranges of them (got #{c.inspect})"
|
|
444
452
|
end
|
|
@@ -485,6 +493,7 @@ class CAFrame
|
|
|
485
493
|
return nil if last.nil? && ncol.nil?
|
|
486
494
|
last = ncol - 1 if last.nil?
|
|
487
495
|
last -= 1 if c.end && c.exclude_end?
|
|
496
|
+
raise ArgumentError, "read_csv: columns: #{c.inspect} selects no column" if last < first
|
|
488
497
|
(first..last).to_a
|
|
489
498
|
end
|
|
490
499
|
end
|
|
@@ -653,7 +662,7 @@ class CAFrame
|
|
|
653
662
|
|
|
654
663
|
private def body_in_c(last = nil, chunk_bytes = CHUNK_BYTES)
|
|
655
664
|
return read_body_into_const_string_columns if last.nil? && @read_text_as_const_string
|
|
656
|
-
chunked = last.nil? && @names && @io.respond_to?(:
|
|
665
|
+
chunked = last.nil? && @names && @io.respond_to?(:internal_encoding) &&
|
|
657
666
|
@io.internal_encoding.nil?
|
|
658
667
|
# Names that have to agree with the first record leave its column
|
|
659
668
|
# count to C, to be checked against them.
|
data/lib/carray/frame/frame.rb
CHANGED
|
@@ -489,7 +489,7 @@ class CAFrame
|
|
|
489
489
|
return false if col.face? || col.data_type != :object
|
|
490
490
|
found = false
|
|
491
491
|
col.each do |v|
|
|
492
|
-
next if UNDEF.equal?(v)
|
|
492
|
+
next if UNDEF.equal?(v) || v.nil?
|
|
493
493
|
return false unless v.is_a?(String)
|
|
494
494
|
found = true
|
|
495
495
|
end
|
|
@@ -650,8 +650,8 @@ class CAFrame
|
|
|
650
650
|
verdicts[key] = verdict
|
|
651
651
|
end
|
|
652
652
|
# The verdicts are a table about the rows, not a part of this one, so the
|
|
653
|
-
# frame's attributes are not carried.
|
|
654
|
-
CAFrame.new(verdicts, axis_name: @axis_name, index: @index)
|
|
653
|
+
# frame's attributes are not carried and the index is its own.
|
|
654
|
+
CAFrame.new(verdicts, axis_name: @axis_name, index: @index&.copy)
|
|
655
655
|
end
|
|
656
656
|
|
|
657
657
|
# A new frame with a column moved to the index and the row axis named
|
|
@@ -707,6 +707,7 @@ class CAFrame
|
|
|
707
707
|
# The first +n+ rows as a positional view-frame (memo §11.3). +n+ larger
|
|
708
708
|
# than +nrow+ yields the whole frame; +n+ of 0 yields an empty frame.
|
|
709
709
|
def head(n = 5)
|
|
710
|
+
raise TypeError, "head: the count is an Integer (got #{n.inspect})" unless n.is_a?(Integer)
|
|
710
711
|
raise ArgumentError, "head count must be non-negative (got #{n})" if n < 0
|
|
711
712
|
row_span(0, [n, @nrow].min)
|
|
712
713
|
end
|
|
@@ -714,6 +715,7 @@ class CAFrame
|
|
|
714
715
|
# The last +n+ rows as a positional view-frame (memo §11.3). +n+ larger
|
|
715
716
|
# than +nrow+ yields the whole frame; +n+ of 0 yields an empty frame.
|
|
716
717
|
def tail(n = 5)
|
|
718
|
+
raise TypeError, "tail: the count is an Integer (got #{n.inspect})" unless n.is_a?(Integer)
|
|
717
719
|
raise ArgumentError, "tail count must be non-negative (got #{n})" if n < 0
|
|
718
720
|
row_span(@nrow - [n, @nrow].min, @nrow)
|
|
719
721
|
end
|
|
@@ -728,6 +730,28 @@ class CAFrame
|
|
|
728
730
|
# a row Hash and stays that way: zero matches raise KeyError, and duplicate
|
|
729
731
|
# labels raise (go through +filter { |f| f.index.eq(label) }+ for the
|
|
730
732
|
# multi-row, frame-returning path). Positional access is +df[i]+.
|
|
733
|
+
# A number as the index compares it without rounding: a whole float beside
|
|
734
|
+
# integers is an integer, and an integer beside floats that no float holds
|
|
735
|
+
# is no label there (a float32 index holds fewer integers than a float64
|
|
736
|
+
# one).
|
|
737
|
+
private def exact_label(label)
|
|
738
|
+
return label if @index.face?
|
|
739
|
+
if @index.integer? && label.is_a?(Float) && label.finite? && label == label.floor
|
|
740
|
+
label.to_i
|
|
741
|
+
elsif @index.float? && label.is_a?(Integer) && !index_float_holds?(label)
|
|
742
|
+
Float::NAN
|
|
743
|
+
else
|
|
744
|
+
label
|
|
745
|
+
end
|
|
746
|
+
end
|
|
747
|
+
|
|
748
|
+
# Whether the index's float type holds +n+ exactly.
|
|
749
|
+
private def index_float_holds?(n)
|
|
750
|
+
f = n.to_f
|
|
751
|
+
f = [f].pack("f").unpack1("f") if @index.data_type == CA_FLOAT32
|
|
752
|
+
f.finite? && f.to_i == n
|
|
753
|
+
end
|
|
754
|
+
|
|
731
755
|
def at(label)
|
|
732
756
|
raise ArgumentError, "at requires an index (set one with set_index)" unless @index
|
|
733
757
|
# An index may hold a masked cell -- an outer join and align both build one
|
|
@@ -751,8 +775,8 @@ class CAFrame
|
|
|
751
775
|
"at: the index is #{@index.data_type}, so the label must be a number " \
|
|
752
776
|
"(got #{label.inspect}); convert it first, e.g. at(Float(s)) or at(Integer(s))"
|
|
753
777
|
end
|
|
754
|
-
nan = label.is_a?(
|
|
755
|
-
hit = nan ? @index.is_in([label]) : @index.eq(label)
|
|
778
|
+
nan = label.is_a?(Numeric) && label != label
|
|
779
|
+
hit = nan ? @index.is_in([label]) : @index.eq(exact_label(label))
|
|
756
780
|
pos = hit.where
|
|
757
781
|
case pos.elements
|
|
758
782
|
when 0
|
|
@@ -1182,7 +1206,7 @@ class CAFrame
|
|
|
1182
1206
|
# an index, the spliced-in frame must have one too (its rows need labels).
|
|
1183
1207
|
private def splice_index(other, lo, hi)
|
|
1184
1208
|
return nil unless @index
|
|
1185
|
-
if other.
|
|
1209
|
+
if other.index.nil?
|
|
1186
1210
|
raise ArgumentError,
|
|
1187
1211
|
"splice frame needs an index because the target frame has one"
|
|
1188
1212
|
end
|
|
@@ -1243,6 +1267,8 @@ class CAFrame
|
|
|
1243
1267
|
end
|
|
1244
1268
|
|
|
1245
1269
|
private def select_rows(selector)
|
|
1270
|
+
# With no column and no index, nothing else bound-checks the selector.
|
|
1271
|
+
selected_row_mask(selector) if @columns.empty? && @index.nil?
|
|
1246
1272
|
cols = {}
|
|
1247
1273
|
@columns.each do |name, col|
|
|
1248
1274
|
cols[name] = col[selector, *([nil] * (col.ndim - 1))]
|
|
@@ -1264,6 +1290,12 @@ class CAFrame
|
|
|
1264
1290
|
raise KeyError, "#{lead}no column #{key.inspect}"
|
|
1265
1291
|
end
|
|
1266
1292
|
|
|
1293
|
+
# The column a verb is given by name, the verb leading the refusal of a
|
|
1294
|
+
# name that is no column.
|
|
1295
|
+
private def named_column(key, verb)
|
|
1296
|
+
@columns.fetch(key) { refuse_missing_column(key, verb) }
|
|
1297
|
+
end
|
|
1298
|
+
|
|
1267
1299
|
# filter takes no keyword. keep_masked:, which it once took, is named
|
|
1268
1300
|
# with where its work went: the selector settles an undetermined row.
|
|
1269
1301
|
def self.refuse_filter_keywords(options, verb)
|