carray 3.0.6 → 3.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +110 -0
- data/README.md +24 -3
- data/carray.gemspec +1 -1
- data/ext/ca_axis_group.c +6 -6
- data/ext/ca_binop_dispatch.c +8 -0
- data/ext/ca_binop_dispatch.h +1 -0
- data/ext/ca_categorical_iterator.c +7 -7
- data/ext/ca_kernel_iterator.h +6 -4
- data/ext/ca_obj_array.c +4 -4
- data/ext/ca_obj_bincmp.c +19 -3
- data/ext/ca_obj_binop.c +36 -11
- data/ext/ca_obj_bitarray.c +2 -4
- data/ext/ca_obj_bitfield.c +86 -4
- data/ext/ca_obj_byte_swap.c +2 -4
- data/ext/ca_obj_const_string.c +422 -257
- data/ext/ca_obj_face.c +71 -33
- data/ext/ca_obj_fake.c +112 -13
- data/ext/ca_obj_farray.c +7 -1
- data/ext/ca_obj_fixlen_string.c +51 -17
- data/ext/ca_obj_grid.c +12 -7
- data/ext/ca_obj_meld.c +5 -5
- data/ext/ca_obj_monop.c +65 -39
- data/ext/ca_obj_object.c +9 -9
- data/ext/ca_obj_reduce.c +2 -4
- data/ext/ca_obj_refer.c +104 -10
- data/ext/ca_obj_repeat.c +1 -1
- data/ext/ca_obj_roll.c +2 -4
- data/ext/ca_obj_select.c +8 -6
- data/ext/ca_obj_select_axis.c +8 -6
- data/ext/ca_obj_stack.c +5 -5
- data/ext/ca_obj_stride.c +46 -8
- data/ext/ca_obj_string.c +2 -2
- data/ext/ca_obj_tile.c +2 -4
- data/ext/ca_obj_triop.c +15 -12
- data/ext/ca_obj_window.c +22 -15
- data/ext/ca_op_ipower.c +63 -15
- data/ext/ca_op_powi.h +6 -29
- data/ext/ca_sort_kernels.h +3 -0
- data/ext/ca_sweep_engine.c +90 -23
- data/ext/ca_sweep_engine.h +5 -6
- data/ext/caframe_csv_reader.c +1 -1
- data/ext/carray.h +3 -1
- data/ext/carray_access.c +98 -39
- data/ext/carray_address_basis.c +98 -24
- data/ext/carray_broadcast.c +2 -2
- data/ext/carray_call_cfunc.c +5 -8
- data/ext/carray_cast.c +311 -37
- data/ext/carray_cast_func.rb +37 -5
- data/ext/carray_conversion.c +56 -2
- data/ext/carray_copy.c +24 -0
- data/ext/carray_core.c +69 -6
- data/ext/carray_count.c +28 -9
- data/ext/carray_element.c +9 -3
- data/ext/carray_factorize.c +11 -11
- data/ext/carray_histogram.c +2 -2
- data/ext/carray_index_classifier.c +7 -5
- data/ext/carray_internal.h +94 -0
- data/ext/carray_kernels_binop.c +55 -55
- data/ext/carray_kernels_init.c +14 -14
- data/ext/carray_kernels_monop.c +90 -140
- data/ext/carray_lazy.c +16 -1
- data/ext/carray_loop.c +9 -2
- data/ext/carray_mask.c +118 -11
- data/ext/carray_median_percentile.c +13 -8
- data/ext/carray_memory_view.c +41 -22
- data/ext/carray_operator.c +72 -11
- data/ext/carray_order.c +52 -9
- data/ext/carray_partition.c +1 -1
- data/ext/carray_random.c +1 -1
- data/ext/carray_round.c +24 -1
- data/ext/carray_scatter.c +4 -0
- data/ext/carray_segment.c +1 -1
- data/ext/carray_slab.c +13 -4
- data/ext/carray_sort.c +6 -3
- data/ext/carray_sort_kernel.c +24 -0
- data/ext/carray_struct.c +21 -6
- data/ext/carray_test.c +85 -39
- data/ext/carray_time_text.c +6 -0
- data/ext/carray_utils.c +124 -54
- data/ext/mk_call_cfunc.rb +6 -11
- data/ext/mkkernel.rb +37 -27
- data/ext/version.h +4 -4
- data/lib/carray/axis_group.rb +9 -6
- data/lib/carray/basics.rb +26 -8
- data/lib/carray/bincount_nd.rb +8 -2
- data/lib/carray/block_iterator.rb +48 -21
- data/lib/carray/categorical.rb +21 -8
- data/lib/carray/categorical_iterator.rb +2 -1
- data/lib/carray/conditional.rb +52 -4
- data/lib/carray/const_string.rb +74 -0
- data/lib/carray/construct.rb +2 -2
- data/lib/carray/core_extensions.rb +19 -68
- data/lib/carray/data_type_extension.rb +1 -0
- data/lib/carray/fixlen_string.rb +83 -1
- data/lib/carray/frame/column_verbs.rb +61 -6
- data/lib/carray/frame/concat.rb +50 -6
- data/lib/carray/frame/csv_parser.rb +13 -4
- data/lib/carray/frame/frame.rb +38 -6
- data/lib/carray/frame/group.rb +67 -40
- data/lib/carray/frame/io.rb +10 -5
- data/lib/carray/frame/join.rb +20 -6
- data/lib/carray/frame/records.rb +6 -4
- data/lib/carray/frame/reshape.rb +25 -5
- data/lib/carray/frame/verbs.rb +78 -25
- data/lib/carray/frame/windows.rb +30 -6
- data/lib/carray/fuse_source.rb +4 -2
- data/lib/carray/iterator.rb +8 -0
- data/lib/carray/key_windows.rb +10 -4
- data/lib/carray/lazy.rb +12 -17
- data/lib/carray/methods/bin.rb +3 -0
- data/lib/carray/methods/composition.rb +1 -1
- data/lib/carray/methods/factorize.rb +2 -0
- data/lib/carray/methods/gather_nd.rb +4 -1
- data/lib/carray/methods/is_in.rb +23 -3
- data/lib/carray/methods/locate_addr.rb +63 -12
- data/lib/carray/methods/repeat.rb +11 -1
- data/lib/carray/record_order.rb +35 -6
- data/lib/carray/runtime.rb +10 -0
- data/lib/carray/segment_iterator.rb +19 -16
- data/lib/carray/serialize.rb +22 -0
- data/lib/carray/slab_iterator.rb +1 -1
- data/lib/carray/stack.rb +3 -3
- data/lib/carray/string.rb +6 -6
- data/lib/carray/string_operation_extension.rb +25 -4
- data/lib/carray/struct_builder.rb +4 -1
- data/lib/carray/time.rb +190 -46
- data/lib/carray/window_iterator.rb +48 -12
- data/yard-stubs/ca_obj_bitfield.rb +4 -1
- data/yard-stubs/ca_obj_refer.rb +3 -0
- data/yard-stubs/ca_obj_window.rb +2 -1
- data/yard-stubs/carray_access.rb +4 -2
- data/yard-stubs/carray_cast.rb +6 -0
- data/yard-stubs/carray_count.rb +6 -3
- data/yard-stubs/carray_math.rb +5 -0
- data/yard-stubs/carray_order.rb +6 -0
- data/yard-stubs/carray_scatter.rb +2 -1
- data/yard-stubs/carray_test.rb +18 -5
- metadata +2 -2
data/lib/carray/frame/group.rb
CHANGED
|
@@ -148,7 +148,7 @@ class CAFrame
|
|
|
148
148
|
# as an empty reduction does: UNDEF for +mean+, 0 for +count+ and +sum+.
|
|
149
149
|
# A row whose time is masked belongs to no bin.
|
|
150
150
|
#
|
|
151
|
-
# +by:+ (a column name or an Array of them) bins each group of rows on its
|
|
151
|
+
# +by:+ (a column name, a CArray key or an Array of them) bins each group of rows on its
|
|
152
152
|
# own: a group is the rows sharing the +by:+ values and a bin. The result
|
|
153
153
|
# then has no index: the +by:+ columns and the bin times (named +name+)
|
|
154
154
|
# are its first columns, the groups in the order their +by:+ values first
|
|
@@ -163,7 +163,9 @@ class CAFrame
|
|
|
163
163
|
# @param origin [nil, Time, String, CATime::Element] where bins start.
|
|
164
164
|
# @param label [:left, :right] which end of a bin names it and is closed.
|
|
165
165
|
# @param fill [Boolean] whether bins with no rows are rows of the result.
|
|
166
|
-
# @param by [nil, String, Array<String>] columns whose groups
|
|
166
|
+
# @param by [nil, String, CArray, Array<String, CArray>] columns whose groups
|
|
167
|
+
# are binned apart; a CArray key (one value per row, as group_by takes it)
|
|
168
|
+
# is labelled "key0", "key1", ... by its place.
|
|
167
169
|
# @return [GroupedFrame]
|
|
168
170
|
# @raise [ArgumentError] when the column is not a CATime, or +label:+ is
|
|
169
171
|
# neither :left nor :right, or +by:+ names the time column.
|
|
@@ -191,13 +193,18 @@ class CAFrame
|
|
|
191
193
|
# resample with by:. A group is a (by values, bin) pair, coded as
|
|
192
194
|
# group * bins + bin, so the codes run through the groups in the order their
|
|
193
195
|
# by values first appear and through each group's bins in time order.
|
|
196
|
+
# A by: key is a column name or, as group_by takes it, a CArray of one
|
|
197
|
+
# value per row; a CArray key is labelled "key0", "key1", ... by its place.
|
|
194
198
|
private def resample_by(name, time, bins, grid, by, fill)
|
|
195
|
-
|
|
196
|
-
raise ArgumentError, "resample: by: names no column" if
|
|
197
|
-
|
|
199
|
+
keys = by.is_a?(Array) ? by : [by]
|
|
200
|
+
raise ArgumentError, "resample: by: names no column" if keys.empty?
|
|
201
|
+
keys.each { |k| column_key(k, "resample") unless k.is_a?(CArray) }
|
|
202
|
+
if keys.include?(name)
|
|
198
203
|
raise ArgumentError, "resample: by: names the time column #{name.inspect}"
|
|
199
204
|
end
|
|
200
|
-
|
|
205
|
+
by_names = keys.each_with_index.map { |k, i| k.is_a?(String) ? k : "key#{i}" }
|
|
206
|
+
CAFrame.__send__(:refuse_repeated_names, by_names + [name], "resample")
|
|
207
|
+
by_cols = keys.map { |k| key_column(k, "resample") }
|
|
201
208
|
gcat = CACategorical.send(:from_keys, by_cols)
|
|
202
209
|
ng = gcat.labels.size
|
|
203
210
|
nb = grid.elements
|
|
@@ -223,7 +230,7 @@ class CAFrame
|
|
|
223
230
|
parts + [blabels[b]]
|
|
224
231
|
end
|
|
225
232
|
cat = CACategorical.from_codes(codes, labels)
|
|
226
|
-
key_names =
|
|
233
|
+
key_names = keys.grep(String) + (@columns.key?(name) ? [name] : [])
|
|
227
234
|
GroupedFrame.new(self, cat, "group", key_names, label_columns: columns)
|
|
228
235
|
end
|
|
229
236
|
|
|
@@ -233,9 +240,19 @@ class CAFrame
|
|
|
233
240
|
# or one-year grid and then given the bins' unit, as the labels without
|
|
234
241
|
# fill: are.
|
|
235
242
|
private def resample_grid(bins, unit)
|
|
236
|
-
|
|
237
|
-
if %i[Y M].include?(base)
|
|
238
|
-
|
|
243
|
+
res = CATime::Resolution.parse(unit)
|
|
244
|
+
if %i[Y M].include?(res.base)
|
|
245
|
+
months = res.base == :Y ? res.count * 12 : res.count
|
|
246
|
+
grid = CArray.time_range(bins.min, bins.max, unit: :M, step: "#{months} months")
|
|
247
|
+
# A key coarser than the bins holds one bin for several grid points.
|
|
248
|
+
# key-labels: the bins of a grid, labelled by the caller
|
|
249
|
+
grid.to_unit(bins.unit).unique(sort: true)
|
|
250
|
+
elsif CATimeUnitAlgebra.ratio(res, bins.unit).denominator != 1
|
|
251
|
+
# A step finer than the key is stepped in its own unit; the key holds
|
|
252
|
+
# one bin for several of its points.
|
|
253
|
+
grid = CArray.time_range(bins.min, bins.max, unit: res.base, step: unit)
|
|
254
|
+
# key-labels: the bins of a grid, labelled by the caller
|
|
255
|
+
grid.to_unit(bins.unit).unique(sort: true)
|
|
239
256
|
else
|
|
240
257
|
CArray.time_range(bins.min, bins.max, unit: bins.unit, step: unit)
|
|
241
258
|
end
|
|
@@ -376,7 +393,7 @@ class CAFrame
|
|
|
376
393
|
end
|
|
377
394
|
cols = CAFrame.__send__(:quantile_columns, named)
|
|
378
395
|
return CAFrame.new(cols) unless keep_rows
|
|
379
|
-
return CAFrame.new(cols, axis_name: @axis_name, index: @index)
|
|
396
|
+
return CAFrame.new(cols, axis_name: @axis_name, index: @index&.copy)
|
|
380
397
|
end
|
|
381
398
|
|
|
382
399
|
# The five columns a quantile gives a column, by name.
|
|
@@ -391,13 +408,10 @@ class CAFrame
|
|
|
391
408
|
private_constant :QUANTILE_SUFFIXES
|
|
392
409
|
|
|
393
410
|
private def reduce_all_rows (reduction, *names, min_count: nil, fill_value: nil, keep_rows: false, args: [])
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
"#{reduction}: fill_value names no column #{unknown.map(&:inspect).join(', ')}"
|
|
399
|
-
end
|
|
400
|
-
end
|
|
411
|
+
walk = ColumnWalk.new(self)
|
|
412
|
+
walk.__send__(:refuse_unknown_fill_columns, reduction, fill_value)
|
|
413
|
+
walk.__send__(:refuse_wrong_reduction_arguments, reduction, args,
|
|
414
|
+
{ min_count: min_count }.compact)
|
|
401
415
|
return reduce_down_the_rows(reduction, names, keep_rows, args) do |name|
|
|
402
416
|
fill = fill_value.is_a?(Hash) ? fill_value[name] : fill_value
|
|
403
417
|
{ min_count: min_count, fill_value: fill }.compact
|
|
@@ -425,7 +439,7 @@ class CAFrame
|
|
|
425
439
|
end
|
|
426
440
|
return CAFrame.new(named) unless keep_rows
|
|
427
441
|
return out if out.is_a?(CArray)
|
|
428
|
-
return CAFrame.new(named, axis_name: @axis_name, index: @index)
|
|
442
|
+
return CAFrame.new(named, axis_name: @axis_name, index: @index&.copy)
|
|
429
443
|
end
|
|
430
444
|
|
|
431
445
|
# Values computed down the rows, one per row, as +GroupedFrame+ computes
|
|
@@ -459,8 +473,9 @@ class CAFrame
|
|
|
459
473
|
out = ColumnWalk.new(self).over(operation, names, [], [], keywords) do |col, _|
|
|
460
474
|
col.public_send(operation, **(col.ndim > 1 ? keywords.merge(axis: 0) : keywords))
|
|
461
475
|
end
|
|
476
|
+
ColumnWalk.new(self).__send__(:refuse_no_column_on_rows, operation, out, @nrow)
|
|
462
477
|
return out if out.is_a?(CArray)
|
|
463
|
-
return CAFrame.new(out, axis_name: @axis_name, index: @index)
|
|
478
|
+
return CAFrame.new(out, axis_name: @axis_name, index: @index&.copy)
|
|
464
479
|
end
|
|
465
480
|
|
|
466
481
|
# The frame's own walk over its columns: the one rule of a frame-wide verb
|
|
@@ -514,6 +529,8 @@ class CAFrame
|
|
|
514
529
|
walk = ColumnWalk.new(self)
|
|
515
530
|
if distance.nil?
|
|
516
531
|
step = n.nil? ? 1 : n
|
|
532
|
+
# A wrong n: would otherwise read as no column having a lag.
|
|
533
|
+
CArray.float64(1).lag(step)
|
|
517
534
|
cols = walk.over(:lag, names, [], [step]) do |col, _|
|
|
518
535
|
(col.ndim > 1 ? col.lag(step, axis: 0) : col.lag(step)).copy
|
|
519
536
|
end
|
|
@@ -524,7 +541,7 @@ class CAFrame
|
|
|
524
541
|
end
|
|
525
542
|
end
|
|
526
543
|
return cols if cols.is_a?(CArray)
|
|
527
|
-
CAFrame.new(cols, axis_name: @axis_name, index: @index)
|
|
544
|
+
CAFrame.new(cols, axis_name: @axis_name, index: @index&.copy)
|
|
528
545
|
end
|
|
529
546
|
|
|
530
547
|
# [key name, key, sign, width] for a lag by distance; nil for one by rows.
|
|
@@ -548,7 +565,7 @@ class CAFrame
|
|
|
548
565
|
on_name = column_key(on, verb)
|
|
549
566
|
end
|
|
550
567
|
key = key_column(on_name, verb)
|
|
551
|
-
unless key.is_a?(CATime) || key.is_a?(CATimedelta) || (!key.face? && key.numeric?)
|
|
568
|
+
unless key.is_a?(CATime) || key.is_a?(CATimedelta) || (!key.face? && key.numeric? && !key.complex?)
|
|
552
569
|
raise ArgumentError,
|
|
553
570
|
"#{verb}: the key #{on_name.inspect} is #{CArray::Inspector.type_label(key)}; " \
|
|
554
571
|
"a distance is taken along a time, a duration or a number"
|
|
@@ -804,7 +821,7 @@ class GroupedFrame
|
|
|
804
821
|
cols[out] =
|
|
805
822
|
case reduction
|
|
806
823
|
when Symbol
|
|
807
|
-
group_reduce(@frame
|
|
824
|
+
group_reduce(@frame.__send__(:named_column, in_name, "aggregate"), reduction, *args, **options)
|
|
808
825
|
when Proc
|
|
809
826
|
unless args.empty? && options.empty?
|
|
810
827
|
raise ArgumentError, "aggregate: a Proc reduction takes no arguments (#{out.inspect})"
|
|
@@ -827,6 +844,7 @@ class GroupedFrame
|
|
|
827
844
|
# first appear; a group whose Hash lacks a name is UNDEF there, as
|
|
828
845
|
# +from_records+ reads a record that lacks a key.
|
|
829
846
|
def table(keep_rows: false)
|
|
847
|
+
raise ArgumentError, "table: give a block" unless block_given?
|
|
830
848
|
refuse_changed_rows("table")
|
|
831
849
|
outs = []
|
|
832
850
|
each_group_frame do |g|
|
|
@@ -953,7 +971,7 @@ class GroupedFrame
|
|
|
953
971
|
end
|
|
954
972
|
cols = CAFrame.__send__(:quantile_columns, named)
|
|
955
973
|
return result_frame(cols, false) unless keep_rows
|
|
956
|
-
return CAFrame.new(cols, axis_name: @frame.axis_name, index: @frame.index)
|
|
974
|
+
return CAFrame.new(cols, axis_name: @frame.axis_name, index: @frame.index&.copy)
|
|
957
975
|
end
|
|
958
976
|
|
|
959
977
|
# The same convenience for the reductions that take no min_count: or
|
|
@@ -1041,7 +1059,7 @@ class GroupedFrame
|
|
|
1041
1059
|
col[addr, *([nil] * (col.ndim - 1))].copy
|
|
1042
1060
|
end
|
|
1043
1061
|
return out if out.is_a?(CArray)
|
|
1044
|
-
CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index)
|
|
1062
|
+
CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index&.copy)
|
|
1045
1063
|
end
|
|
1046
1064
|
|
|
1047
1065
|
def order (*names, descending: false, method: :ordinal, kind: :quick)
|
|
@@ -1104,11 +1122,10 @@ class GroupedFrame
|
|
|
1104
1122
|
when :ffill, :bfill
|
|
1105
1123
|
@frame.__send__(:refuse_if_frozen)
|
|
1106
1124
|
key = @frame.__send__(:column_key, name, "fill")
|
|
1107
|
-
col = @frame
|
|
1125
|
+
col = @frame.__send__(:named_column, key, "fill")
|
|
1108
1126
|
# A row in no group is UNDEF in the fill, so writing it changes nothing.
|
|
1109
1127
|
held = group_transform(col, :strip_mask, [], { method: method_or_value })
|
|
1110
|
-
|
|
1111
|
-
col[target] = held[target] if target.any
|
|
1128
|
+
@frame.__send__(:write_filled_cells, col, held)
|
|
1112
1129
|
when :linear
|
|
1113
1130
|
@frame.__send__(:refuse_if_frozen)
|
|
1114
1131
|
key = @frame.__send__(:column_key, name, "fill")
|
|
@@ -1122,7 +1139,7 @@ class GroupedFrame
|
|
|
1122
1139
|
when Symbol
|
|
1123
1140
|
raise ArgumentError,
|
|
1124
1141
|
"fill: unknown method #{method_or_value.inspect} " \
|
|
1125
|
-
"(:ffill | :bfill, or a constant fill value)"
|
|
1142
|
+
"(:ffill | :bfill | :linear, or a constant fill value)"
|
|
1126
1143
|
else
|
|
1127
1144
|
@frame.fill(name, method_or_value)
|
|
1128
1145
|
end
|
|
@@ -1230,6 +1247,7 @@ class GroupedFrame
|
|
|
1230
1247
|
# shape [ngroup].
|
|
1231
1248
|
def filter (**options)
|
|
1232
1249
|
CAFrame.__send__(:refuse_filter_keywords, options, "filter")
|
|
1250
|
+
raise ArgumentError, "filter: give a block" unless block_given?
|
|
1233
1251
|
refuse_changed_rows("filter")
|
|
1234
1252
|
chosen = yield(self)
|
|
1235
1253
|
unless chosen.is_a?(CArray) && chosen.data_type == :boolean
|
|
@@ -1278,17 +1296,27 @@ class GroupedFrame
|
|
|
1278
1296
|
out = over_named_columns(op, names, @key_names, args, keywords) do |col|
|
|
1279
1297
|
group_transform(col, op, args, keywords)
|
|
1280
1298
|
end
|
|
1299
|
+
refuse_no_column_on_rows(op, out, @frame.nrow)
|
|
1281
1300
|
return out if out.is_a?(CArray)
|
|
1282
|
-
CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index)
|
|
1301
|
+
CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index&.copy)
|
|
1283
1302
|
end
|
|
1284
1303
|
|
|
1285
1304
|
# A column's within-group transform: an N-D column runs along the rows.
|
|
1286
1305
|
private def group_transform(col, op, args, keywords)
|
|
1306
|
+
return group_lag_by_rows(col, *args) if op == :lag && col.face? && !col.__send__(:holds_new_values?)
|
|
1287
1307
|
it = col.group_by_category(@cat)
|
|
1288
1308
|
keywords = keywords.merge(axis: 0) if col.ndim > 1
|
|
1289
1309
|
it.public_send(op, *args, **keywords)
|
|
1290
1310
|
end
|
|
1291
1311
|
|
|
1312
|
+
# A Face that cannot hold new values answers an iterator's lag in object
|
|
1313
|
+
# cells. Reading each cell from the row it lags keeps the Face, as
|
|
1314
|
+
# lag(before:) does.
|
|
1315
|
+
private def group_lag_by_rows(col, step)
|
|
1316
|
+
rows = CArray.int64(@frame.nrow).seq.group_by_category(@cat).lag(step)
|
|
1317
|
+
col[rows, *([nil] * (col.ndim - 1))].copy
|
|
1318
|
+
end
|
|
1319
|
+
|
|
1292
1320
|
# Each result gets its own index.
|
|
1293
1321
|
private def label_index
|
|
1294
1322
|
return @index.copy if @index
|
|
@@ -1302,13 +1330,8 @@ class GroupedFrame
|
|
|
1302
1330
|
refuse_key_as_result(reduction, names) unless keep_rows
|
|
1303
1331
|
keywords = keywords.compact
|
|
1304
1332
|
fills = keywords.delete(:fill_value)
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
unless unknown.empty?
|
|
1308
|
-
raise ArgumentError,
|
|
1309
|
-
"#{reduction}: fill_value names no column #{unknown.map(&:inspect).join(', ')}"
|
|
1310
|
-
end
|
|
1311
|
-
end
|
|
1333
|
+
refuse_unknown_fill_columns(reduction, fills)
|
|
1334
|
+
refuse_wrong_reduction_arguments(reduction, args, keywords)
|
|
1312
1335
|
named = {}
|
|
1313
1336
|
out = over_named_columns(reduction, names, @key_names, args) do |col, name|
|
|
1314
1337
|
fill = fills.is_a?(Hash) ? fills[name] : fills
|
|
@@ -1322,7 +1345,7 @@ class GroupedFrame
|
|
|
1322
1345
|
# labels beside it.
|
|
1323
1346
|
if keep_rows
|
|
1324
1347
|
return out if out.is_a?(CArray)
|
|
1325
|
-
return CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index)
|
|
1348
|
+
return CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index&.copy)
|
|
1326
1349
|
end
|
|
1327
1350
|
result_frame(named, false)
|
|
1328
1351
|
end
|
|
@@ -1360,7 +1383,7 @@ class GroupedFrame
|
|
|
1360
1383
|
return CAFrame.new(@label_columns.transform_values(&:copy).merge(cols))
|
|
1361
1384
|
end
|
|
1362
1385
|
CAFrame.new(cols.transform_values { |col| spread_to_rows(col) },
|
|
1363
|
-
axis_name: @frame.axis_name, index: @frame.index)
|
|
1386
|
+
axis_name: @frame.axis_name, index: @frame.index&.copy)
|
|
1364
1387
|
end
|
|
1365
1388
|
|
|
1366
1389
|
# A per-group result (one entry per group along axis 0) laid back over the
|
|
@@ -1377,6 +1400,10 @@ class GroupedFrame
|
|
|
1377
1400
|
# A column's group reduction. An N-D column is reduced along the rows, so
|
|
1378
1401
|
# each group keeps the trailing shape: a profile column (N, L) gives (G, L).
|
|
1379
1402
|
private def group_reduce(col, reduction, *args, **keywords)
|
|
1403
|
+
if %i[first last].include?(reduction) && col.ndim == 1 && args.empty? && keywords.empty? &&
|
|
1404
|
+
col.face? && !col.__send__(:holds_new_values?)
|
|
1405
|
+
return col[present_rows(col).group_by_category(@cat).public_send(reduction)].copy
|
|
1406
|
+
end
|
|
1380
1407
|
it = col.group_by_category(@cat)
|
|
1381
1408
|
if col.ndim > 1 && !keywords.key?(:axis)
|
|
1382
1409
|
it.public_send(reduction, *args, axis: 0, **keywords)
|
|
@@ -1428,7 +1455,7 @@ class GroupedFrame
|
|
|
1428
1455
|
end
|
|
1429
1456
|
|
|
1430
1457
|
private def each_group_slice(in_name)
|
|
1431
|
-
col = @frame
|
|
1458
|
+
col = @frame.__send__(:named_column, in_name, "aggregate")
|
|
1432
1459
|
ngroup.times do |k|
|
|
1433
1460
|
yield col[group_address(k), *([nil] * (col.ndim - 1))]
|
|
1434
1461
|
end
|
data/lib/carray/frame/io.rb
CHANGED
|
@@ -231,7 +231,7 @@ class CAFrame
|
|
|
231
231
|
map = types == :infer ? { default: :infer } : types
|
|
232
232
|
return types unless map.is_a?(Hash)
|
|
233
233
|
frame.column_names.each do |name|
|
|
234
|
-
given =
|
|
234
|
+
given = map.keys.find { |k| k == name || k == name.to_sym || (k.is_a?(Array) && k.include?(name)) }
|
|
235
235
|
next unless (given ? map[given] : map[:default]) == :infer
|
|
236
236
|
map = map.merge(name => :string) if blank_value_in?(frame[name])
|
|
237
237
|
end
|
|
@@ -454,6 +454,7 @@ class CAFrame
|
|
|
454
454
|
unless precision.nil? || (precision.is_a?(Integer) && precision >= 0)
|
|
455
455
|
raise ArgumentError, "to_csv: precision: is a number of decimal places, 0 or more (got #{precision.inspect})"
|
|
456
456
|
end
|
|
457
|
+
CSVParser::Tokenizer.check_delimiters(sep, quote)
|
|
457
458
|
default, per_column = CAFrame.send(:split_missing, missing, "to_csv")
|
|
458
459
|
[default, *per_column.values].each do |text|
|
|
459
460
|
next if text.nil? || text.is_a?(String)
|
|
@@ -510,7 +511,11 @@ class CAFrame
|
|
|
510
511
|
formatted.each_with_index do |fcol, j|
|
|
511
512
|
token = per_column.fetch(names[j], default)
|
|
512
513
|
next if token.nil? || token.empty? # the empty field, as without missing:
|
|
513
|
-
|
|
514
|
+
# A cell in another encoding is the token too once transcoded.
|
|
515
|
+
i = fcol.index(token) || fcol.index do |text|
|
|
516
|
+
text && text.encoding != token.encoding &&
|
|
517
|
+
text.encode(token.encoding, invalid: :replace, undef: :replace) == token
|
|
518
|
+
end
|
|
514
519
|
if i
|
|
515
520
|
raise ArgumentError,
|
|
516
521
|
"to_csv: row #{i} of #{names[j].inspect} is written as " \
|
|
@@ -757,8 +762,8 @@ class CAFrame
|
|
|
757
762
|
# Still a float, so precision: 0 writes 3.0 and the column reads
|
|
758
763
|
# back as floats.
|
|
759
764
|
csv_rounded(e, precision).to_s
|
|
760
|
-
elsif precision && e.is_a?(Complex) && e.real.is_a?(Float)
|
|
761
|
-
Complex(*[e.real, e.imaginary].map { |x| csv_rounded(x, precision) }).to_s
|
|
765
|
+
elsif precision && e.is_a?(Complex) && (e.real.is_a?(Float) || e.imaginary.is_a?(Float))
|
|
766
|
+
Complex(*[e.real, e.imaginary].map { |x| x.is_a?(Float) ? csv_rounded(x, precision) : x }).to_s
|
|
762
767
|
else
|
|
763
768
|
e.to_s
|
|
764
769
|
end
|
|
@@ -866,7 +871,7 @@ class CAFrame
|
|
|
866
871
|
if text.empty? || text.include?(sep) || text.include?(quote) ||
|
|
867
872
|
text.include?("\n") || text.include?("\r") || text.each_byte.all? { |b| b == 0x20 || b == 0x09 } ||
|
|
868
873
|
(sep.size > 1 && (text + sep).index(sep) != text.size)
|
|
869
|
-
quote + text.gsub(quote
|
|
874
|
+
quote + text.gsub(quote) { quote * 2 } + quote
|
|
870
875
|
else
|
|
871
876
|
text
|
|
872
877
|
end
|
data/lib/carray/frame/join.rb
CHANGED
|
@@ -135,13 +135,17 @@ class CAFrame
|
|
|
135
135
|
key.__send__(:nearest_refuse_bytes, key, "join_asof: the on: key #{on.inspect}",
|
|
136
136
|
"join on it exactly with join")
|
|
137
137
|
end
|
|
138
|
+
refuse_rounded_keys(lkey, rkey, "join_asof")
|
|
139
|
+
if [:round, :floor, :ceil].include?(direction)
|
|
140
|
+
rkey.__send__(:nearest_refuse_distanceless, rkey, direction, tolerance, "join_asof")
|
|
141
|
+
end
|
|
138
142
|
addr = if exact.empty?
|
|
139
143
|
lkey.locate_nearest_addr(rkey, direction: direction, tolerance: tolerance)
|
|
140
144
|
else
|
|
141
145
|
lexact = exact.map { |name| key_column(name, "join_asof") }
|
|
142
146
|
rexact = rexact_names.map { |name| other.__send__(:key_column, name, "join_asof") }
|
|
143
147
|
join_check_key_widths(exact, lexact, rexact, "join_asof")
|
|
144
|
-
_, lcode, rcode, k = join_codes(lexact, rexact)
|
|
148
|
+
_, lcode, rcode, k = join_codes(lexact, rexact, "join_asof")
|
|
145
149
|
join_asof_by(lkey, rkey, lcode, rcode, k, direction, tolerance)
|
|
146
150
|
end
|
|
147
151
|
join_by_addr(other, [on, *exact], [ron, *rexact_names], addr, plan, view)
|
|
@@ -177,6 +181,7 @@ class CAFrame
|
|
|
177
181
|
raise ArgumentError,
|
|
178
182
|
"align: the reference has shape #{ref.shape.inspect}; give one key value per row of the result"
|
|
179
183
|
end
|
|
184
|
+
refuse_rounded_keys(keys, ref, "align")
|
|
180
185
|
addr = ref.locate_addr(keys)
|
|
181
186
|
# The result's keys are the reference's values, taken as they are now
|
|
182
187
|
# and shown with the key's attributes; the caller's array is left as
|
|
@@ -320,8 +325,9 @@ class CAFrame
|
|
|
320
325
|
# so the number stays below the row count. Returns [parts, lcode, rcode,
|
|
321
326
|
# k], parts holding each column's [distinct values, left codes, right
|
|
322
327
|
# codes].
|
|
323
|
-
private def join_codes(lkeys, rkeys)
|
|
328
|
+
private def join_codes(lkeys, rkeys, verb = "join")
|
|
324
329
|
parts = lkeys.zip(rkeys).map do |lkey, rkey|
|
|
330
|
+
refuse_rounded_keys(lkey, rkey, verb)
|
|
325
331
|
common = CArray.align_addr(lkey, rkey, join: :outer)[0]
|
|
326
332
|
[common, lkey.locate_addr(common), rkey.locate_addr(common)]
|
|
327
333
|
end
|
|
@@ -338,6 +344,14 @@ class CAFrame
|
|
|
338
344
|
[parts, lcode, rcode, k]
|
|
339
345
|
end
|
|
340
346
|
|
|
347
|
+
# The two sides of a key meet at one type; a value that type rounds would
|
|
348
|
+
# match another value.
|
|
349
|
+
private def refuse_rounded_keys(lkey, rkey, verb)
|
|
350
|
+
return if lkey.face? || rkey.face?
|
|
351
|
+
CAFrame.__send__(:refuse_rounded_integers, [lkey, rkey],
|
|
352
|
+
lkey.send(:query_match_type, rkey), verb, "a key")
|
|
353
|
+
end
|
|
354
|
+
|
|
341
355
|
# The rows of a :semi or :anti join: whether each row's key code occurs
|
|
342
356
|
# among the other side's codes. A row without a code (a masked key cell)
|
|
343
357
|
# has no match.
|
|
@@ -412,7 +426,7 @@ class CAFrame
|
|
|
412
426
|
hi = rk.project((start + lt).tap { |a| a[has_hi.not] = UNDEF })
|
|
413
427
|
# an exact match takes its first reference; otherwise the nearer
|
|
414
428
|
# neighbour, the upper one on a tie
|
|
415
|
-
upper = (hi
|
|
429
|
+
upper = lkey.__send__(:nearest_distance, hi, qk).le(lkey.__send__(:nearest_distance, qk, lo)).strip_mask(false)
|
|
416
430
|
upper[has_lo.not] = true
|
|
417
431
|
upper[has_hi.not] = false
|
|
418
432
|
upper[hi.eq(qk).strip_mask(false)] = true
|
|
@@ -421,8 +435,7 @@ class CAFrame
|
|
|
421
435
|
local[local.lt(0) | local.ge(size)] = UNDEF
|
|
422
436
|
found = rrows.project(gstart.project(qc) + local)
|
|
423
437
|
if limit
|
|
424
|
-
|
|
425
|
-
found[dist > limit] = UNDEF
|
|
438
|
+
found[lkey.__send__(:nearest_beyond, rkey.project(found), lkey.project(lrows), limit)] = UNDEF
|
|
426
439
|
end
|
|
427
440
|
addr[lrows] = found
|
|
428
441
|
addr
|
|
@@ -433,7 +446,8 @@ class CAFrame
|
|
|
433
446
|
private def asof_searchable(key, code)
|
|
434
447
|
ok = code.is_not_masked.strip_mask(false)
|
|
435
448
|
ok &= key.is_not_masked.strip_mask(false) if key.has_mask?
|
|
436
|
-
|
|
449
|
+
nan = key.__send__(:nearest_nan_cells, key)
|
|
450
|
+
ok &= nan.strip_mask(false).not if nan
|
|
437
451
|
ok
|
|
438
452
|
end
|
|
439
453
|
|
data/lib/carray/frame/records.rb
CHANGED
|
@@ -181,7 +181,8 @@ class CAFrame
|
|
|
181
181
|
# :cmplx128 when Complex values join them, otherwise nil (keep object).
|
|
182
182
|
# Other Numerics (Rational, BigDecimal) stay object: a float column would
|
|
183
183
|
# round them, and a Complex one has room for them only as floats too. So
|
|
184
|
-
# does an Integer an int64 cannot hold
|
|
184
|
+
# does an Integer an int64 cannot hold, and one past 2**53 beside a Float:
|
|
185
|
+
# it is kept as it is, not refused.
|
|
185
186
|
INT64_RANGE = (-2**63..2**63 - 1)
|
|
186
187
|
private_constant :INT64_RANGE
|
|
187
188
|
|
|
@@ -198,9 +199,10 @@ class CAFrame
|
|
|
198
199
|
|
|
199
200
|
def self.numeric_leaf_type(values)
|
|
200
201
|
return nil if values.any? { |v| v.is_a?(Integer) && !INT64_RANGE.cover?(v) }
|
|
201
|
-
if values.all? { |v| v.is_a?(Integer) }
|
|
202
|
-
|
|
203
|
-
|
|
202
|
+
return :int64 if values.all? { |v| v.is_a?(Integer) }
|
|
203
|
+
# Beside a float an integer is read as one, which rounds past 2**53.
|
|
204
|
+
return nil if values.any? { |v| v.is_a?(Integer) && v.abs > 2**53 }
|
|
205
|
+
if values.all? { |v| v.is_a?(Integer) || v.is_a?(Float) }
|
|
204
206
|
:float64
|
|
205
207
|
elsif values.all? { |v| v.is_a?(Integer) || v.is_a?(Float) || v.is_a?(Complex) }
|
|
206
208
|
:cmplx128
|
data/lib/carray/frame/reshape.rb
CHANGED
|
@@ -210,6 +210,24 @@ class CAFrame
|
|
|
210
210
|
Ractor.make_shareable(SIGNIFICAND_BITS)
|
|
211
211
|
private_constant :INTEGER_BITS, :SIGNIFICAND_BITS
|
|
212
212
|
|
|
213
|
+
# Integers past what +type+'s significand holds would meet as one value
|
|
214
|
+
# once read in it, so they are refused; the verb reads every other integer
|
|
215
|
+
# exactly. +type+ is the float the columns meet at, or anything else, which
|
|
216
|
+
# rounds nothing.
|
|
217
|
+
def self.refuse_rounded_integers (cols, type, verb, what)
|
|
218
|
+
digits = SIGNIFICAND_BITS[CArray.data_type_name(type)]
|
|
219
|
+
return if digits.nil?
|
|
220
|
+
cols.each do |col|
|
|
221
|
+
next if col.face? || !col.integer? || col.count_not_masked == 0
|
|
222
|
+
big = [col.min.abs, col.max.abs].max
|
|
223
|
+
next if big <= 2**digits
|
|
224
|
+
raise ArgumentError,
|
|
225
|
+
"#{verb}: #{what} holds #{big}, past 2**#{digits}, beyond which " \
|
|
226
|
+
"#{CArray.data_type_name(type)} does not hold every integer; cast them to one type first"
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
private_class_method :refuse_rounded_integers
|
|
230
|
+
|
|
213
231
|
# Whether every value of data type +from+ is exactly a value of +to+, for
|
|
214
232
|
# numbers and booleans: a boolean in any number, an integer in a wide
|
|
215
233
|
# enough integer (a signed one never in an unsigned) or in a float whose
|
|
@@ -269,9 +287,11 @@ class CAFrame
|
|
|
269
287
|
# melted columns and each id column is +CArray.meld+ of itself, so a write
|
|
270
288
|
# to the result lands in +self+. Number columns of different types meet
|
|
271
289
|
# at a type that holds every value of each column's type (int8 and uint8
|
|
272
|
-
# at int16, int32 and float64 at float64), so a value written
|
|
273
|
-
#
|
|
274
|
-
#
|
|
290
|
+
# at int16, int32 and float64 at float64), so a value written to one of
|
|
291
|
+
# the columns later reads back unchanged. A value written to the result
|
|
292
|
+
# lands in the column's own type, as an array stored into it does: outside
|
|
293
|
+
# that type it wraps. A mixture no type holds (int64 and float64) raises,
|
|
294
|
+
# and so do other mixtures of types. The value columns share one trailing
|
|
275
295
|
# shape; cast first when they do not.
|
|
276
296
|
#
|
|
277
297
|
# @param id [String, Array<String>] columns repeated alongside each value.
|
|
@@ -332,11 +352,11 @@ class CAFrame
|
|
|
332
352
|
begin
|
|
333
353
|
out[value_name] = CArray.meld(pieces)
|
|
334
354
|
rescue ArgumentError => e
|
|
335
|
-
raise unless e.message.start_with?("
|
|
355
|
+
raise unless e.message.start_with?("meld:", "CAMeld parents")
|
|
336
356
|
labels = vars.zip(pieces.map { |c| CArray::Inspector.type_label(c) }).to_h
|
|
337
357
|
raise ArgumentError,
|
|
338
358
|
"melt: value columns #{labels} do not meet as one column " \
|
|
339
|
-
"(#{e.message.delete_prefix("
|
|
359
|
+
"(#{e.message.delete_prefix("meld: ")}); cast them to one type first"
|
|
340
360
|
end
|
|
341
361
|
CAFrame.new(out)
|
|
342
362
|
end
|