carray 3.0.6 → 3.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +110 -0
  3. data/README.md +24 -3
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_group.c +6 -6
  6. data/ext/ca_binop_dispatch.c +8 -0
  7. data/ext/ca_binop_dispatch.h +1 -0
  8. data/ext/ca_categorical_iterator.c +7 -7
  9. data/ext/ca_kernel_iterator.h +6 -4
  10. data/ext/ca_obj_array.c +4 -4
  11. data/ext/ca_obj_bincmp.c +19 -3
  12. data/ext/ca_obj_binop.c +36 -11
  13. data/ext/ca_obj_bitarray.c +2 -4
  14. data/ext/ca_obj_bitfield.c +86 -4
  15. data/ext/ca_obj_byte_swap.c +2 -4
  16. data/ext/ca_obj_const_string.c +422 -257
  17. data/ext/ca_obj_face.c +71 -33
  18. data/ext/ca_obj_fake.c +112 -13
  19. data/ext/ca_obj_farray.c +7 -1
  20. data/ext/ca_obj_fixlen_string.c +51 -17
  21. data/ext/ca_obj_grid.c +12 -7
  22. data/ext/ca_obj_meld.c +5 -5
  23. data/ext/ca_obj_monop.c +65 -39
  24. data/ext/ca_obj_object.c +9 -9
  25. data/ext/ca_obj_reduce.c +2 -4
  26. data/ext/ca_obj_refer.c +104 -10
  27. data/ext/ca_obj_repeat.c +1 -1
  28. data/ext/ca_obj_roll.c +2 -4
  29. data/ext/ca_obj_select.c +8 -6
  30. data/ext/ca_obj_select_axis.c +8 -6
  31. data/ext/ca_obj_stack.c +5 -5
  32. data/ext/ca_obj_stride.c +46 -8
  33. data/ext/ca_obj_string.c +2 -2
  34. data/ext/ca_obj_tile.c +2 -4
  35. data/ext/ca_obj_triop.c +15 -12
  36. data/ext/ca_obj_window.c +22 -15
  37. data/ext/ca_op_ipower.c +63 -15
  38. data/ext/ca_op_powi.h +6 -29
  39. data/ext/ca_sort_kernels.h +3 -0
  40. data/ext/ca_sweep_engine.c +90 -23
  41. data/ext/ca_sweep_engine.h +5 -6
  42. data/ext/caframe_csv_reader.c +1 -1
  43. data/ext/carray.h +3 -1
  44. data/ext/carray_access.c +98 -39
  45. data/ext/carray_address_basis.c +98 -24
  46. data/ext/carray_broadcast.c +2 -2
  47. data/ext/carray_call_cfunc.c +5 -8
  48. data/ext/carray_cast.c +311 -37
  49. data/ext/carray_cast_func.rb +37 -5
  50. data/ext/carray_conversion.c +56 -2
  51. data/ext/carray_copy.c +24 -0
  52. data/ext/carray_core.c +69 -6
  53. data/ext/carray_count.c +28 -9
  54. data/ext/carray_element.c +9 -3
  55. data/ext/carray_factorize.c +11 -11
  56. data/ext/carray_histogram.c +2 -2
  57. data/ext/carray_index_classifier.c +7 -5
  58. data/ext/carray_internal.h +94 -0
  59. data/ext/carray_kernels_binop.c +55 -55
  60. data/ext/carray_kernels_init.c +14 -14
  61. data/ext/carray_kernels_monop.c +90 -140
  62. data/ext/carray_lazy.c +16 -1
  63. data/ext/carray_loop.c +9 -2
  64. data/ext/carray_mask.c +118 -11
  65. data/ext/carray_median_percentile.c +13 -8
  66. data/ext/carray_memory_view.c +41 -22
  67. data/ext/carray_operator.c +72 -11
  68. data/ext/carray_order.c +52 -9
  69. data/ext/carray_partition.c +1 -1
  70. data/ext/carray_random.c +1 -1
  71. data/ext/carray_round.c +24 -1
  72. data/ext/carray_scatter.c +4 -0
  73. data/ext/carray_segment.c +1 -1
  74. data/ext/carray_slab.c +13 -4
  75. data/ext/carray_sort.c +6 -3
  76. data/ext/carray_sort_kernel.c +24 -0
  77. data/ext/carray_struct.c +21 -6
  78. data/ext/carray_test.c +85 -39
  79. data/ext/carray_time_text.c +6 -0
  80. data/ext/carray_utils.c +124 -54
  81. data/ext/mk_call_cfunc.rb +6 -11
  82. data/ext/mkkernel.rb +37 -27
  83. data/ext/version.h +4 -4
  84. data/lib/carray/axis_group.rb +9 -6
  85. data/lib/carray/basics.rb +26 -8
  86. data/lib/carray/bincount_nd.rb +8 -2
  87. data/lib/carray/block_iterator.rb +48 -21
  88. data/lib/carray/categorical.rb +21 -8
  89. data/lib/carray/categorical_iterator.rb +2 -1
  90. data/lib/carray/conditional.rb +52 -4
  91. data/lib/carray/const_string.rb +74 -0
  92. data/lib/carray/construct.rb +2 -2
  93. data/lib/carray/core_extensions.rb +19 -68
  94. data/lib/carray/data_type_extension.rb +1 -0
  95. data/lib/carray/fixlen_string.rb +83 -1
  96. data/lib/carray/frame/column_verbs.rb +61 -6
  97. data/lib/carray/frame/concat.rb +50 -6
  98. data/lib/carray/frame/csv_parser.rb +13 -4
  99. data/lib/carray/frame/frame.rb +38 -6
  100. data/lib/carray/frame/group.rb +67 -40
  101. data/lib/carray/frame/io.rb +10 -5
  102. data/lib/carray/frame/join.rb +20 -6
  103. data/lib/carray/frame/records.rb +6 -4
  104. data/lib/carray/frame/reshape.rb +25 -5
  105. data/lib/carray/frame/verbs.rb +78 -25
  106. data/lib/carray/frame/windows.rb +30 -6
  107. data/lib/carray/fuse_source.rb +4 -2
  108. data/lib/carray/iterator.rb +8 -0
  109. data/lib/carray/key_windows.rb +10 -4
  110. data/lib/carray/lazy.rb +12 -17
  111. data/lib/carray/methods/bin.rb +3 -0
  112. data/lib/carray/methods/composition.rb +1 -1
  113. data/lib/carray/methods/factorize.rb +2 -0
  114. data/lib/carray/methods/gather_nd.rb +4 -1
  115. data/lib/carray/methods/is_in.rb +23 -3
  116. data/lib/carray/methods/locate_addr.rb +63 -12
  117. data/lib/carray/methods/repeat.rb +11 -1
  118. data/lib/carray/record_order.rb +35 -6
  119. data/lib/carray/runtime.rb +10 -0
  120. data/lib/carray/segment_iterator.rb +19 -16
  121. data/lib/carray/serialize.rb +22 -0
  122. data/lib/carray/slab_iterator.rb +1 -1
  123. data/lib/carray/stack.rb +3 -3
  124. data/lib/carray/string.rb +6 -6
  125. data/lib/carray/string_operation_extension.rb +25 -4
  126. data/lib/carray/struct_builder.rb +4 -1
  127. data/lib/carray/time.rb +190 -46
  128. data/lib/carray/window_iterator.rb +48 -12
  129. data/yard-stubs/ca_obj_bitfield.rb +4 -1
  130. data/yard-stubs/ca_obj_refer.rb +3 -0
  131. data/yard-stubs/ca_obj_window.rb +2 -1
  132. data/yard-stubs/carray_access.rb +4 -2
  133. data/yard-stubs/carray_cast.rb +6 -0
  134. data/yard-stubs/carray_count.rb +6 -3
  135. data/yard-stubs/carray_math.rb +5 -0
  136. data/yard-stubs/carray_order.rb +6 -0
  137. data/yard-stubs/carray_scatter.rb +2 -1
  138. data/yard-stubs/carray_test.rb +18 -5
  139. metadata +2 -2
@@ -148,7 +148,7 @@ class CAFrame
148
148
  # as an empty reduction does: UNDEF for +mean+, 0 for +count+ and +sum+.
149
149
  # A row whose time is masked belongs to no bin.
150
150
  #
151
- # +by:+ (a column name or an Array of them) bins each group of rows on its
151
+ # +by:+ (a column name, a CArray key or an Array of them) bins each group of rows on its
152
152
  # own: a group is the rows sharing the +by:+ values and a bin. The result
153
153
  # then has no index: the +by:+ columns and the bin times (named +name+)
154
154
  # are its first columns, the groups in the order their +by:+ values first
@@ -163,7 +163,9 @@ class CAFrame
163
163
  # @param origin [nil, Time, String, CATime::Element] where bins start.
164
164
  # @param label [:left, :right] which end of a bin names it and is closed.
165
165
  # @param fill [Boolean] whether bins with no rows are rows of the result.
166
- # @param by [nil, String, Array<String>] columns whose groups are binned apart.
166
+ # @param by [nil, String, CArray, Array<String, CArray>] columns whose groups
167
+ # are binned apart; a CArray key (one value per row, as group_by takes it)
168
+ # is labelled "key0", "key1", ... by its place.
167
169
  # @return [GroupedFrame]
168
170
  # @raise [ArgumentError] when the column is not a CATime, or +label:+ is
169
171
  # neither :left nor :right, or +by:+ names the time column.
@@ -191,13 +193,18 @@ class CAFrame
191
193
  # resample with by:. A group is a (by values, bin) pair, coded as
192
194
  # group * bins + bin, so the codes run through the groups in the order their
193
195
  # by values first appear and through each group's bins in time order.
196
+ # A by: key is a column name or, as group_by takes it, a CArray of one
197
+ # value per row; a CArray key is labelled "key0", "key1", ... by its place.
194
198
  private def resample_by(name, time, bins, grid, by, fill)
195
- by_names = column_keys(Array(by), "resample")
196
- raise ArgumentError, "resample: by: names no column" if by_names.empty?
197
- if by_names.include?(name)
199
+ keys = by.is_a?(Array) ? by : [by]
200
+ raise ArgumentError, "resample: by: names no column" if keys.empty?
201
+ keys.each { |k| column_key(k, "resample") unless k.is_a?(CArray) }
202
+ if keys.include?(name)
198
203
  raise ArgumentError, "resample: by: names the time column #{name.inspect}"
199
204
  end
200
- by_cols = by_names.map { |n| key_column(n, "resample") }
205
+ by_names = keys.each_with_index.map { |k, i| k.is_a?(String) ? k : "key#{i}" }
206
+ CAFrame.__send__(:refuse_repeated_names, by_names + [name], "resample")
207
+ by_cols = keys.map { |k| key_column(k, "resample") }
201
208
  gcat = CACategorical.send(:from_keys, by_cols)
202
209
  ng = gcat.labels.size
203
210
  nb = grid.elements
@@ -223,7 +230,7 @@ class CAFrame
223
230
  parts + [blabels[b]]
224
231
  end
225
232
  cat = CACategorical.from_codes(codes, labels)
226
- key_names = by_names + (@columns.key?(name) ? [name] : [])
233
+ key_names = keys.grep(String) + (@columns.key?(name) ? [name] : [])
227
234
  GroupedFrame.new(self, cat, "group", key_names, label_columns: columns)
228
235
  end
229
236
 
@@ -233,9 +240,19 @@ class CAFrame
233
240
  # or one-year grid and then given the bins' unit, as the labels without
234
241
  # fill: are.
235
242
  private def resample_grid(bins, unit)
236
- base = CATime::Resolution.parse(unit).base
237
- if %i[Y M].include?(base)
238
- CArray.time_range(bins.min, bins.max, unit: base, step: unit).to_unit(bins.unit)
243
+ res = CATime::Resolution.parse(unit)
244
+ if %i[Y M].include?(res.base)
245
+ months = res.base == :Y ? res.count * 12 : res.count
246
+ grid = CArray.time_range(bins.min, bins.max, unit: :M, step: "#{months} months")
247
+ # A key coarser than the bins holds one bin for several grid points.
248
+ # key-labels: the bins of a grid, labelled by the caller
249
+ grid.to_unit(bins.unit).unique(sort: true)
250
+ elsif CATimeUnitAlgebra.ratio(res, bins.unit).denominator != 1
251
+ # A step finer than the key is stepped in its own unit; the key holds
252
+ # one bin for several of its points.
253
+ grid = CArray.time_range(bins.min, bins.max, unit: res.base, step: unit)
254
+ # key-labels: the bins of a grid, labelled by the caller
255
+ grid.to_unit(bins.unit).unique(sort: true)
239
256
  else
240
257
  CArray.time_range(bins.min, bins.max, unit: bins.unit, step: unit)
241
258
  end
@@ -376,7 +393,7 @@ class CAFrame
376
393
  end
377
394
  cols = CAFrame.__send__(:quantile_columns, named)
378
395
  return CAFrame.new(cols) unless keep_rows
379
- return CAFrame.new(cols, axis_name: @axis_name, index: @index)
396
+ return CAFrame.new(cols, axis_name: @axis_name, index: @index&.copy)
380
397
  end
381
398
 
382
399
  # The five columns a quantile gives a column, by name.
@@ -391,13 +408,10 @@ class CAFrame
391
408
  private_constant :QUANTILE_SUFFIXES
392
409
 
393
410
  private def reduce_all_rows (reduction, *names, min_count: nil, fill_value: nil, keep_rows: false, args: [])
394
- if fill_value.is_a?(Hash)
395
- unknown = fill_value.keys - column_names
396
- unless unknown.empty?
397
- raise ArgumentError,
398
- "#{reduction}: fill_value names no column #{unknown.map(&:inspect).join(', ')}"
399
- end
400
- end
411
+ walk = ColumnWalk.new(self)
412
+ walk.__send__(:refuse_unknown_fill_columns, reduction, fill_value)
413
+ walk.__send__(:refuse_wrong_reduction_arguments, reduction, args,
414
+ { min_count: min_count }.compact)
401
415
  return reduce_down_the_rows(reduction, names, keep_rows, args) do |name|
402
416
  fill = fill_value.is_a?(Hash) ? fill_value[name] : fill_value
403
417
  { min_count: min_count, fill_value: fill }.compact
@@ -425,7 +439,7 @@ class CAFrame
425
439
  end
426
440
  return CAFrame.new(named) unless keep_rows
427
441
  return out if out.is_a?(CArray)
428
- return CAFrame.new(named, axis_name: @axis_name, index: @index)
442
+ return CAFrame.new(named, axis_name: @axis_name, index: @index&.copy)
429
443
  end
430
444
 
431
445
  # Values computed down the rows, one per row, as +GroupedFrame+ computes
@@ -459,8 +473,9 @@ class CAFrame
459
473
  out = ColumnWalk.new(self).over(operation, names, [], [], keywords) do |col, _|
460
474
  col.public_send(operation, **(col.ndim > 1 ? keywords.merge(axis: 0) : keywords))
461
475
  end
476
+ ColumnWalk.new(self).__send__(:refuse_no_column_on_rows, operation, out, @nrow)
462
477
  return out if out.is_a?(CArray)
463
- return CAFrame.new(out, axis_name: @axis_name, index: @index)
478
+ return CAFrame.new(out, axis_name: @axis_name, index: @index&.copy)
464
479
  end
465
480
 
466
481
  # The frame's own walk over its columns: the one rule of a frame-wide verb
@@ -514,6 +529,8 @@ class CAFrame
514
529
  walk = ColumnWalk.new(self)
515
530
  if distance.nil?
516
531
  step = n.nil? ? 1 : n
532
+ # A wrong n: would otherwise read as no column having a lag.
533
+ CArray.float64(1).lag(step)
517
534
  cols = walk.over(:lag, names, [], [step]) do |col, _|
518
535
  (col.ndim > 1 ? col.lag(step, axis: 0) : col.lag(step)).copy
519
536
  end
@@ -524,7 +541,7 @@ class CAFrame
524
541
  end
525
542
  end
526
543
  return cols if cols.is_a?(CArray)
527
- CAFrame.new(cols, axis_name: @axis_name, index: @index)
544
+ CAFrame.new(cols, axis_name: @axis_name, index: @index&.copy)
528
545
  end
529
546
 
530
547
  # [key name, key, sign, width] for a lag by distance; nil for one by rows.
@@ -548,7 +565,7 @@ class CAFrame
548
565
  on_name = column_key(on, verb)
549
566
  end
550
567
  key = key_column(on_name, verb)
551
- unless key.is_a?(CATime) || key.is_a?(CATimedelta) || (!key.face? && key.numeric?)
568
+ unless key.is_a?(CATime) || key.is_a?(CATimedelta) || (!key.face? && key.numeric? && !key.complex?)
552
569
  raise ArgumentError,
553
570
  "#{verb}: the key #{on_name.inspect} is #{CArray::Inspector.type_label(key)}; " \
554
571
  "a distance is taken along a time, a duration or a number"
@@ -804,7 +821,7 @@ class GroupedFrame
804
821
  cols[out] =
805
822
  case reduction
806
823
  when Symbol
807
- group_reduce(@frame[in_name], reduction, *args, **options)
824
+ group_reduce(@frame.__send__(:named_column, in_name, "aggregate"), reduction, *args, **options)
808
825
  when Proc
809
826
  unless args.empty? && options.empty?
810
827
  raise ArgumentError, "aggregate: a Proc reduction takes no arguments (#{out.inspect})"
@@ -827,6 +844,7 @@ class GroupedFrame
827
844
  # first appear; a group whose Hash lacks a name is UNDEF there, as
828
845
  # +from_records+ reads a record that lacks a key.
829
846
  def table(keep_rows: false)
847
+ raise ArgumentError, "table: give a block" unless block_given?
830
848
  refuse_changed_rows("table")
831
849
  outs = []
832
850
  each_group_frame do |g|
@@ -953,7 +971,7 @@ class GroupedFrame
953
971
  end
954
972
  cols = CAFrame.__send__(:quantile_columns, named)
955
973
  return result_frame(cols, false) unless keep_rows
956
- return CAFrame.new(cols, axis_name: @frame.axis_name, index: @frame.index)
974
+ return CAFrame.new(cols, axis_name: @frame.axis_name, index: @frame.index&.copy)
957
975
  end
958
976
 
959
977
  # The same convenience for the reductions that take no min_count: or
@@ -1041,7 +1059,7 @@ class GroupedFrame
1041
1059
  col[addr, *([nil] * (col.ndim - 1))].copy
1042
1060
  end
1043
1061
  return out if out.is_a?(CArray)
1044
- CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index)
1062
+ CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index&.copy)
1045
1063
  end
1046
1064
 
1047
1065
  def order (*names, descending: false, method: :ordinal, kind: :quick)
@@ -1104,11 +1122,10 @@ class GroupedFrame
1104
1122
  when :ffill, :bfill
1105
1123
  @frame.__send__(:refuse_if_frozen)
1106
1124
  key = @frame.__send__(:column_key, name, "fill")
1107
- col = @frame[key]
1125
+ col = @frame.__send__(:named_column, key, "fill")
1108
1126
  # A row in no group is UNDEF in the fill, so writing it changes nothing.
1109
1127
  held = group_transform(col, :strip_mask, [], { method: method_or_value })
1110
- target = col.is_masked
1111
- col[target] = held[target] if target.any
1128
+ @frame.__send__(:write_filled_cells, col, held)
1112
1129
  when :linear
1113
1130
  @frame.__send__(:refuse_if_frozen)
1114
1131
  key = @frame.__send__(:column_key, name, "fill")
@@ -1122,7 +1139,7 @@ class GroupedFrame
1122
1139
  when Symbol
1123
1140
  raise ArgumentError,
1124
1141
  "fill: unknown method #{method_or_value.inspect} " \
1125
- "(:ffill | :bfill, or a constant fill value)"
1142
+ "(:ffill | :bfill | :linear, or a constant fill value)"
1126
1143
  else
1127
1144
  @frame.fill(name, method_or_value)
1128
1145
  end
@@ -1230,6 +1247,7 @@ class GroupedFrame
1230
1247
  # shape [ngroup].
1231
1248
  def filter (**options)
1232
1249
  CAFrame.__send__(:refuse_filter_keywords, options, "filter")
1250
+ raise ArgumentError, "filter: give a block" unless block_given?
1233
1251
  refuse_changed_rows("filter")
1234
1252
  chosen = yield(self)
1235
1253
  unless chosen.is_a?(CArray) && chosen.data_type == :boolean
@@ -1278,17 +1296,27 @@ class GroupedFrame
1278
1296
  out = over_named_columns(op, names, @key_names, args, keywords) do |col|
1279
1297
  group_transform(col, op, args, keywords)
1280
1298
  end
1299
+ refuse_no_column_on_rows(op, out, @frame.nrow)
1281
1300
  return out if out.is_a?(CArray)
1282
- CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index)
1301
+ CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index&.copy)
1283
1302
  end
1284
1303
 
1285
1304
  # A column's within-group transform: an N-D column runs along the rows.
1286
1305
  private def group_transform(col, op, args, keywords)
1306
+ return group_lag_by_rows(col, *args) if op == :lag && col.face? && !col.__send__(:holds_new_values?)
1287
1307
  it = col.group_by_category(@cat)
1288
1308
  keywords = keywords.merge(axis: 0) if col.ndim > 1
1289
1309
  it.public_send(op, *args, **keywords)
1290
1310
  end
1291
1311
 
1312
+ # A Face that cannot hold new values answers an iterator's lag in object
1313
+ # cells. Reading each cell from the row it lags keeps the Face, as
1314
+ # lag(before:) does.
1315
+ private def group_lag_by_rows(col, step)
1316
+ rows = CArray.int64(@frame.nrow).seq.group_by_category(@cat).lag(step)
1317
+ col[rows, *([nil] * (col.ndim - 1))].copy
1318
+ end
1319
+
1292
1320
  # Each result gets its own index.
1293
1321
  private def label_index
1294
1322
  return @index.copy if @index
@@ -1302,13 +1330,8 @@ class GroupedFrame
1302
1330
  refuse_key_as_result(reduction, names) unless keep_rows
1303
1331
  keywords = keywords.compact
1304
1332
  fills = keywords.delete(:fill_value)
1305
- if fills.is_a?(Hash)
1306
- unknown = fills.keys - @frame.column_names
1307
- unless unknown.empty?
1308
- raise ArgumentError,
1309
- "#{reduction}: fill_value names no column #{unknown.map(&:inspect).join(', ')}"
1310
- end
1311
- end
1333
+ refuse_unknown_fill_columns(reduction, fills)
1334
+ refuse_wrong_reduction_arguments(reduction, args, keywords)
1312
1335
  named = {}
1313
1336
  out = over_named_columns(reduction, names, @key_names, args) do |col, name|
1314
1337
  fill = fills.is_a?(Hash) ? fills[name] : fills
@@ -1322,7 +1345,7 @@ class GroupedFrame
1322
1345
  # labels beside it.
1323
1346
  if keep_rows
1324
1347
  return out if out.is_a?(CArray)
1325
- return CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index)
1348
+ return CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index&.copy)
1326
1349
  end
1327
1350
  result_frame(named, false)
1328
1351
  end
@@ -1360,7 +1383,7 @@ class GroupedFrame
1360
1383
  return CAFrame.new(@label_columns.transform_values(&:copy).merge(cols))
1361
1384
  end
1362
1385
  CAFrame.new(cols.transform_values { |col| spread_to_rows(col) },
1363
- axis_name: @frame.axis_name, index: @frame.index)
1386
+ axis_name: @frame.axis_name, index: @frame.index&.copy)
1364
1387
  end
1365
1388
 
1366
1389
  # A per-group result (one entry per group along axis 0) laid back over the
@@ -1377,6 +1400,10 @@ class GroupedFrame
1377
1400
  # A column's group reduction. An N-D column is reduced along the rows, so
1378
1401
  # each group keeps the trailing shape: a profile column (N, L) gives (G, L).
1379
1402
  private def group_reduce(col, reduction, *args, **keywords)
1403
+ if %i[first last].include?(reduction) && col.ndim == 1 && args.empty? && keywords.empty? &&
1404
+ col.face? && !col.__send__(:holds_new_values?)
1405
+ return col[present_rows(col).group_by_category(@cat).public_send(reduction)].copy
1406
+ end
1380
1407
  it = col.group_by_category(@cat)
1381
1408
  if col.ndim > 1 && !keywords.key?(:axis)
1382
1409
  it.public_send(reduction, *args, axis: 0, **keywords)
@@ -1428,7 +1455,7 @@ class GroupedFrame
1428
1455
  end
1429
1456
 
1430
1457
  private def each_group_slice(in_name)
1431
- col = @frame[in_name]
1458
+ col = @frame.__send__(:named_column, in_name, "aggregate")
1432
1459
  ngroup.times do |k|
1433
1460
  yield col[group_address(k), *([nil] * (col.ndim - 1))]
1434
1461
  end
@@ -231,7 +231,7 @@ class CAFrame
231
231
  map = types == :infer ? { default: :infer } : types
232
232
  return types unless map.is_a?(Hash)
233
233
  frame.column_names.each do |name|
234
- given = [name, name.to_sym].find { |k| map.key?(k) }
234
+ given = map.keys.find { |k| k == name || k == name.to_sym || (k.is_a?(Array) && k.include?(name)) }
235
235
  next unless (given ? map[given] : map[:default]) == :infer
236
236
  map = map.merge(name => :string) if blank_value_in?(frame[name])
237
237
  end
@@ -454,6 +454,7 @@ class CAFrame
454
454
  unless precision.nil? || (precision.is_a?(Integer) && precision >= 0)
455
455
  raise ArgumentError, "to_csv: precision: is a number of decimal places, 0 or more (got #{precision.inspect})"
456
456
  end
457
+ CSVParser::Tokenizer.check_delimiters(sep, quote)
457
458
  default, per_column = CAFrame.send(:split_missing, missing, "to_csv")
458
459
  [default, *per_column.values].each do |text|
459
460
  next if text.nil? || text.is_a?(String)
@@ -510,7 +511,11 @@ class CAFrame
510
511
  formatted.each_with_index do |fcol, j|
511
512
  token = per_column.fetch(names[j], default)
512
513
  next if token.nil? || token.empty? # the empty field, as without missing:
513
- i = fcol.index(token)
514
+ # A cell in another encoding is the token too once transcoded.
515
+ i = fcol.index(token) || fcol.index do |text|
516
+ text && text.encoding != token.encoding &&
517
+ text.encode(token.encoding, invalid: :replace, undef: :replace) == token
518
+ end
514
519
  if i
515
520
  raise ArgumentError,
516
521
  "to_csv: row #{i} of #{names[j].inspect} is written as " \
@@ -757,8 +762,8 @@ class CAFrame
757
762
  # Still a float, so precision: 0 writes 3.0 and the column reads
758
763
  # back as floats.
759
764
  csv_rounded(e, precision).to_s
760
- elsif precision && e.is_a?(Complex) && e.real.is_a?(Float)
761
- Complex(*[e.real, e.imaginary].map { |x| csv_rounded(x, precision) }).to_s
765
+ elsif precision && e.is_a?(Complex) && (e.real.is_a?(Float) || e.imaginary.is_a?(Float))
766
+ Complex(*[e.real, e.imaginary].map { |x| x.is_a?(Float) ? csv_rounded(x, precision) : x }).to_s
762
767
  else
763
768
  e.to_s
764
769
  end
@@ -866,7 +871,7 @@ class CAFrame
866
871
  if text.empty? || text.include?(sep) || text.include?(quote) ||
867
872
  text.include?("\n") || text.include?("\r") || text.each_byte.all? { |b| b == 0x20 || b == 0x09 } ||
868
873
  (sep.size > 1 && (text + sep).index(sep) != text.size)
869
- quote + text.gsub(quote, quote * 2) + quote
874
+ quote + text.gsub(quote) { quote * 2 } + quote
870
875
  else
871
876
  text
872
877
  end
@@ -135,13 +135,17 @@ class CAFrame
135
135
  key.__send__(:nearest_refuse_bytes, key, "join_asof: the on: key #{on.inspect}",
136
136
  "join on it exactly with join")
137
137
  end
138
+ refuse_rounded_keys(lkey, rkey, "join_asof")
139
+ if [:round, :floor, :ceil].include?(direction)
140
+ rkey.__send__(:nearest_refuse_distanceless, rkey, direction, tolerance, "join_asof")
141
+ end
138
142
  addr = if exact.empty?
139
143
  lkey.locate_nearest_addr(rkey, direction: direction, tolerance: tolerance)
140
144
  else
141
145
  lexact = exact.map { |name| key_column(name, "join_asof") }
142
146
  rexact = rexact_names.map { |name| other.__send__(:key_column, name, "join_asof") }
143
147
  join_check_key_widths(exact, lexact, rexact, "join_asof")
144
- _, lcode, rcode, k = join_codes(lexact, rexact)
148
+ _, lcode, rcode, k = join_codes(lexact, rexact, "join_asof")
145
149
  join_asof_by(lkey, rkey, lcode, rcode, k, direction, tolerance)
146
150
  end
147
151
  join_by_addr(other, [on, *exact], [ron, *rexact_names], addr, plan, view)
@@ -177,6 +181,7 @@ class CAFrame
177
181
  raise ArgumentError,
178
182
  "align: the reference has shape #{ref.shape.inspect}; give one key value per row of the result"
179
183
  end
184
+ refuse_rounded_keys(keys, ref, "align")
180
185
  addr = ref.locate_addr(keys)
181
186
  # The result's keys are the reference's values, taken as they are now
182
187
  # and shown with the key's attributes; the caller's array is left as
@@ -320,8 +325,9 @@ class CAFrame
320
325
  # so the number stays below the row count. Returns [parts, lcode, rcode,
321
326
  # k], parts holding each column's [distinct values, left codes, right
322
327
  # codes].
323
- private def join_codes(lkeys, rkeys)
328
+ private def join_codes(lkeys, rkeys, verb = "join")
324
329
  parts = lkeys.zip(rkeys).map do |lkey, rkey|
330
+ refuse_rounded_keys(lkey, rkey, verb)
325
331
  common = CArray.align_addr(lkey, rkey, join: :outer)[0]
326
332
  [common, lkey.locate_addr(common), rkey.locate_addr(common)]
327
333
  end
@@ -338,6 +344,14 @@ class CAFrame
338
344
  [parts, lcode, rcode, k]
339
345
  end
340
346
 
347
+ # The two sides of a key meet at one type; a value that type rounds would
348
+ # match another value.
349
+ private def refuse_rounded_keys(lkey, rkey, verb)
350
+ return if lkey.face? || rkey.face?
351
+ CAFrame.__send__(:refuse_rounded_integers, [lkey, rkey],
352
+ lkey.send(:query_match_type, rkey), verb, "a key")
353
+ end
354
+
341
355
  # The rows of a :semi or :anti join: whether each row's key code occurs
342
356
  # among the other side's codes. A row without a code (a masked key cell)
343
357
  # has no match.
@@ -412,7 +426,7 @@ class CAFrame
412
426
  hi = rk.project((start + lt).tap { |a| a[has_hi.not] = UNDEF })
413
427
  # an exact match takes its first reference; otherwise the nearer
414
428
  # neighbour, the upper one on a tie
415
- upper = (hi - qk).le(qk - lo).strip_mask(false)
429
+ upper = lkey.__send__(:nearest_distance, hi, qk).le(lkey.__send__(:nearest_distance, qk, lo)).strip_mask(false)
416
430
  upper[has_lo.not] = true
417
431
  upper[has_hi.not] = false
418
432
  upper[hi.eq(qk).strip_mask(false)] = true
@@ -421,8 +435,7 @@ class CAFrame
421
435
  local[local.lt(0) | local.ge(size)] = UNDEF
422
436
  found = rrows.project(gstart.project(qc) + local)
423
437
  if limit
424
- dist = (rkey.project(found) - lkey.project(lrows)).abs
425
- found[dist > limit] = UNDEF
438
+ found[lkey.__send__(:nearest_beyond, rkey.project(found), lkey.project(lrows), limit)] = UNDEF
426
439
  end
427
440
  addr[lrows] = found
428
441
  addr
@@ -433,7 +446,8 @@ class CAFrame
433
446
  private def asof_searchable(key, code)
434
447
  ok = code.is_not_masked.strip_mask(false)
435
448
  ok &= key.is_not_masked.strip_mask(false) if key.has_mask?
436
- ok &= key.is_nan.strip_mask(false).not if key.float?
449
+ nan = key.__send__(:nearest_nan_cells, key)
450
+ ok &= nan.strip_mask(false).not if nan
437
451
  ok
438
452
  end
439
453
 
@@ -181,7 +181,8 @@ class CAFrame
181
181
  # :cmplx128 when Complex values join them, otherwise nil (keep object).
182
182
  # Other Numerics (Rational, BigDecimal) stay object: a float column would
183
183
  # round them, and a Complex one has room for them only as floats too. So
184
- # does an Integer an int64 cannot hold: it is kept as it is, not refused.
184
+ # does an Integer an int64 cannot hold, and one past 2**53 beside a Float:
185
+ # it is kept as it is, not refused.
185
186
  INT64_RANGE = (-2**63..2**63 - 1)
186
187
  private_constant :INT64_RANGE
187
188
 
@@ -198,9 +199,10 @@ class CAFrame
198
199
 
199
200
  def self.numeric_leaf_type(values)
200
201
  return nil if values.any? { |v| v.is_a?(Integer) && !INT64_RANGE.cover?(v) }
201
- if values.all? { |v| v.is_a?(Integer) }
202
- :int64
203
- elsif values.all? { |v| v.is_a?(Integer) || v.is_a?(Float) }
202
+ return :int64 if values.all? { |v| v.is_a?(Integer) }
203
+ # Beside a float an integer is read as one, which rounds past 2**53.
204
+ return nil if values.any? { |v| v.is_a?(Integer) && v.abs > 2**53 }
205
+ if values.all? { |v| v.is_a?(Integer) || v.is_a?(Float) }
204
206
  :float64
205
207
  elsif values.all? { |v| v.is_a?(Integer) || v.is_a?(Float) || v.is_a?(Complex) }
206
208
  :cmplx128
@@ -210,6 +210,24 @@ class CAFrame
210
210
  Ractor.make_shareable(SIGNIFICAND_BITS)
211
211
  private_constant :INTEGER_BITS, :SIGNIFICAND_BITS
212
212
 
213
+ # Integers past what +type+'s significand holds would meet as one value
214
+ # once read in it, so they are refused; the verb reads every other integer
215
+ # exactly. +type+ is the float the columns meet at, or anything else, which
216
+ # rounds nothing.
217
+ def self.refuse_rounded_integers (cols, type, verb, what)
218
+ digits = SIGNIFICAND_BITS[CArray.data_type_name(type)]
219
+ return if digits.nil?
220
+ cols.each do |col|
221
+ next if col.face? || !col.integer? || col.count_not_masked == 0
222
+ big = [col.min.abs, col.max.abs].max
223
+ next if big <= 2**digits
224
+ raise ArgumentError,
225
+ "#{verb}: #{what} holds #{big}, past 2**#{digits}, beyond which " \
226
+ "#{CArray.data_type_name(type)} does not hold every integer; cast them to one type first"
227
+ end
228
+ end
229
+ private_class_method :refuse_rounded_integers
230
+
213
231
  # Whether every value of data type +from+ is exactly a value of +to+, for
214
232
  # numbers and booleans: a boolean in any number, an integer in a wide
215
233
  # enough integer (a signed one never in an unsigned) or in a float whose
@@ -269,9 +287,11 @@ class CAFrame
269
287
  # melted columns and each id column is +CArray.meld+ of itself, so a write
270
288
  # to the result lands in +self+. Number columns of different types meet
271
289
  # at a type that holds every value of each column's type (int8 and uint8
272
- # at int16, int32 and float64 at float64), so a value written later reads
273
- # back unchanged; a mixture no type holds (int64 and float64) raises, and
274
- # so do other mixtures of types. The value columns share one trailing
290
+ # at int16, int32 and float64 at float64), so a value written to one of
291
+ # the columns later reads back unchanged. A value written to the result
292
+ # lands in the column's own type, as an array stored into it does: outside
293
+ # that type it wraps. A mixture no type holds (int64 and float64) raises,
294
+ # and so do other mixtures of types. The value columns share one trailing
275
295
  # shape; cast first when they do not.
276
296
  #
277
297
  # @param id [String, Array<String>] columns repeated alongside each value.
@@ -332,11 +352,11 @@ class CAFrame
332
352
  begin
333
353
  out[value_name] = CArray.meld(pieces)
334
354
  rescue ArgumentError => e
335
- raise unless e.message.start_with?("promote_list:")
355
+ raise unless e.message.start_with?("meld:", "CAMeld parents")
336
356
  labels = vars.zip(pieces.map { |c| CArray::Inspector.type_label(c) }).to_h
337
357
  raise ArgumentError,
338
358
  "melt: value columns #{labels} do not meet as one column " \
339
- "(#{e.message.delete_prefix("promote_list: ")}); cast them to one type first"
359
+ "(#{e.message.delete_prefix("meld: ")}); cast them to one type first"
340
360
  end
341
361
  CAFrame.new(out)
342
362
  end