carray 3.0.6 → 3.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +110 -0
  3. data/README.md +24 -3
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_group.c +6 -6
  6. data/ext/ca_binop_dispatch.c +8 -0
  7. data/ext/ca_binop_dispatch.h +1 -0
  8. data/ext/ca_categorical_iterator.c +7 -7
  9. data/ext/ca_kernel_iterator.h +6 -4
  10. data/ext/ca_obj_array.c +4 -4
  11. data/ext/ca_obj_bincmp.c +19 -3
  12. data/ext/ca_obj_binop.c +36 -11
  13. data/ext/ca_obj_bitarray.c +2 -4
  14. data/ext/ca_obj_bitfield.c +86 -4
  15. data/ext/ca_obj_byte_swap.c +2 -4
  16. data/ext/ca_obj_const_string.c +422 -257
  17. data/ext/ca_obj_face.c +71 -33
  18. data/ext/ca_obj_fake.c +112 -13
  19. data/ext/ca_obj_farray.c +7 -1
  20. data/ext/ca_obj_fixlen_string.c +51 -17
  21. data/ext/ca_obj_grid.c +12 -7
  22. data/ext/ca_obj_meld.c +5 -5
  23. data/ext/ca_obj_monop.c +65 -39
  24. data/ext/ca_obj_object.c +9 -9
  25. data/ext/ca_obj_reduce.c +2 -4
  26. data/ext/ca_obj_refer.c +104 -10
  27. data/ext/ca_obj_repeat.c +1 -1
  28. data/ext/ca_obj_roll.c +2 -4
  29. data/ext/ca_obj_select.c +8 -6
  30. data/ext/ca_obj_select_axis.c +8 -6
  31. data/ext/ca_obj_stack.c +5 -5
  32. data/ext/ca_obj_stride.c +46 -8
  33. data/ext/ca_obj_string.c +2 -2
  34. data/ext/ca_obj_tile.c +2 -4
  35. data/ext/ca_obj_triop.c +15 -12
  36. data/ext/ca_obj_window.c +22 -15
  37. data/ext/ca_op_ipower.c +63 -15
  38. data/ext/ca_op_powi.h +6 -29
  39. data/ext/ca_sort_kernels.h +3 -0
  40. data/ext/ca_sweep_engine.c +90 -23
  41. data/ext/ca_sweep_engine.h +5 -6
  42. data/ext/caframe_csv_reader.c +1 -1
  43. data/ext/carray.h +3 -1
  44. data/ext/carray_access.c +98 -39
  45. data/ext/carray_address_basis.c +98 -24
  46. data/ext/carray_broadcast.c +2 -2
  47. data/ext/carray_call_cfunc.c +5 -8
  48. data/ext/carray_cast.c +311 -37
  49. data/ext/carray_cast_func.rb +37 -5
  50. data/ext/carray_conversion.c +56 -2
  51. data/ext/carray_copy.c +24 -0
  52. data/ext/carray_core.c +69 -6
  53. data/ext/carray_count.c +28 -9
  54. data/ext/carray_element.c +9 -3
  55. data/ext/carray_factorize.c +11 -11
  56. data/ext/carray_histogram.c +2 -2
  57. data/ext/carray_index_classifier.c +7 -5
  58. data/ext/carray_internal.h +94 -0
  59. data/ext/carray_kernels_binop.c +55 -55
  60. data/ext/carray_kernels_init.c +14 -14
  61. data/ext/carray_kernels_monop.c +90 -140
  62. data/ext/carray_lazy.c +16 -1
  63. data/ext/carray_loop.c +9 -2
  64. data/ext/carray_mask.c +118 -11
  65. data/ext/carray_median_percentile.c +13 -8
  66. data/ext/carray_memory_view.c +41 -22
  67. data/ext/carray_operator.c +72 -11
  68. data/ext/carray_order.c +52 -9
  69. data/ext/carray_partition.c +1 -1
  70. data/ext/carray_random.c +1 -1
  71. data/ext/carray_round.c +24 -1
  72. data/ext/carray_scatter.c +4 -0
  73. data/ext/carray_segment.c +1 -1
  74. data/ext/carray_slab.c +13 -4
  75. data/ext/carray_sort.c +6 -3
  76. data/ext/carray_sort_kernel.c +24 -0
  77. data/ext/carray_struct.c +21 -6
  78. data/ext/carray_test.c +85 -39
  79. data/ext/carray_time_text.c +6 -0
  80. data/ext/carray_utils.c +124 -54
  81. data/ext/mk_call_cfunc.rb +6 -11
  82. data/ext/mkkernel.rb +37 -27
  83. data/ext/version.h +4 -4
  84. data/lib/carray/axis_group.rb +9 -6
  85. data/lib/carray/basics.rb +26 -8
  86. data/lib/carray/bincount_nd.rb +8 -2
  87. data/lib/carray/block_iterator.rb +48 -21
  88. data/lib/carray/categorical.rb +21 -8
  89. data/lib/carray/categorical_iterator.rb +2 -1
  90. data/lib/carray/conditional.rb +52 -4
  91. data/lib/carray/const_string.rb +74 -0
  92. data/lib/carray/construct.rb +2 -2
  93. data/lib/carray/core_extensions.rb +19 -68
  94. data/lib/carray/data_type_extension.rb +1 -0
  95. data/lib/carray/fixlen_string.rb +83 -1
  96. data/lib/carray/frame/column_verbs.rb +61 -6
  97. data/lib/carray/frame/concat.rb +50 -6
  98. data/lib/carray/frame/csv_parser.rb +13 -4
  99. data/lib/carray/frame/frame.rb +38 -6
  100. data/lib/carray/frame/group.rb +67 -40
  101. data/lib/carray/frame/io.rb +10 -5
  102. data/lib/carray/frame/join.rb +20 -6
  103. data/lib/carray/frame/records.rb +6 -4
  104. data/lib/carray/frame/reshape.rb +25 -5
  105. data/lib/carray/frame/verbs.rb +78 -25
  106. data/lib/carray/frame/windows.rb +30 -6
  107. data/lib/carray/fuse_source.rb +4 -2
  108. data/lib/carray/iterator.rb +8 -0
  109. data/lib/carray/key_windows.rb +10 -4
  110. data/lib/carray/lazy.rb +12 -17
  111. data/lib/carray/methods/bin.rb +3 -0
  112. data/lib/carray/methods/composition.rb +1 -1
  113. data/lib/carray/methods/factorize.rb +2 -0
  114. data/lib/carray/methods/gather_nd.rb +4 -1
  115. data/lib/carray/methods/is_in.rb +23 -3
  116. data/lib/carray/methods/locate_addr.rb +63 -12
  117. data/lib/carray/methods/repeat.rb +11 -1
  118. data/lib/carray/record_order.rb +35 -6
  119. data/lib/carray/runtime.rb +10 -0
  120. data/lib/carray/segment_iterator.rb +19 -16
  121. data/lib/carray/serialize.rb +22 -0
  122. data/lib/carray/slab_iterator.rb +1 -1
  123. data/lib/carray/stack.rb +3 -3
  124. data/lib/carray/string.rb +6 -6
  125. data/lib/carray/string_operation_extension.rb +25 -4
  126. data/lib/carray/struct_builder.rb +4 -1
  127. data/lib/carray/time.rb +190 -46
  128. data/lib/carray/window_iterator.rb +48 -12
  129. data/yard-stubs/ca_obj_bitfield.rb +4 -1
  130. data/yard-stubs/ca_obj_refer.rb +3 -0
  131. data/yard-stubs/ca_obj_window.rb +2 -1
  132. data/yard-stubs/carray_access.rb +4 -2
  133. data/yard-stubs/carray_cast.rb +6 -0
  134. data/yard-stubs/carray_count.rb +6 -3
  135. data/yard-stubs/carray_math.rb +5 -0
  136. data/yard-stubs/carray_order.rb +6 -0
  137. data/yard-stubs/carray_scatter.rb +2 -1
  138. data/yard-stubs/carray_test.rb +18 -5
  139. metadata +2 -2
@@ -9,7 +9,8 @@
9
9
  # Provides:
10
10
  # - Postfix math on Float/Integer/Rational (sqrt, exp, log, sin, ...)
11
11
  # enabling scalar/CArray polymorphism: the same expression
12
- # `(0.0415*(t-218.8)).tanh` works whether t is a Float or a CArray.
12
+ # `(0.0415*(t-218.8)).tanh` works whether t is a Float or a CArray,
13
+ # and answers as the array does (`(-1.0).sqrt` is NaN, not an exception).
13
14
  # - The same postfix math on Complex, so an expression written for a
14
15
  # complex CArray still reads for a single cell taken out of it.
15
16
  # - Angle normalisation on Numeric (deg_360 / deg_180 / rad_2pi / rad_pi).
@@ -43,11 +44,18 @@ class CArray
43
44
  # lexically scoped: nothing changes for code that does not opt in.
44
45
  module CoreExtensions
45
46
 
47
+ # What a real scalar answers through the float64 kernel of the array form.
48
+ # The refinement exists so that one expression reads for a scalar and for
49
+ # an array, so it answers as the array does: NaN outside a function's
50
+ # domain where Ruby's Math raises, the sign of a zero, and the accuracy of
51
+ # expm1 and log1p near zero. The cost is one CScalar per call.
52
+ #
46
53
  # @!visibility private
47
54
  MATH_METHODS = %i[
48
55
  sqrt exp log log10
49
56
  sin cos tan sinh cosh tanh
50
57
  asin acos atan asinh acosh atanh
58
+ expm1 log1p rsqrt signbit
51
59
  ].freeze
52
60
 
53
61
  [Float, Integer, Rational].each do |klass|
@@ -55,7 +63,7 @@ class CArray
55
63
  MATH_METHODS.each do |m|
56
64
  module_eval <<~RUBY, __FILE__, __LINE__ + 1
57
65
  def #{m}
58
- Math.#{m}(self)
66
+ CA_FLOAT64(self).#{m}[0]
59
67
  end
60
68
  RUBY
61
69
  end
@@ -68,26 +76,12 @@ class CArray
68
76
  def distance(other)
69
77
  (self.to_f - other.to_f).abs
70
78
  end
71
- # M.1 (PyTorch alignment) scalar polymorphism additions.
72
79
  # `trunc` is left to Ruby's built-in semantics (Float#truncate
73
80
  # returns Integer when arg=0) — its CArray counterpart preserves
74
81
  # data type, so scalar-side use should call .to_f.trunc explicitly.
75
- # Math.expm1 / Math.log1p don't exist in stdlib; hand-roll.
76
- def expm1
77
- Math.exp(self) - 1.0
78
- end
79
- def log1p
80
- Math.log(1.0 + self)
81
- end
82
- def rsqrt
83
- 1.0 / Math.sqrt(self)
84
- end
85
82
  def square
86
83
  self * self
87
84
  end
88
- def signbit
89
- self.negative?
90
- end
91
85
  end
92
86
  end
93
87
 
@@ -109,14 +103,9 @@ class CArray
109
103
  square rsqrt
110
104
  ].freeze
111
105
 
112
- # Complex is not handled the way Float / Integer / Rational are.
113
- # Math.tanh(Complex(1,2)) raises RangeError -- of the stdlib Math
114
- # methods only Math.sqrt accepts a Complex -- so the complex forms
115
- # need their own implementation.
116
- #
117
- # Each one runs the value through a one-element cmplx128 CScalar, so
118
- # the answer comes from the very kernel the array form would have
119
- # used. That is the point: the branch cuts of csqrt / clog / casin /
106
+ # Complex goes the same way, through a one-element cmplx128 CScalar, so
107
+ # the answer comes from the very kernel the array form would have used.
108
+ # That is the point: the branch cuts of csqrt / clog / casin /
120
109
  # cacosh / catanh, and the sign of a zero on either side of them, are
121
110
  # whatever the platform's C99 complex.h says they are, and they are
122
111
  # not the same everywhere. On this machine, for instance,
@@ -143,50 +132,12 @@ class CArray
143
132
  end
144
133
 
145
134
  refine Numeric do
146
- def deg_360
147
- a = self.to_f
148
- fa = a / 360.0
149
- if a >= 0
150
- (fa - fa.floor) * 360.0
151
- else
152
- (fa - fa.ceil + 1) * 360.0
153
- end
154
- end
155
-
156
- def deg_180
157
- a = self.to_f
158
- fa = (a + 180.0) / 360.0
159
- b = if a >= 0
160
- (fa - fa.floor) * 360.0 - 180.0
161
- else
162
- (fa - fa.ceil) * 360.0 - 180.0
163
- end
164
- b += 360.0 if b <= -180.0
165
- b
166
- end
167
-
168
- def rad_2pi
169
- a = self.to_f
170
- two_pi = 2 * Math::PI
171
- fa = a / two_pi
172
- if a >= 0
173
- (fa - fa.floor) * two_pi
174
- else
175
- (fa - fa.ceil + 1) * two_pi
176
- end
177
- end
178
-
179
- def rad_pi
180
- a = self.to_f
181
- two_pi = 2 * Math::PI
182
- fa = (a + Math::PI) / two_pi
183
- b = if a >= 0
184
- (fa - fa.floor) * two_pi - Math::PI
185
- else
186
- (fa - fa.ceil) * two_pi - Math::PI
187
- end
188
- b += two_pi if b <= -Math::PI
189
- b
135
+ %i[deg_360 deg_180 rad_2pi rad_pi].each do |m|
136
+ module_eval <<~RUBY, __FILE__, __LINE__ + 1
137
+ def #{m}
138
+ CA_FLOAT64(self.to_f).#{m}[0]
139
+ end
140
+ RUBY
190
141
  end
191
142
 
192
143
  def eq(other)
@@ -151,6 +151,7 @@ class CArray
151
151
  # @param n [Integer] number of samples.
152
152
  # @return [CArray]
153
153
  def linspace (x1, x2, n = 100)
154
+ raise TypeError, "linspace: n must be an Integer (got #{n.class})" unless n.is_a?(Integer)
154
155
  data_type = self::DataType
155
156
  unless data_type
156
157
  guess = guess_data_type_from_values(x1, x2)
@@ -87,7 +87,7 @@ class CArray
87
87
  values.each_with_index do |s, i|
88
88
  entity[i] = s.nil? ? UNDEF : s.to_s
89
89
  end
90
- CAFixlenString.wrap(entity)
90
+ CAFixlenString.wrap(entity, encoding: CAFixlenString.__send__(:encoding_of_strings, values))
91
91
  end
92
92
 
93
93
  end
@@ -95,6 +95,88 @@ end
95
95
  # Mutable fixed-width String Face: L1 ops + in-place variants. In-place
96
96
  # transforms re-store through the native fixlen path (silent truncate).
97
97
  class CAFixlenString
98
+
99
+ # The greater String of each cell and `other`, cell by cell: `other` is a
100
+ # CAFixlenString of the same shape or one String. The result is as wide as
101
+ # the wider of the two.
102
+ # @return [CAFixlenString]
103
+ def pmax (other)
104
+ refuse_unordered_operands(__callee__, other)
105
+ pick_by_order(other) { |a, b| a.lt(b) }
106
+ end
107
+
108
+ # The lesser String of each cell and `other`; see {#pmax}.
109
+ # @return [CAFixlenString]
110
+ def pmin (other)
111
+ refuse_unordered_operands(__callee__, other)
112
+ pick_by_order(other) { |a, b| a.gt(b) }
113
+ end
114
+
115
+ # A String has no NaN to let win, so these are other names of pmax / pmin.
116
+ alias_method :maximum, :pmax
117
+ alias_method :minimum, :pmin
118
+ alias_method :fmax, :pmax
119
+ alias_method :fmin, :pmin
120
+
121
+ # @overload min_addr(axis: nil)
122
+ # @return [Integer, CArray] the flat address of the smallest String.
123
+ def min_addr (*args, **kw)
124
+ to_string.min_addr(*args, **kw)
125
+ end
126
+
127
+ # @overload max_addr(axis: nil)
128
+ # @return [Integer, CArray] the flat address of the largest String.
129
+ def max_addr (*args, **kw)
130
+ to_string.max_addr(*args, **kw)
131
+ end
132
+
133
+ # A value that is no String is in no cell, and the kernels below take only
134
+ # bytes: these ask the Strings, which answer it as any other value.
135
+ CArray.send(:define_named_methods, self,
136
+ [:eq, :ne, :count, :search, :search_addr, :is_in, :union, :difference, :intersection],
137
+ :through_strings_unless_text, super_block: true)
138
+
139
+ private def through_strings_unless_text (op, args, kw)
140
+ v = args[0]
141
+ return yield(*args, **kw) if args.empty? || v.is_a?(String) || v.is_a?(CArray) ||
142
+ v.is_a?(Array) || UNDEF.equal?(v)
143
+ to_string.public_send(op, *args, **kw)
144
+ end
145
+
146
+ # `other` where the block says self's cell loses to it, else self's cell.
147
+ # Another array of Strings (a CAString, a CAConstString, an object array)
148
+ # is taken as fixed-length strings, so the result is one.
149
+ private def pick_by_order (other)
150
+ return copy.tap { |out| out[] = UNDEF } if UNDEF.equal?(other)
151
+ if other.is_a?(CArray) && !other.is_a?(CAFixlenString)
152
+ other = CArray.string(other) if other.data_type == CA_OBJECT && !other.face?
153
+ other = other.to_fixlen_string if other.respond_to?(:to_fixlen_string)
154
+ end
155
+ yield(self, other).then_else(other, self)
156
+ end
157
+
158
+ # Both branches of a selection at the width of the wider one.
159
+ private def branches_in_common (x, y)
160
+ width = [x, y].map { |v|
161
+ case v
162
+ when CArray then v.bytes
163
+ when String then v.bytesize
164
+ else 1
165
+ end
166
+ }.max
167
+ [x, y].map { |v|
168
+ v.is_a?(CAFixlenString) && !v.scalar? ? v.to_fixlen_string(bytes: width) : v
169
+ }
170
+ end
171
+
172
+ # The encoding a column of these strings holds its text in: that of the
173
+ # strings beyond ASCII, UTF-8 if none is, and BINARY if they disagree,
174
+ # since the column cannot read them all back.
175
+ private_class_method def self.encoding_of_strings (values)
176
+ text = values.compact.map(&:to_s).reject(&:ascii_only?).map(&:encoding).uniq
177
+ text.size > 1 ? Encoding::BINARY : (text.first || Encoding::UTF_8)
178
+ end
179
+
98
180
  # inspect's name for the array: the class and the cell width, which is the
99
181
  # longest string a cell holds.
100
182
  private def inspect_label
@@ -26,13 +26,36 @@ class CAFrame
26
26
  # rows), so the answer is about the type and not the data: a refusal
27
27
  # there means the operation does not apply, while an error in the verb
28
28
  # itself is left to raise. An object column is the exception: its type
29
- # says nothing of what it holds, so one with no present value is left out
30
- # of a verb run over every column. An N-D column (each row a profile) is
29
+ # says nothing of what it holds, so one with no present value, or whose
30
+ # cells do not answer, is left out of a verb run over every column. An N-D column (each row a profile) is
31
31
  # asked, and computed, along the rows.
32
32
  module ColumnVerbs
33
33
 
34
34
  private
35
35
 
36
+ # The row of each cell, UNDEF where the cell is masked: a first or last
37
+ # taken of it picks the row, and the row picks the value of a Face that
38
+ # cannot hold new values, as lag picks it.
39
+ def present_rows (col)
40
+ rows = CArray.int64(col.elements).seq
41
+ rows[col.is_masked] = UNDEF if col.has_mask?
42
+ rows
43
+ end
44
+
45
+ # A verb whose answers lie on the frame's rows, with no column to compute:
46
+ # a frame of no column would also have no row.
47
+ def refuse_no_column_on_rows (verb, out, nrow)
48
+ return unless out.is_a?(Hash) && out.empty? && nrow > 0
49
+ raise ArgumentError, "#{verb}: no column to compute (every column is a key or has no #{verb})"
50
+ end
51
+
52
+ def refuse_unknown_fill_columns (verb, fills)
53
+ return unless fills.is_a?(Hash)
54
+ unknown = fills.keys - @frame.column_names
55
+ return if unknown.empty?
56
+ raise ArgumentError, "#{verb}: fill_value names no column #{unknown.map(&:inspect).join(', ')}"
57
+ end
58
+
36
59
  def over_named_columns (verb, names, key_names, args = [], keywords = {})
37
60
  if names.empty?
38
61
  cols = {}
@@ -41,7 +64,8 @@ class CAFrame
41
64
  col = @frame[name]
42
65
  next if valueless_object_column?(col, verb, args)
43
66
  next unless column_has?(col, verb, args, keywords)
44
- cols[name] = yield(col, name)
67
+ cols[name] = answer_of_every_column(col, name) { |c, n| yield(c, n) }
68
+ cols.delete(name) if cols[name].equal?(NO_ANSWER)
45
69
  end
46
70
  return cols
47
71
  end
@@ -50,7 +74,7 @@ class CAFrame
50
74
  names.map { |name| @frame.__send__(:column_key, name, verb) }, verb)
51
75
  results = keys.map do |key|
52
76
  name = key
53
- col = @frame[key]
77
+ col = @frame.__send__(:named_column, key, verb)
54
78
  unless column_has?(col, verb, args, keywords)
55
79
  raise CArray::DataTypeError,
56
80
  "#{verb}: column #{name.inspect} " \
@@ -61,11 +85,42 @@ class CAFrame
61
85
  results.size == 1 ? results.first : keys.zip(results).to_h
62
86
  end
63
87
 
88
+ NO_ANSWER = Object.new.freeze
89
+ private_constant :NO_ANSWER
90
+
91
+ # What an object column holds shows only in the verb itself: one whose
92
+ # cells do not answer is left out of a verb run over every column, as a
93
+ # column of a type without the operation is, wherever those cells sit.
94
+ # refuse_wrong_reduction_arguments has refused what no column would take.
95
+ def answer_of_every_column (col, name)
96
+ return yield(col, name) if col.face? || col.data_type != CA_OBJECT
97
+ begin
98
+ yield(col, name)
99
+ rescue TypeError, ArgumentError, NoMethodError, RangeError
100
+ NO_ANSWER
101
+ end
102
+ end
103
+
104
+ # The arguments of a reduction, asked of the core on a number before any
105
+ # column, so a wrong one is not taken for a column that does not answer.
106
+ # fill_value: is of each column's type, and stays out of this.
107
+ def refuse_wrong_reduction_arguments (verb, args, keywords)
108
+ return if verb == :count
109
+ CArray.float64(1).public_send(verb, *args, **keywords.reject { |k, _| k == :fill_value })
110
+ rescue CArray::DataTypeError, NoMethodError
111
+ nil
112
+ end
113
+
64
114
  # An object column with no present cell says nothing of what it holds,
65
115
  # so a verb run over every column leaves it out, as select leaves it out
66
- # of every kind; counting the present cells applies to any column.
116
+ # of every kind; counting the present cells applies to any column. A
117
+ # Face kept in object cells (a CAString) with rows, every one masked, is
118
+ # the same: the type would be asked of masked cells only, which answer
119
+ # with the identity whatever the type. With no rows the type is asked of
120
+ # blank cells of its kind.
67
121
  def valueless_object_column? (col, verb, args)
68
- return false if col.face? || col.data_type != CA_OBJECT
122
+ return false if col.data_type != CA_OBJECT
123
+ return false if col.face? && col.elements == 0
69
124
  return false if %i[count_not_masked count_masked].include?(verb)
70
125
  return false if verb == :count && args.empty?
71
126
  col.count_not_masked == 0
@@ -52,10 +52,13 @@ class CAFrame
52
52
  index_pieces(frames, verb: "meld")
53
53
  check_trailing_shapes(frames, names, verb: "meld")
54
54
  frames = frames_with_rows(frames)
55
+ if first.index
56
+ check_same_column_type(frames.map(&:index), first.axis_name, verb: "meld")
57
+ end
55
58
  cols = {}
56
59
  names.each do |name|
57
60
  check_same_column_type(frames.map { |f| f[name] }, name, verb: "meld")
58
- cols[name] = CArray.meld(frames.map { |f| f[name] }, axis: 0)
61
+ cols[name] = welding_column(name, "meld") { CArray.meld(frames.map { |f| f[name] }, axis: 0) }
59
62
  end
60
63
  new(cols, axis_name: first.axis_name, index: meld_index(frames))
61
64
  end
@@ -85,7 +88,9 @@ class CAFrame
85
88
  frames = frames_with_rows(frames)
86
89
  cols = {}
87
90
  names.each do |name|
88
- cols[name] = CArray.concatenate(frames.map { |f| f[name] })
91
+ pieces = face_pieces_in_one_state(frames.map { |f| f[name] })
92
+ refuse_rounded_pieces(pieces, "concatenate", "column #{name.inspect}")
93
+ cols[name] = welding_column(name, "concatenate") { CArray.concatenate(pieces) }
89
94
  end
90
95
  new(cols, axis_name: first.axis_name, index: concatenate_index(frames))
91
96
  end
@@ -135,7 +140,8 @@ class CAFrame
135
140
  "stack: axis #{axis} out of range for column #{name.inspect} " \
136
141
  "(a #{parts.first.ndim}-D column takes 1 to #{parts.first.ndim})"
137
142
  end
138
- cols[name] = CArray.stack(parts, axis: axis)
143
+ refuse_rounded_pieces(parts, "stack", "column #{name.inspect}")
144
+ cols[name] = welding_column(name, "stack") { CArray.stack(parts, axis: axis) }
139
145
  end
140
146
  new(cols, axis_name: first.axis_name, index: first.index)
141
147
  end
@@ -194,6 +200,43 @@ class CAFrame
194
200
 
195
201
  # ---- shared validators -------------------------------------------------
196
202
 
203
+ # Pieces of one Face in different states brought to one, as an outer join
204
+ # brings its two keys: times and durations to a common unit, categoricals
205
+ # to the labels of all, the first piece's first.
206
+ def self.face_pieces_in_one_state(pieces)
207
+ first = pieces.first
208
+ return pieces unless first.face? && pieces.all? { |p| p.class == first.class }
209
+ # names-check: units, not names
210
+ if first.respond_to?(:in_common_unit_with) && pieces.map(&:unit).uniq.size > 1
211
+ common = pieces.drop(1).reduce(first) { |acc, p| acc.in_common_unit_with(p).first }
212
+ return pieces.map { |p| p.unit == common.unit ? p : p.to_unit(common.unit) }
213
+ end
214
+ # names-check: label lists, not names
215
+ if first.is_a?(CACategorical) && pieces.map(&:labels).uniq.size > 1
216
+ labels = pieces.map(&:labels).reduce(:|)
217
+ return pieces.map { |p| CArray.object(*p.shape) { p.to_a }.categorize(labels: labels) }
218
+ end
219
+ pieces
220
+ end
221
+ private_class_method :face_pieces_in_one_state
222
+
223
+ # Pieces brought to a float: a value it rounds would be another value.
224
+ def self.refuse_rounded_pieces(pieces, verb, what)
225
+ return if pieces.size < 2 || pieces.any?(&:face?) || pieces.any? { |c| c.data_type == CA_FIXLEN }
226
+ refuse_rounded_integers(pieces, CArray.result_type(*pieces), verb, what)
227
+ end
228
+ private_class_method :refuse_rounded_pieces
229
+
230
+ # A refusal of the pieces reads as the verb the frames were given to, with
231
+ # the column it came from.
232
+ def self.welding_column(name, verb)
233
+ yield
234
+ rescue ArgumentError, CArray::DataTypeError => e
235
+ raise unless e.message.start_with?("#{verb}: ", "can't promote", "CAStack parents")
236
+ raise e.class, "#{verb}: column #{name.inspect}: #{e.message.delete_prefix("#{verb}: ")}"
237
+ end
238
+ private_class_method :welding_column
239
+
197
240
  def self.check_concat_inputs(frames, verb:)
198
241
  raise ArgumentError, "#{verb} requires at least one frame" if frames.empty?
199
242
  unless frames.all? { |f| f.is_a?(CAFrame) }
@@ -267,8 +310,7 @@ class CAFrame
267
310
  def self.meld_index(frames)
268
311
  indexes, axis = index_pieces(frames, verb: "meld")
269
312
  return nil unless indexes
270
- axis # unused; kept for symmetry
271
- CArray.meld(indexes, axis: 0)
313
+ welding_column(axis, "meld") { CArray.meld(indexes, axis: 0) }
272
314
  end
273
315
  private_class_method :meld_index
274
316
 
@@ -276,7 +318,9 @@ class CAFrame
276
318
  def self.concatenate_index(frames)
277
319
  indexes, _axis = index_pieces(frames, verb: "concatenate")
278
320
  return nil unless indexes
279
- CArray.concatenate(indexes)
321
+ indexes = face_pieces_in_one_state(indexes)
322
+ refuse_rounded_pieces(indexes, "concatenate", "the index")
323
+ welding_column(frames.first.axis_name, "concatenate") { CArray.concatenate(indexes) }
280
324
  end
281
325
  private_class_method :concatenate_index
282
326
 
@@ -99,16 +99,24 @@ class CAFrame
99
99
  class Tokenizer
100
100
  attr_reader :sep, :quote, :strip
101
101
 
102
- def initialize(sep, quote, strip)
102
+ # What a separator and a quote have to be, for reading and for writing.
103
+ def self.check_delimiters(sep, quote)
103
104
  unless sep.is_a?(String) && !sep.empty?
104
105
  raise ArgumentError, "sep: must be a non-empty String (got #{sep.inspect})"
105
106
  end
107
+ if sep.match?(/[\r\n]/)
108
+ raise ArgumentError, "sep: cannot hold a line end (got #{sep.inspect})"
109
+ end
106
110
  unless quote.is_a?(String) && quote.size == 1
107
111
  raise ArgumentError, "quote: must be one character (got #{quote.inspect})"
108
112
  end
109
113
  if sep.include?(quote)
110
114
  raise ArgumentError, "sep: #{sep.inspect} cannot contain the quote character"
111
115
  end
116
+ end
117
+
118
+ def initialize(sep, quote, strip)
119
+ Tokenizer.check_delimiters(sep, quote)
112
120
  @sep = sep
113
121
  @quote = quote
114
122
  @strip = strip
@@ -239,7 +247,7 @@ class CAFrame
239
247
  part = part + @sep + parts[i]
240
248
  end
241
249
  inner = part[1...-1]
242
- fields << (inner.include?(@quote) ? inner.gsub(@qq, @quote) : inner)
250
+ fields << (inner.include?(@quote) ? inner.gsub(@qq) { @quote } : inner)
243
251
  elsif part.include?(@quote)
244
252
  return nil
245
253
  else
@@ -438,7 +446,7 @@ class CAFrame
438
446
  CAFrame.__send__(:column_key, c, "read_csv", "columns: a column name") if c.is_a?(Symbol)
439
447
  next if c.is_a?(Integer) && c >= 0
440
448
  next if c.is_a?(Range) && (c.begin.nil? || (c.begin.is_a?(Integer) && c.begin >= 0)) &&
441
- (c.end.nil? || c.end.is_a?(Integer))
449
+ (c.end.nil? || (c.end.is_a?(Integer) && c.end >= 0))
442
450
  raise ArgumentError, "read_csv: columns: takes column names, or indexes from 0 " \
443
451
  "and Ranges of them (got #{c.inspect})"
444
452
  end
@@ -485,6 +493,7 @@ class CAFrame
485
493
  return nil if last.nil? && ncol.nil?
486
494
  last = ncol - 1 if last.nil?
487
495
  last -= 1 if c.end && c.exclude_end?
496
+ raise ArgumentError, "read_csv: columns: #{c.inspect} selects no column" if last < first
488
497
  (first..last).to_a
489
498
  end
490
499
  end
@@ -653,7 +662,7 @@ class CAFrame
653
662
 
654
663
  private def body_in_c(last = nil, chunk_bytes = CHUNK_BYTES)
655
664
  return read_body_into_const_string_columns if last.nil? && @read_text_as_const_string
656
- chunked = last.nil? && @names && @io.respond_to?(:external_encoding) &&
665
+ chunked = last.nil? && @names && @io.respond_to?(:internal_encoding) &&
657
666
  @io.internal_encoding.nil?
658
667
  # Names that have to agree with the first record leave its column
659
668
  # count to C, to be checked against them.
@@ -489,7 +489,7 @@ class CAFrame
489
489
  return false if col.face? || col.data_type != :object
490
490
  found = false
491
491
  col.each do |v|
492
- next if UNDEF.equal?(v)
492
+ next if UNDEF.equal?(v) || v.nil?
493
493
  return false unless v.is_a?(String)
494
494
  found = true
495
495
  end
@@ -650,8 +650,8 @@ class CAFrame
650
650
  verdicts[key] = verdict
651
651
  end
652
652
  # The verdicts are a table about the rows, not a part of this one, so the
653
- # frame's attributes are not carried.
654
- CAFrame.new(verdicts, axis_name: @axis_name, index: @index)
653
+ # frame's attributes are not carried and the index is its own.
654
+ CAFrame.new(verdicts, axis_name: @axis_name, index: @index&.copy)
655
655
  end
656
656
 
657
657
  # A new frame with a column moved to the index and the row axis named
@@ -707,6 +707,7 @@ class CAFrame
707
707
  # The first +n+ rows as a positional view-frame (memo §11.3). +n+ larger
708
708
  # than +nrow+ yields the whole frame; +n+ of 0 yields an empty frame.
709
709
  def head(n = 5)
710
+ raise TypeError, "head: the count is an Integer (got #{n.inspect})" unless n.is_a?(Integer)
710
711
  raise ArgumentError, "head count must be non-negative (got #{n})" if n < 0
711
712
  row_span(0, [n, @nrow].min)
712
713
  end
@@ -714,6 +715,7 @@ class CAFrame
714
715
  # The last +n+ rows as a positional view-frame (memo §11.3). +n+ larger
715
716
  # than +nrow+ yields the whole frame; +n+ of 0 yields an empty frame.
716
717
  def tail(n = 5)
718
+ raise TypeError, "tail: the count is an Integer (got #{n.inspect})" unless n.is_a?(Integer)
717
719
  raise ArgumentError, "tail count must be non-negative (got #{n})" if n < 0
718
720
  row_span(@nrow - [n, @nrow].min, @nrow)
719
721
  end
@@ -728,6 +730,28 @@ class CAFrame
728
730
  # a row Hash and stays that way: zero matches raise KeyError, and duplicate
729
731
  # labels raise (go through +filter { |f| f.index.eq(label) }+ for the
730
732
  # multi-row, frame-returning path). Positional access is +df[i]+.
733
+ # A number as the index compares it without rounding: a whole float beside
734
+ # integers is an integer, and an integer beside floats that no float holds
735
+ # is no label there (a float32 index holds fewer integers than a float64
736
+ # one).
737
+ private def exact_label(label)
738
+ return label if @index.face?
739
+ if @index.integer? && label.is_a?(Float) && label.finite? && label == label.floor
740
+ label.to_i
741
+ elsif @index.float? && label.is_a?(Integer) && !index_float_holds?(label)
742
+ Float::NAN
743
+ else
744
+ label
745
+ end
746
+ end
747
+
748
+ # Whether the index's float type holds +n+ exactly.
749
+ private def index_float_holds?(n)
750
+ f = n.to_f
751
+ f = [f].pack("f").unpack1("f") if @index.data_type == CA_FLOAT32
752
+ f.finite? && f.to_i == n
753
+ end
754
+
731
755
  def at(label)
732
756
  raise ArgumentError, "at requires an index (set one with set_index)" unless @index
733
757
  # An index may hold a masked cell -- an outer join and align both build one
@@ -751,8 +775,8 @@ class CAFrame
751
775
  "at: the index is #{@index.data_type}, so the label must be a number " \
752
776
  "(got #{label.inspect}); convert it first, e.g. at(Float(s)) or at(Integer(s))"
753
777
  end
754
- nan = label.is_a?(Float) && label.nan?
755
- hit = nan ? @index.is_in([label]) : @index.eq(label)
778
+ nan = label.is_a?(Numeric) && label != label
779
+ hit = nan ? @index.is_in([label]) : @index.eq(exact_label(label))
756
780
  pos = hit.where
757
781
  case pos.elements
758
782
  when 0
@@ -1182,7 +1206,7 @@ class CAFrame
1182
1206
  # an index, the spliced-in frame must have one too (its rows need labels).
1183
1207
  private def splice_index(other, lo, hi)
1184
1208
  return nil unless @index
1185
- if other.nrow > 0 && other.index.nil?
1209
+ if other.index.nil?
1186
1210
  raise ArgumentError,
1187
1211
  "splice frame needs an index because the target frame has one"
1188
1212
  end
@@ -1243,6 +1267,8 @@ class CAFrame
1243
1267
  end
1244
1268
 
1245
1269
  private def select_rows(selector)
1270
+ # With no column and no index, nothing else bound-checks the selector.
1271
+ selected_row_mask(selector) if @columns.empty? && @index.nil?
1246
1272
  cols = {}
1247
1273
  @columns.each do |name, col|
1248
1274
  cols[name] = col[selector, *([nil] * (col.ndim - 1))]
@@ -1264,6 +1290,12 @@ class CAFrame
1264
1290
  raise KeyError, "#{lead}no column #{key.inspect}"
1265
1291
  end
1266
1292
 
1293
+ # The column a verb is given by name, the verb leading the refusal of a
1294
+ # name that is no column.
1295
+ private def named_column(key, verb)
1296
+ @columns.fetch(key) { refuse_missing_column(key, verb) }
1297
+ end
1298
+
1267
1299
  # filter takes no keyword. keep_masked:, which it once took, is named
1268
1300
  # with where its work went: the selector settles an undetermined row.
1269
1301
  def self.refuse_filter_keywords(options, verb)