carray 3.0.1 → 3.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +824 -0
  3. data/README.md +5 -6
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_dispatch.c +258 -4
  6. data/ext/ca_axis_group.c +368 -184
  7. data/ext/ca_binop_dispatch.c +54 -32
  8. data/ext/ca_binop_dispatch.h +1 -0
  9. data/ext/ca_categorical_iterator.c +90 -41
  10. data/ext/ca_compare.c +8 -2
  11. data/ext/ca_for_buffer.h +21 -68
  12. data/ext/ca_group_iter.c +27 -11
  13. data/ext/ca_kernel_iterator.c +960 -245
  14. data/ext/ca_kernel_iterator.h +352 -181
  15. data/ext/ca_monop_dispatch.c +4 -4
  16. data/ext/ca_monop_dispatch.h +4 -0
  17. data/ext/ca_obj_array.c +194 -42
  18. data/ext/ca_obj_bincmp.c +152 -121
  19. data/ext/ca_obj_binop.c +107 -210
  20. data/ext/ca_obj_bitarray.c +37 -24
  21. data/ext/ca_obj_bitfield.c +95 -62
  22. data/ext/ca_obj_block.c +5 -4
  23. data/ext/ca_obj_byte_swap.c +40 -40
  24. data/ext/ca_obj_const_string.c +180 -50
  25. data/ext/ca_obj_face.c +428 -62
  26. data/ext/ca_obj_face.h +107 -47
  27. data/ext/ca_obj_fake.c +21 -50
  28. data/ext/ca_obj_farray.c +1 -0
  29. data/ext/ca_obj_field.c +16 -3
  30. data/ext/ca_obj_fixlen_string.c +39 -26
  31. data/ext/ca_obj_grid.c +181 -124
  32. data/ext/ca_obj_meld.c +235 -55
  33. data/ext/ca_obj_moncmp.c +61 -61
  34. data/ext/ca_obj_monop.c +125 -107
  35. data/ext/ca_obj_object.c +551 -233
  36. data/ext/ca_obj_record.c +51 -16
  37. data/ext/ca_obj_reduce.c +5 -8
  38. data/ext/ca_obj_refer.c +117 -33
  39. data/ext/ca_obj_remap.c +57 -17
  40. data/ext/ca_obj_repeat.c +21 -5
  41. data/ext/ca_obj_roll.c +115 -11
  42. data/ext/ca_obj_select.c +88 -75
  43. data/ext/ca_obj_select_axis.c +58 -60
  44. data/ext/ca_obj_shift.c +22 -102
  45. data/ext/ca_obj_stack.c +223 -123
  46. data/ext/ca_obj_stride.c +267 -35
  47. data/ext/ca_obj_string.c +25 -22
  48. data/ext/ca_obj_tile.c +32 -16
  49. data/ext/ca_obj_time.c +23 -25
  50. data/ext/ca_obj_timedelta.c +21 -23
  51. data/ext/ca_obj_transpose.c +12 -3
  52. data/ext/ca_obj_triop.c +110 -117
  53. data/ext/ca_obj_window.c +157 -29
  54. data/ext/ca_op_ipower.c +25 -13
  55. data/ext/ca_op_powi.h +57 -30
  56. data/ext/ca_pow5_table.h +656 -0
  57. data/ext/ca_rng_normal.h +42 -0
  58. data/ext/ca_rng_xoshiro256pp.h +105 -0
  59. data/ext/ca_sort_kernels.h +70 -0
  60. data/ext/ca_sweep_engine.c +385 -198
  61. data/ext/ca_sweep_engine.h +31 -23
  62. data/ext/ca_triop_dispatch.c +4 -1
  63. data/ext/ca_triop_dispatch.h +5 -2
  64. data/ext/caframe_csv_reader.c +463 -0
  65. data/ext/carray.h +90 -28
  66. data/ext/carray_access.c +487 -231
  67. data/ext/carray_address_basis.c +731 -0
  68. data/ext/carray_attribute.c +45 -3
  69. data/ext/carray_bincount.c +32 -2
  70. data/ext/carray_broadcast.c +49 -17
  71. data/ext/carray_call_cfunc.c +667 -511
  72. data/ext/carray_cast.c +1556 -247
  73. data/ext/carray_cast_func.rb +34 -21
  74. data/ext/carray_conversion.c +197 -73
  75. data/ext/carray_copy.c +84 -38
  76. data/ext/carray_core.c +591 -119
  77. data/ext/carray_count.c +148 -96
  78. data/ext/carray_element.c +18 -27
  79. data/ext/carray_factorize.c +604 -291
  80. data/ext/carray_generate.c +76 -40
  81. data/ext/carray_histogram.c +152 -94
  82. data/ext/carray_hold.c +2 -1
  83. data/ext/carray_index_classifier.c +63 -96
  84. data/ext/carray_internal.h +400 -1
  85. data/ext/carray_lazy.c +834 -79
  86. data/ext/carray_loop.c +88 -66
  87. data/ext/carray_mask.c +705 -324
  88. data/ext/carray_math_kernel.h +1 -0
  89. data/ext/carray_median_percentile.c +263 -63
  90. data/ext/carray_memory_view.c +254 -248
  91. data/ext/carray_operator.c +1093 -363
  92. data/ext/carray_order.c +133 -69
  93. data/ext/carray_partition.c +102 -56
  94. data/ext/carray_random.c +582 -204
  95. data/ext/carray_scatter.c +308 -65
  96. data/ext/carray_segment.c +257 -0
  97. data/ext/carray_slab.c +183 -70
  98. data/ext/carray_sort.c +235 -240
  99. data/ext/carray_sort_kernel.c +8 -1
  100. data/ext/carray_struct.c +99 -21
  101. data/ext/carray_test.c +167 -64
  102. data/ext/carray_time_text.c +1182 -0
  103. data/ext/carray_utils.c +363 -98
  104. data/ext/extconf.rb +27 -2
  105. data/ext/mk_call_cfunc.rb +103 -118
  106. data/ext/mk_pow5_table.rb +35 -0
  107. data/ext/mkkernel.rb +1349 -851
  108. data/ext/ruby_carray.c +17 -2
  109. data/ext/version.h +4 -4
  110. data/ext/version.rb +4 -2
  111. data/lib/carray/attribute.rb +39 -0
  112. data/lib/carray/autoload_carray.rb +24 -3
  113. data/lib/carray/autoload_method_extension.rb +18 -0
  114. data/lib/carray/axis_group.rb +245 -20
  115. data/lib/carray/basics.rb +95 -25
  116. data/lib/carray/bincount_nd.rb +6 -0
  117. data/lib/carray/block_iterator.rb +200 -54
  118. data/lib/carray/boolean_reduce.rb +21 -29
  119. data/lib/carray/categorical.rb +244 -59
  120. data/lib/carray/categorical_iterator.rb +391 -757
  121. data/lib/carray/complex.rb +36 -29
  122. data/lib/carray/conditional.rb +94 -7
  123. data/lib/carray/const_string.rb +177 -27
  124. data/lib/carray/construct.rb +40 -0
  125. data/lib/carray/data_type_extension.rb +3 -0
  126. data/lib/carray/data_type_limits.rb +91 -0
  127. data/lib/carray/fixlen_string.rb +1 -1
  128. data/lib/carray/frame/concat.rb +157 -6
  129. data/lib/carray/frame/convert.rb +13 -7
  130. data/lib/carray/frame/csv_parser.rb +590 -76
  131. data/lib/carray/frame/describe.rb +79 -0
  132. data/lib/carray/frame/frame.rb +205 -35
  133. data/lib/carray/frame/group.rb +199 -40
  134. data/lib/carray/frame/io.rb +452 -60
  135. data/lib/carray/frame/join.rb +134 -46
  136. data/lib/carray/frame/records.rb +54 -19
  137. data/lib/carray/frame/reshape.rb +448 -0
  138. data/lib/carray/frame/sort.rb +26 -2
  139. data/lib/carray/frame/verbs.rb +726 -56
  140. data/lib/carray/frame.rb +2 -0
  141. data/lib/carray/fuse_source.rb +111 -2
  142. data/lib/carray/fusion.rb +229 -38
  143. data/lib/carray/inspect.rb +71 -20
  144. data/lib/carray/iterator.rb +157 -2
  145. data/lib/carray/lazy.rb +54 -67
  146. data/lib/carray/mask_gap_fill.rb +66 -50
  147. data/lib/carray/math.rb +32 -19
  148. data/lib/carray/meld_reduce.rb +40 -7
  149. data/lib/carray/methods/align_addr.rb +8 -4
  150. data/lib/carray/methods/bincount.rb +2 -2
  151. data/lib/carray/methods/bit_string.rb +14 -3
  152. data/lib/carray/methods/composition.rb +16 -10
  153. data/lib/carray/methods/discovery_along.rb +74 -0
  154. data/lib/carray/methods/factorize.rb +50 -0
  155. data/lib/carray/methods/gather_nd.rb +13 -2
  156. data/lib/carray/methods/insert_block.rb +9 -14
  157. data/lib/carray/methods/is_in.rb +130 -6
  158. data/lib/carray/methods/join.rb +1 -1
  159. data/lib/carray/methods/locate_addr.rb +235 -21
  160. data/lib/carray/methods/mask_duplicates.rb +35 -1
  161. data/lib/carray/methods/mode.rb +18 -1
  162. data/lib/carray/methods/nunique.rb +22 -1
  163. data/lib/carray/methods/repeat.rb +91 -0
  164. data/lib/carray/methods/resize.rb +21 -23
  165. data/lib/carray/methods/snap.rb +34 -12
  166. data/lib/carray/methods/unique.rb +52 -3
  167. data/lib/carray/record_order.rb +208 -0
  168. data/lib/carray/rng.rb +86 -0
  169. data/lib/carray/runtime.rb +14 -11
  170. data/lib/carray/segment_iterator.rb +990 -0
  171. data/lib/carray/serialize.rb +197 -27
  172. data/lib/carray/slab_iterator.rb +109 -40
  173. data/lib/carray/stack.rb +10 -4
  174. data/lib/carray/string_operation_extension.rb +9 -4
  175. data/lib/carray/struct.rb +142 -51
  176. data/lib/carray/struct_builder.rb +110 -14
  177. data/lib/carray/table.rb +2 -2
  178. data/lib/carray/time.rb +535 -105
  179. data/lib/carray/window_iterator.rb +259 -82
  180. data/lib/carray.rb +4 -2
  181. data/yard-stubs/ca_obj_array.rb +4 -4
  182. data/yard-stubs/ca_obj_bitarray.rb +7 -0
  183. data/yard-stubs/ca_obj_bitfield.rb +23 -12
  184. data/yard-stubs/ca_obj_block.rb +2 -7
  185. data/yard-stubs/ca_obj_fake.rb +1 -1
  186. data/yard-stubs/ca_obj_field.rb +1 -1
  187. data/yard-stubs/ca_obj_refer.rb +3 -3
  188. data/yard-stubs/ca_obj_roll.rb +0 -1
  189. data/yard-stubs/ca_obj_stride.rb +10 -7
  190. data/yard-stubs/ca_obj_transpose.rb +1 -1
  191. data/yard-stubs/ca_obj_window.rb +11 -4
  192. data/yard-stubs/carray_access.rb +6 -5
  193. data/yard-stubs/carray_broadcast.rb +2 -2
  194. data/yard-stubs/carray_cast.rb +78 -43
  195. data/yard-stubs/carray_conversion.rb +13 -1
  196. data/yard-stubs/carray_copy.rb +8 -3
  197. data/yard-stubs/carray_core.rb +3 -83
  198. data/yard-stubs/carray_count.rb +16 -2
  199. data/yard-stubs/carray_lazy.rb +210 -1
  200. data/yard-stubs/carray_loop.rb +4 -1
  201. data/yard-stubs/carray_mask.rb +34 -10
  202. data/yard-stubs/carray_math.rb +1499 -4
  203. data/yard-stubs/carray_median_percentile.rb +26 -8
  204. data/yard-stubs/carray_order.rb +43 -82
  205. data/yard-stubs/carray_scatter.rb +10 -3
  206. data/yard-stubs/carray_segment.rb +72 -0
  207. data/yard-stubs/carray_slab.rb +11 -8
  208. data/yard-stubs/carray_sort.rb +50 -8
  209. data/yard-stubs/carray_test.rb +10 -0
  210. data/yard-stubs/carray_utils.rb +0 -26
  211. metadata +19 -19
  212. data/ext/ca_for_each_element.h +0 -239
  213. data/ext/carray_build_flags.h +0 -3
  214. data/ext/carray_config.h +0 -28
  215. data/ext/carray_kernels_bincmp.c +0 -4446
  216. data/ext/carray_kernels_binop.c +0 -11001
  217. data/ext/carray_kernels_init.c +0 -1131
  218. data/ext/carray_kernels_map.c +0 -3467
  219. data/ext/carray_kernels_moncmp.c +0 -2097
  220. data/ext/carray_kernels_monop.c +0 -18313
  221. data/ext/carray_kernels_reduce_aggregate.c +0 -25837
  222. data/ext/carray_kernels_reduce_boolean.c +0 -330
  223. data/ext/carray_kernels_reduce_cumulative.c +0 -14593
  224. data/ext/carray_kernels_reduce_extreme.c +0 -16948
  225. data/ext/carray_kernels_reduce_variance.c +0 -3910
  226. data/ext/carray_kernels_scan.c +0 -3693
  227. data/ext/carray_kernels_search.c +0 -32138
  228. data/ext/carray_kernels_sort.c +0 -10626
  229. data/ext/carray_kernels_triop.c +0 -1392
@@ -52,6 +52,14 @@ class CArray
52
52
  # framework's read-only chain does not replace the CAField mutable view).
53
53
  # ---------------------------------------------------------------------------
54
54
 
55
+ # Whether a view kept by {#real} / {#imag} was made for this array.
56
+ # `dup` and `clone` copy instance variables, so a copy arrives holding the
57
+ # original's view, whose parent is the original; the copy makes its own.
58
+ def part_of_self? (view)
59
+ view && view.parent.equal?(self)
60
+ end
61
+ private :part_of_self?
62
+
55
63
  # @overload real
56
64
  # Returns the real part of `self` as a zero-copy view. For a
57
65
  # complex array the view is a mutable {CAField} into the
@@ -59,19 +67,16 @@ class CArray
59
67
  # over `self`. Writing to the view updates `self` in place.
60
68
  # @return [CArray]
61
69
  def real
62
- if not @__real__
63
- if complex?
64
- @__real__ = case data_type
65
- when CA_CMPLX64
66
- field(0, CA_FLOAT32)
67
- when CA_CMPLX128
68
- field(0, CA_FLOAT64)
69
- end
70
- else
71
- @__real__ = self[]
72
- end
73
- end
74
- @__real__
70
+ return @__real__ if part_of_self?(@__real__)
71
+ view = if complex?
72
+ case data_type
73
+ when CA_CMPLX64 then field(0, CA_FLOAT32)
74
+ when CA_CMPLX128 then field(0, CA_FLOAT64)
75
+ end
76
+ else
77
+ self[]
78
+ end
79
+ frozen? ? view : (@__real__ = view)
75
80
  end
76
81
 
77
82
  # @overload real=(val)
@@ -85,35 +90,37 @@ class CArray
85
90
  # @overload imag
86
91
  # Returns the imaginary part of `self`. For a complex array the
87
92
  # result is a mutable {CAField} view of the imaginary slot;
88
- # writing to it updates `self` in place. For a real array the
89
- # result is a fresh independent CArray filled with 0.
93
+ # writing to it updates `self` in place. For an object array each
94
+ # cell's `imaginary`; for a real array a fresh independent CArray
95
+ # filled with 0. Masked cells stay masked.
90
96
  # @return [CArray]
91
97
  def imag
92
- if not @__imag__
93
- if complex?
94
- @__imag__ = case data_type
95
- when CA_CMPLX64
96
- field(4, CA_FLOAT32)
97
- when CA_CMPLX128
98
- field(8, CA_FLOAT64)
99
- end
100
- else
101
- @__imag__ = self.template { 0 }
102
- end
98
+ if object?
99
+ return imag_i
100
+ end
101
+ if not complex?
102
+ zero = template { 0 }
103
+ zero[is_masked] = UNDEF if has_mask?
104
+ return zero
103
105
  end
104
- return @__imag__
106
+ return @__imag__ if part_of_self?(@__imag__)
107
+ view = case data_type
108
+ when CA_CMPLX64 then field(4, CA_FLOAT32)
109
+ when CA_CMPLX128 then field(8, CA_FLOAT64)
110
+ end
111
+ frozen? ? view : (@__imag__ = view)
105
112
  end
106
113
 
107
114
  # @overload imag=(val)
108
115
  # Sets the imaginary-part slot to `val` (complex arrays only).
109
116
  # @param val [CArray, Numeric] value to broadcast.
110
117
  # @return [Object] `val`.
111
- # @raise [RuntimeError] when `self` is not a complex array.
118
+ # @raise [CArray::DataTypeError] when `self` is not a complex array.
112
119
  def imag= (val)
113
120
  if complex?
114
121
  imag[] = val
115
122
  else
116
- raise "not a complex array"
123
+ raise CArray::DataTypeError, "not a complex array"
117
124
  end
118
125
  end
119
126
 
@@ -6,7 +6,7 @@
6
6
  # same "pick a value per cell based on a boolean condition" contract but
7
7
  # differ in where the condition lives and how the branches are supplied:
8
8
  #
9
- # cond.then_else(x, y) # boolean receiver, 2 values (eager)
9
+ # cond.then_else(x, y) # boolean receiver, 2 values (lazy if an operand is)
10
10
  # self.replace_where(cond, b) # value receiver, 1 replacement (eager)
11
11
  # self.conditional(cond, f_then, f_else) # value receiver, 2 callables (per-region)
12
12
  #
@@ -25,10 +25,20 @@ class CArray
25
25
  # reading as "if `self` then `x` else `y`". The boolean 2-way
26
26
  # case of {#choose} that additionally propagates `self`'s mask
27
27
  # (`UNDEF` in `self` produces `UNDEF` in the result).
28
+ # The result's data type is the common type of the two branches; a
29
+ # branch given as `UNDEF` masks its cells and leaves the type to the
30
+ # other branch (`cond.then_else(a, UNDEF)` has `a`'s data type).
31
+ # When `self` or a branch is a lazy view, the result is a lazy view
32
+ # (read-only) that joins the expression around it; `copy` makes an
33
+ # array of it. An object result is always computed at once.
28
34
  # @param x [CArray, Numeric, Object] true-branch value(s).
29
35
  # @param y [CArray, Numeric, Object] false-branch value(s).
30
- # @return [CArray] new array with the same shape as `self`.
31
- # @raise [ArgumentError] when `self` is not a boolean CArray.
36
+ # @return [CArray] new array with the same shape as `self`, or a
37
+ # lazy view (CATriOp) when an operand is lazy.
38
+ # @raise [ArgumentError] when `self` is not a boolean CArray, or when
39
+ # an array branch's shape does not pair with `self`'s as an
40
+ # operator's operands would (equal, or size-1 axes of the same ndim;
41
+ # an object or fixlen result needs `self`'s shape exactly).
32
42
  def then_else (x, y)
33
43
  # Guard: self must be boolean. Integer / float receiver would be
34
44
  # silently reinterpreted by the indexer setter (`result[self] = ...`)
@@ -38,10 +48,27 @@ class CArray
38
48
  raise ArgumentError,
39
49
  "then_else: receiver must be a boolean CArray (data_type == CA_BOOLEAN), got #{self.data_type}"
40
50
  end
41
- # Promote data_type from both branches via CArray.result_type
42
- # (a CScalar contributes its own data_type, so CA_INT32(0) keeps int32
43
- # where a bare Ruby Integer would widen to int64).
44
- dt = CArray.result_type(x, y)
51
+ reshaped = [x, y].any? { |v| v.is_a?(CArray) && !v.scalar? && v.shape != shape }
52
+ face = [x, y].find { |v| v.is_a?(CArray) && !v.scalar? && v.face? }
53
+ if face
54
+ refuse_branch_shape(x, y) if reshaped
55
+ return face_then_else(x, y, face)
56
+ end
57
+ # Promote data_type from both branches via CArray.result_type, which
58
+ # lets a scalar branch take the type of an array branch as an operator
59
+ # does (cond.then_else(0, int32_array) is int32). UNDEF is a missing
60
+ # value, not a value of some type: a branch that is UNDEF leaves the
61
+ # type to the other.
62
+ typed = [x, y].reject { |v| v.equal?(UNDEF) }
63
+ dt = typed.empty? ? CA_OBJECT : CArray.result_type(*typed)
64
+ return lazy_then_else(x, y, dt) if lazy_then_else?(x, y, dt)
65
+ # A branch shaped unlike the condition goes through the lazy select,
66
+ # which pairs the shapes by the operator rule (equal, or size-1 axes
67
+ # with the same ndim) and refuses the rest.
68
+ if reshaped
69
+ refuse_branch_shape(x, y) if scalar? || dt == CA_OBJECT || dt == CA_FIXLEN
70
+ return lazy_then_else(x, y, dt).copy
71
+ end
45
72
  # A CScalar (scalar? CArray) is treated as a scalar value, not as a
46
73
  # self-shaped operand: full CArray -> gather/copy, scalar -> broadcast.
47
74
  y_full = y.is_a?(CArray) && !y.scalar?
@@ -60,6 +87,66 @@ class CArray
60
87
  result
61
88
  end
62
89
 
90
+ # A lazy receiver or branch makes the selection a lazy node (CATriOp
91
+ # select), so it joins the expression around it rather than computing
92
+ # it here. The branches carry the result type: a scalar branch becomes
93
+ # a CScalar of it, UNDEF a masked one.
94
+ private def lazy_then_else? (x, y, dt)
95
+ return false if scalar? || dt == CA_OBJECT || dt == CA_FIXLEN
96
+ [self, x, y].any? { |v| v.is_a?(CArray) && v.__lazy_view__? }
97
+ end
98
+
99
+ # The types the lazy select cannot carry keep the strict form: a full
100
+ # branch must have the condition's shape.
101
+ private def refuse_branch_shape (x, y)
102
+ bad = [x, y].find { |v| v.is_a?(CArray) && !v.scalar? && v.shape != shape }
103
+ raise ArgumentError,
104
+ "then_else: branch shape #{bad.shape.inspect} does not match the condition's #{shape.inspect}"
105
+ end
106
+
107
+ private def lazy_then_else (x, y, dt)
108
+ branch = lambda do |v|
109
+ next v if v.is_a?(CArray) && !v.scalar?
110
+ cell = CScalar.new(dt)
111
+ cell[0] = v.is_a?(CArray) ? v[0] : v
112
+ cell
113
+ end
114
+ CATriOp.__build__(self, branch.(x), branch.(y), CATriOp::OP_SELECT)
115
+ end
116
+
117
+ private def face_then_else (x, y, face)
118
+ # A Face branch decides the result: the other branch is written into a
119
+ # copy of it through the Face's own store, so an instant stays an instant.
120
+ # Promoting the two branches to a common data type would compare a Face's
121
+ # surface type with its storage's. A CAConstString cannot be written, so
122
+ # it selects among its strings and packs them again.
123
+ if face.is_a?(CAConstString)
124
+ sx = x.is_a?(CAConstString) ? x.to_string : x
125
+ sy = y.is_a?(CAConstString) ? y.to_string : y
126
+ return CArray.const_string(then_else(sx, sy), encoding: face.encoding)
127
+ end
128
+ x_full = x.is_a?(CArray) && !x.scalar?
129
+ y_full = y.is_a?(CArray) && !y.scalar?
130
+ base = y_full ? y : x
131
+ if base.read_only?
132
+ # No store to write through (a CACategorical's codes): select among
133
+ # the values themselves.
134
+ result = CArray.object(*shape)
135
+ result[] = y_full ? y.to_type(CA_OBJECT) : y
136
+ result[self] = x_full ? x.to_type(CA_OBJECT)[self] : x
137
+ result[self.is_masked] = UNDEF if self.has_mask?
138
+ return result
139
+ end
140
+ result = base.copy
141
+ if y_full
142
+ result[self] = x_full ? x[self] : x
143
+ else
144
+ result[self.not] = y.is_a?(CArray) ? y[0] : y
145
+ end
146
+ result[self.is_masked] = UNDEF if self.has_mask?
147
+ result
148
+ end
149
+
63
150
  # @overload replace_where(cond, b)
64
151
  # Returns a copy of `self` with cells where `cond` is true
65
152
  # replaced by `b`. Functional sibling of the destructive
@@ -5,12 +5,15 @@
5
5
  # CAConstString high-level construction + conversion surface.
6
6
  #
7
7
  # CAConstString itself (the Face + tail buffer + fetch decode + numeric gate) lives
8
- # in ext/ca_obj_const_string.c. This file provides the ergonomic builders that pack
9
- # Ruby Strings into the internal length-prefix buffer + int64 offset entity
10
- # and wrap them via the CAConstString.wrap C primitive.
8
+ # in ext/ca_obj_const_string.c. This file provides the ergonomic builders that
9
+ # pack Ruby Strings into the shared buffer + (start,end) pair entity, via the
10
+ # CAConstString.__build__ C primitive.
11
11
  #
12
- # Internal buffer format: concatenation of `[int32 length][bytes...]` records
13
- # (self-delimiting → one offset per element, permutation-safe view).
12
+ # Internal buffer format: a pure concatenation of the element bytes, with no
13
+ # per-record length prefix (= the Arrow values buffer). Each element carries
14
+ # its own `(start, end)` byte range in the storage, so it is self-describing
15
+ # and a gather / sort / select view over the pairs decodes correctly with no
16
+ # buffer copy.
14
17
  #
15
18
  # ----------------------------------------------------------------------------
16
19
 
@@ -89,21 +92,125 @@ class CAConstString
89
92
  # override the L1 generics where present.
90
93
  include CArray::StringOperationMixin
91
94
 
92
- # @overload sort
93
- # Returns a byte-order sorted view built by gathering the
94
- # offsets with `sort_index`. Neither bytes nor offsets are
95
- # copied.
95
+ # ---- ordering family ---------------------------------------------------
96
+ #
97
+ # A storage cell is a `(start, end)` byte range into the shared buffer, so
98
+ # storage order is the order the strings were packed in, not the order they
99
+ # compare in. Without these overrides the family sorted the offsets: the
100
+ # answers came back well-formed and wrong -- `%w[pear apple].sort_addr` gave
101
+ # the identity, and `partition_copy` gave NUL bytes.
102
+ #
103
+ # Every member therefore reads the bytes. The flat forms do it natively
104
+ # (a byte-memcmp scan over the packed buffer, an order of magnitude faster
105
+ # than decoding a Ruby String per cell), the per-axis forms through
106
+ # {#to_string}, where a cell is the string and CArray's own kernels apply.
107
+ # Arguments are forwarded verbatim, so `kind:` / `masked_position:` /
108
+ # `keep_axis:` mean here what they mean on CArray.
109
+ #
110
+ # Anything whose cells are strings comes back a CAConstString, gathered
111
+ # over the same buffer where it can be (sort / sort_copy) and rebuilt where
112
+ # it cannot. Indices, addresses and counts need no conversion.
113
+
114
+ # @overload sort(axis: nil, kind: :quick, masked_position: :last)
115
+ # Returns a sorted {CAConstString} view, gathered over the same buffer
116
+ # and offsets. With no `axis:` the array is flattened first, as
117
+ # `CArray#sort` does.
96
118
  # @return [CAConstString]
97
- def sort
98
- self[sort_index]
119
+ def sort (*args, **kw)
120
+ addr = sort_addr(*args, **kw)
121
+ kw[:axis].nil? ? self[addr.flatten] : self[addr]
122
+ end
123
+
124
+ # @overload sort_copy(axis: nil, kind: :quick, masked_position: :last)
125
+ # Returns an owned sorted {CAConstString}; the materialised counterpart
126
+ # to {#sort}.
127
+ # @return [CAConstString]
128
+ def sort_copy (*args, **kw)
129
+ sort(*args, **kw).copy
130
+ end
131
+
132
+ # @overload sort_addr(axis: nil, kind: :quick, masked_position: :last)
133
+ # Returns the view-flat addresses that index a sort by string order.
134
+ # @return [CArray] `:int64` addresses.
135
+ def sort_addr (*args, **kw)
136
+ # The native scan has no notion of an incomparable sentinel, so a masked
137
+ # column goes the same way a per-axis one does.
138
+ return __sort_addr_bytes__ if args.empty? && kw.empty? && ! has_mask?
139
+ to_string.sort_addr(*args, **kw)
140
+ end
141
+
142
+ # @overload sort_index(axis: nil, kind: :quick, masked_position: :last)
143
+ # Returns the per-fiber indices that index a sort by string order.
144
+ # @return [CArray] `:int64` indices.
145
+ def sort_index (*args, **kw)
146
+ to_string.sort_index(*args, **kw)
147
+ end
148
+
149
+ # @overload rank_index(axis: nil)
150
+ # @return [CArray] each cell's rank in string order.
151
+ def rank_index (*args, **kw)
152
+ to_string.rank_index(*args, **kw)
153
+ end
154
+
155
+ # @overload order(*args)
156
+ # @return [CArray] the ordering of the cells by string order.
157
+ def order (*args, **kw)
158
+ to_string.order(*args, **kw)
159
+ end
160
+
161
+ # @overload min(axis: nil, keep_axis: false)
162
+ # Returns the byte-smallest string, skipping masked cells; UNDEF when
163
+ # every cell is masked or the array is empty.
164
+ # @return [String, CAConstString]
165
+ def min (*args, **kw)
166
+ return __min_bytes__ || UNDEF if args.empty? && kw.empty?
167
+ const_string_lift(to_string.min(*args, **kw))
168
+ end
169
+
170
+ # @overload max(axis: nil, keep_axis: false)
171
+ # Byte-largest counterpart of {#min}.
172
+ # @return [String, CAConstString]
173
+ def max (*args, **kw)
174
+ return __max_bytes__ || UNDEF if args.empty? && kw.empty?
175
+ const_string_lift(to_string.max(*args, **kw))
176
+ end
177
+
178
+ # @overload minmax(axis: nil, keep_axis: false)
179
+ # @return [Array] `[min, max]`.
180
+ def minmax (*args, **kw)
181
+ return [min, max] if args.empty? && kw.empty?
182
+ to_string.minmax(*args, **kw).map { |r| const_string_lift(r) }
99
183
  end
100
184
 
101
- # @overload sort_copy
102
- # Returns an owned sorted {CAConstString}; the materialised
103
- # counterpart to {#sort}.
185
+ # @overload clip(min, max = nil, fill_value = nil, lfill: nil, ufill: nil)
186
+ # Clips the strings in byte order (see {CArray#clip}).
104
187
  # @return [CAConstString]
105
- def sort_copy
106
- self[sort_index].copy
188
+ def clip (*args, **kw)
189
+ const_string_lift(to_string.clip(*args, **kw))
190
+ end
191
+
192
+ # @overload min_index(axis: nil)
193
+ # @return [Integer, CArray] where the byte-smallest string sits.
194
+ def min_index (*args, **kw)
195
+ to_string.min_index(*args, **kw)
196
+ end
197
+
198
+ # @overload max_index(axis: nil)
199
+ # @return [Integer, CArray] where the byte-largest string sits.
200
+ def max_index (*args, **kw)
201
+ to_string.max_index(*args, **kw)
202
+ end
203
+
204
+ # @overload partition_copy(kth, axis: nil)
205
+ # @return [CAConstString] partitioned about the `kth` string in order.
206
+ def partition_copy (*args, **kw)
207
+ const_string_lift(to_string.partition_copy(*args, **kw))
208
+ end
209
+
210
+ # @overload partition_index(kth, axis: nil)
211
+ # @return [CArray] the indices that partition about the `kth` string.
212
+ def partition_index (*args, **kw)
213
+ to_string.partition_index(*args, **kw)
107
214
  end
108
215
 
109
216
  # CAConstString is the one Face with three separate entities -- storage
@@ -144,14 +251,14 @@ class CAConstString
144
251
  # @overload unique(sort: false)
145
252
  # @return [CAConstString] the distinct strings.
146
253
  def unique (sort: false)
147
- to_string.unique(sort: sort).to_const_string
254
+ const_string_lift(to_string.unique(sort: sort))
148
255
  end
149
256
 
150
257
  # @overload value_counts(sort: false)
151
258
  # @return [Array(CAConstString, CArray)] `[values, counts]`.
152
259
  def value_counts (sort: false)
153
260
  values, counts = to_string.value_counts(sort: sort)
154
- [values.to_const_string, counts]
261
+ [const_string_lift(values), counts]
155
262
  end
156
263
 
157
264
  # @overload nunique(axis: nil, keep_axis: false)
@@ -164,11 +271,7 @@ class CAConstString
164
271
  # @return [CAConstString, Array] the most frequent string(s).
165
272
  def mode (axis: nil)
166
273
  r = to_string.mode(axis: axis)
167
- case r
168
- when CArray then r.to_const_string
169
- when Array then r.map { |c| c.is_a?(CArray) ? c.to_const_string : c }
170
- else r
171
- end
274
+ r.is_a?(Array) ? r.map { |c| const_string_lift(c) } : const_string_lift(r)
172
275
  end
173
276
 
174
277
  # @overload is_mode(axis: nil)
@@ -180,7 +283,7 @@ class CAConstString
180
283
  # @overload mask_duplicates(axis: nil)
181
284
  # @return [CAConstString] a copy with every repeat occurrence masked.
182
285
  def mask_duplicates (axis: nil)
183
- to_string.mask_duplicates(axis: axis).to_const_string
286
+ const_string_lift(to_string.mask_duplicates(axis: axis))
184
287
  end
185
288
 
186
289
  # @overload is_in(values)
@@ -192,19 +295,19 @@ class CAConstString
192
295
  # @overload intersection(other, sort: false)
193
296
  # @return [CAConstString] the distinct strings present in both.
194
297
  def intersection (other, sort: false)
195
- to_string.intersection(string_operand(other), sort: sort).to_const_string
298
+ const_string_lift(to_string.intersection(string_operand(other), sort: sort))
196
299
  end
197
300
 
198
301
  # @overload difference(other, sort: false)
199
302
  # @return [CAConstString] the distinct strings only `self` has.
200
303
  def difference (other, sort: false)
201
- to_string.difference(string_operand(other), sort: sort).to_const_string
304
+ const_string_lift(to_string.difference(string_operand(other), sort: sort))
202
305
  end
203
306
 
204
307
  # @overload union(other, sort: false)
205
308
  # @return [CAConstString] the distinct strings of either side.
206
309
  def union (other, sort: false)
207
- to_string.union(string_operand(other), sort: sort).to_const_string
310
+ const_string_lift(to_string.union(string_operand(other), sort: sort))
208
311
  end
209
312
 
210
313
  # @overload locate_addr(ref)
@@ -219,10 +322,57 @@ class CAConstString
219
322
  to_string.categorize(labels: labels, sort_labels: sort_labels)
220
323
  end
221
324
 
325
+ # ---- comparison operators ---------------------------------------------
326
+ #
327
+ # The core operators compare storage, and a storage cell is a byte range:
328
+ # `x < "dd"` compared where each string sits in the buffer and answered
329
+ # true everywhere. They read the strings here instead. `eq` has a native
330
+ # byte scan and stays; `<=>` composes from `>` and `<`, so it follows.
331
+
332
+ # @!method ne(other)
333
+ # @return [CArray] boolean, true where the strings differ.
334
+ # @!method lt(other)
335
+ # @return [CArray] boolean, true where self sorts before `other`.
336
+ # @!method le(other)
337
+ # @return [CArray] boolean, true where self sorts before or equals `other`.
338
+ # @!method gt(other)
339
+ # @return [CArray] boolean, true where self sorts after `other`.
340
+ # @!method ge(other)
341
+ # @return [CArray] boolean, true where self sorts after or equals `other`.
342
+ [:ne, :lt, :le, :gt, :ge].each do |op|
343
+ define_method(op) { |other| to_string.public_send(op, string_operand(other)) }
344
+ end
345
+
346
+ alias < lt
347
+ alias <= le
348
+ alias > gt
349
+ alias >= ge
350
+
222
351
  # A CAConstString operand has to be decoded too; anything else (a CAString,
223
352
  # an object array, an Array) already compares as strings.
224
353
  private def string_operand (other)
225
354
  other.is_a?(CAConstString) ? other.to_string : other
226
355
  end
227
356
 
357
+ # Pack a string-bearing result back into a column of this one's encoding.
358
+ # The encoding has to be carried: the builder checks each element against
359
+ # the column's, so a Shift_JIS column rebuilt as the UTF-8 default raised
360
+ # rather than coming back.
361
+ private def const_string_lift (x)
362
+ x.is_a?(CArray) ? CArray.const_string(x, encoding: encoding) : x
363
+ end
364
+
365
+ end
366
+
367
+ class CAConstString
368
+ # The decimal readers of CAFrame#infer_types / #cast, read from the buffer
369
+ # without a String per cell.
370
+ def __infer_number_type_of_text__
371
+ CArray.__const_string_infer_number_type_of_text__(self)
372
+ end
373
+
374
+ def __read_text_as_number__(type, report = nil)
375
+ out = CArray.__const_string_read_text_as_number__(self, type, report)
376
+ out && ndim > 1 ? out.reshape(*shape) : out
377
+ end
228
378
  end
@@ -256,6 +256,46 @@ class CArray
256
256
  TypeSymbol = nil
257
257
  # @!visibility private
258
258
  DataType = nil
259
+
260
+ # @overload empty(data_type, dim, bytes: nil)
261
+ # Returns a new CArray of `data_type` with the shape `dim`, whose
262
+ # contents are **undefined**. This is {CArray.new} without the fill:
263
+ # the arguments are read the same way, and the cost of zeroing the
264
+ # buffer is not paid. Use it when every cell is written before it is
265
+ # read; otherwise use `CArray.new`, which zero-fills.
266
+ #
267
+ # `CA_OBJECT` is the exception: its cells are zero-initialised
268
+ # anyway, because the garbage collector walks them.
269
+ #
270
+ # @param data_type [Symbol, String, Integer] element type.
271
+ # @param dim [Array<Integer>] shape of the new CArray.
272
+ # @param bytes [Integer, nil] element width, for `:fixlen` only.
273
+ # @return [CArray]
274
+ # @raise [ArgumentError] if a block is given -- filling an array is
275
+ # what `CArray.new` is for.
276
+ # @overload empty(*shape)
277
+ # (Numo / NumPy compatibility) Returns an uninitialised CArray of
278
+ # the given shape. The element type comes from the receiver, so
279
+ # `CArray` itself gives `CA_FLOAT64`; see
280
+ # {DataTypeExtension#empty}.
281
+ # @param shape [Array<Integer>, Array<Array<Integer>>] shape.
282
+ # @return [CArray]
283
+ def self.empty (*args, **options, &block)
284
+ if block
285
+ raise ArgumentError,
286
+ "CArray.empty does not take a block " \
287
+ "(its contents are left undefined); use CArray.new to fill."
288
+ end
289
+ if args.size == 2 and args[1].is_a?(Array)
290
+ return __empty__(*args, **options)
291
+ end
292
+ unless options.empty?
293
+ raise ArgumentError,
294
+ "CArray.empty(*shape) takes no options; bytes: belongs to " \
295
+ "CArray.empty(data_type, dim)"
296
+ end
297
+ super(*args)
298
+ end
259
299
  end
260
300
 
261
301
  class CArray
@@ -237,6 +237,9 @@ class CArray
237
237
  # **uninitialised**. The caller must overwrite the array
238
238
  # before reading from it. `CA_OBJECT` silently falls back to
239
239
  # a zero-VALUE init required for GC safety.
240
+ #
241
+ # carray's own spelling is `CArray.empty(data_type, dim)`, which
242
+ # names the element type instead of taking it from the receiver.
240
243
  # @param shape [Array<Integer>, Array<Array<Integer>>]
241
244
  # @return [CArray]
242
245
  def empty (*args)
@@ -0,0 +1,91 @@
1
+ # What each numeric data type can hold, as constants on the typed
2
+ # classes.
3
+ #
4
+ # CArray::Int32::MAX # => 2147483647
5
+ # CArray::Float32::EPSILON # => 1.1920928955078125e-07
6
+ #
7
+ # The names follow NumPy's iinfo / finfo rather than Ruby's Float, so
8
+ # that MIN and MAX bracket the range for every numeric type alike:
9
+ #
10
+ # CArray::Float64::MIN # => -1.7976931348623157e+308
11
+ # Float::MIN # => 2.2250738585072014e-308
12
+ #
13
+ # Those are not the same number and not the same question. Ruby's
14
+ # Float::MIN is the smallest positive normal, which is TINY here; MIN
15
+ # is the bottom of the range, which for an integer type is the only
16
+ # thing MIN could mean. Code that has to bracket a type without
17
+ # knowing whether it is integer or float reads MIN and MAX and is
18
+ # right either way, and that is the reason for the choice.
19
+ #
20
+ # Not every data type has limits. boolean, fixlen and object have no
21
+ # numeric range, so they carry none of these constants -- asking gives
22
+ # a NameError rather than an answer that would have to be invented.
23
+
24
+ class CArray
25
+
26
+ # The integer widths come from the extension rather than from a table
27
+ # written here, so a platform where a type is not the usual width is
28
+ # described correctly instead of confidently mis-described.
29
+
30
+ {
31
+ Int8 => true, Int16 => true, Int32 => true, Int64 => true,
32
+ UInt8 => false, UInt16 => false, UInt32 => false, UInt64 => false,
33
+ }.each do |klass, signed|
34
+ bits = 8 * CArray.sizeof(klass::TypeSymbol)
35
+ if signed
36
+ klass.const_set(:MIN, -(2 ** (bits - 1)))
37
+ klass.const_set(:MAX, 2 ** (bits - 1) - 1)
38
+ else
39
+ klass.const_set(:MIN, 0)
40
+ klass.const_set(:MAX, 2 ** bits - 1)
41
+ end
42
+ end
43
+
44
+ # float32 and float64 are IEEE-754 binary32 and binary64, and the four
45
+ # values below follow from the format: with `p` significand bits (the
46
+ # implicit one included) and `emax` the exponent of the largest finite
47
+ # value,
48
+ #
49
+ # EPSILON = 2 ** (1 - p) the step from 1.0 to the next float
50
+ # MAX = (2 - EPSILON) * 2 ** emax
51
+ # MIN = -MAX every format here is symmetric
52
+ # TINY = 2 ** (1 - emax) the smallest positive normal
53
+ #
54
+ # All four are exactly representable as a Ruby Float, binary32
55
+ # included, so nothing is rounded on the way in. The width is
56
+ # checked rather than assumed: on a platform where C float or double
57
+ # is not one of these formats the arithmetic below would be wrong,
58
+ # and a wrong limit is worse than a missing one.
59
+
60
+ {
61
+ Float32 => [4, 24, 127],
62
+ Float64 => [8, 53, 1023],
63
+ }.each do |klass, (bytes, precision, max_exponent)|
64
+ actual = CArray.sizeof(klass::TypeSymbol)
65
+ unless actual == bytes
66
+ raise "#{klass} is #{actual} bytes wide, not the #{bytes * 8}-bit IEEE-754 " \
67
+ "format its limits are derived from"
68
+ end
69
+ epsilon = 2.0 ** (1 - precision)
70
+ max = (2.0 - epsilon) * 2.0 ** max_exponent
71
+ klass.const_set(:EPSILON, epsilon)
72
+ klass.const_set(:MAX, max)
73
+ klass.const_set(:MIN, -max)
74
+ klass.const_set(:TINY, 2.0 ** (1 - max_exponent))
75
+ end
76
+
77
+ # A complex type is a pair of floats, so its limits are its
78
+ # component's -- MIN and MAX bound the real and the imaginary part
79
+ # separately, not any magnitude of the pair. This is what
80
+ # np.finfo(np.complex64) answers too.
81
+
82
+ {
83
+ Complex64 => Float32,
84
+ Complex128 => Float64,
85
+ }.each do |klass, component|
86
+ [:MIN, :MAX, :TINY, :EPSILON].each do |name|
87
+ klass.const_set(name, component.const_get(name))
88
+ end
89
+ end
90
+
91
+ end
@@ -3,7 +3,7 @@
3
3
  # CAFixlenString high-level construction surface.
4
4
  #
5
5
  # CAFixlenString itself (the string interpretation of CA_FIXLEN storage) lives
6
- # in ext/ca_obj_string.c. This file provides the ergonomic builder that packs
6
+ # in ext/ca_obj_fixlen_string.c. This file provides the ergonomic builder that packs
7
7
  # Ruby Strings into a fixed-width CA_FIXLEN entity and wraps it via
8
8
  # CAFixlenString.wrap.
9
9
  #