carray 3.0.2 → 3.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +568 -0
  3. data/README.md +6 -6
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_descriptor.h +27 -24
  6. data/ext/ca_axis_dispatch.c +220 -0
  7. data/ext/ca_axis_group.c +201 -123
  8. data/ext/ca_binop_dispatch.c +46 -52
  9. data/ext/ca_binop_dispatch.h +1 -1
  10. data/ext/ca_categorical_iterator.c +249 -183
  11. data/ext/ca_compare.c +7 -12
  12. data/ext/ca_for_buffer.h +21 -68
  13. data/ext/ca_group_iter.c +38 -23
  14. data/ext/ca_kernel_iterator.c +1765 -2076
  15. data/ext/ca_kernel_iterator.h +584 -566
  16. data/ext/ca_monop_dispatch.c +86 -63
  17. data/ext/ca_monop_dispatch.h +5 -0
  18. data/ext/ca_obj_array.c +223 -56
  19. data/ext/ca_obj_bincmp.c +172 -138
  20. data/ext/ca_obj_binop.c +116 -230
  21. data/ext/ca_obj_bitarray.c +40 -26
  22. data/ext/ca_obj_bitfield.c +100 -68
  23. data/ext/ca_obj_block.c +19 -9
  24. data/ext/ca_obj_byte_swap.c +45 -46
  25. data/ext/ca_obj_const_string.c +269 -44
  26. data/ext/ca_obj_face.c +612 -89
  27. data/ext/ca_obj_face.h +111 -47
  28. data/ext/ca_obj_fake.c +36 -79
  29. data/ext/ca_obj_farray.c +7 -3
  30. data/ext/ca_obj_field.c +33 -8
  31. data/ext/ca_obj_fixlen_string.c +23 -24
  32. data/ext/ca_obj_grid.c +541 -210
  33. data/ext/ca_obj_meld.c +331 -111
  34. data/ext/ca_obj_moncmp.c +80 -65
  35. data/ext/ca_obj_monop.c +177 -149
  36. data/ext/ca_obj_object.c +664 -248
  37. data/ext/ca_obj_record.c +63 -23
  38. data/ext/ca_obj_reduce.c +18 -16
  39. data/ext/ca_obj_refer.c +140 -58
  40. data/ext/ca_obj_remap.c +85 -24
  41. data/ext/ca_obj_repeat.c +24 -9
  42. data/ext/ca_obj_roll.c +121 -16
  43. data/ext/ca_obj_select.c +61 -58
  44. data/ext/ca_obj_select_axis.c +73 -107
  45. data/ext/ca_obj_shift.c +32 -102
  46. data/ext/ca_obj_stack.c +372 -122
  47. data/ext/ca_obj_stride.c +348 -140
  48. data/ext/ca_obj_string.c +88 -22
  49. data/ext/ca_obj_tile.c +38 -21
  50. data/ext/ca_obj_time.c +45 -31
  51. data/ext/ca_obj_timedelta.c +43 -29
  52. data/ext/ca_obj_transpose.c +17 -6
  53. data/ext/ca_obj_triop.c +125 -141
  54. data/ext/ca_obj_window.c +330 -44
  55. data/ext/ca_op_ipower.c +24 -11
  56. data/ext/ca_op_powi.h +57 -30
  57. data/ext/ca_pow5_table.h +656 -0
  58. data/ext/ca_sort_kernels.h +70 -0
  59. data/ext/ca_sweep_engine.c +104 -81
  60. data/ext/ca_sweep_engine.h +7 -20
  61. data/ext/ca_triop_dispatch.c +4 -1
  62. data/ext/ca_triop_dispatch.h +5 -2
  63. data/ext/caframe_csv_reader.c +542 -0
  64. data/ext/carray.h +241 -92
  65. data/ext/carray_access.c +608 -276
  66. data/ext/carray_address_basis.c +494 -97
  67. data/ext/carray_attribute.c +129 -4
  68. data/ext/carray_bincount.c +39 -7
  69. data/ext/carray_broadcast.c +56 -19
  70. data/ext/carray_build_flags.h +1 -1
  71. data/ext/carray_call_cfunc.c +108 -28
  72. data/ext/carray_cast.c +1576 -223
  73. data/ext/carray_cast_func.rb +77 -24
  74. data/ext/carray_class.c +4 -3
  75. data/ext/carray_conversion.c +208 -75
  76. data/ext/carray_copy.c +82 -24
  77. data/ext/carray_core.c +646 -162
  78. data/ext/carray_count.c +171 -88
  79. data/ext/carray_element.c +94 -60
  80. data/ext/carray_factorize.c +670 -324
  81. data/ext/carray_generate.c +260 -83
  82. data/ext/carray_histogram.c +209 -106
  83. data/ext/carray_hold.c +14 -16
  84. data/ext/carray_index_classifier.c +422 -243
  85. data/ext/carray_internal.h +692 -3
  86. data/ext/carray_kernels_bincmp.c +7 -0
  87. data/ext/carray_kernels_binop.c +487 -120
  88. data/ext/carray_kernels_init.c +653 -50
  89. data/ext/carray_kernels_moncmp.c +15 -8
  90. data/ext/carray_kernels_monop.c +99 -43
  91. data/ext/carray_kernels_reduce_aggregate.c +3925 -6112
  92. data/ext/carray_kernels_reduce_boolean.c +112 -36
  93. data/ext/carray_kernels_reduce_cumulative.c +2127 -6903
  94. data/ext/carray_kernels_reduce_extreme.c +3032 -2892
  95. data/ext/carray_kernels_reduce_variance.c +439 -500
  96. data/ext/carray_kernels_scan.c +1097 -488
  97. data/ext/carray_kernels_search.c +3787 -2176
  98. data/ext/carray_kernels_sort.c +1200 -920
  99. data/ext/carray_kernels_triop.c +479 -7
  100. data/ext/carray_key_windows.c +216 -0
  101. data/ext/carray_lazy.c +967 -153
  102. data/ext/carray_loop.c +81 -65
  103. data/ext/carray_mask.c +910 -324
  104. data/ext/carray_math_kernel.h +1 -0
  105. data/ext/carray_median_percentile.c +299 -100
  106. data/ext/carray_memory_view.c +872 -758
  107. data/ext/carray_operator.c +1171 -364
  108. data/ext/carray_order.c +505 -133
  109. data/ext/carray_partition.c +112 -65
  110. data/ext/carray_random.c +327 -239
  111. data/ext/carray_round.c +273 -0
  112. data/ext/carray_scatter.c +346 -78
  113. data/ext/carray_segment.c +257 -0
  114. data/ext/carray_slab.c +249 -86
  115. data/ext/carray_sort.c +224 -227
  116. data/ext/carray_sort_kernel.c +8 -1
  117. data/ext/carray_struct.c +118 -39
  118. data/ext/carray_test.c +193 -69
  119. data/ext/carray_time_text.c +1182 -0
  120. data/ext/carray_undef.c +4 -0
  121. data/ext/carray_utils.c +579 -113
  122. data/ext/extconf.rb +27 -2
  123. data/ext/mk_call_cfunc.rb +19 -4
  124. data/ext/mk_pow5_table.rb +35 -0
  125. data/ext/mkkernel.rb +1202 -1014
  126. data/ext/ruby_carray.c +17 -5
  127. data/ext/version.h +4 -4
  128. data/ext/version.rb +4 -2
  129. data/lib/carray/attribute.rb +160 -17
  130. data/lib/carray/autoload_carray.rb +23 -27
  131. data/lib/carray/autoload_method_extension.rb +20 -5
  132. data/lib/carray/axis_group.rb +360 -63
  133. data/lib/carray/basics.rb +162 -28
  134. data/lib/carray/bincount_nd.rb +18 -2
  135. data/lib/carray/block_iterator.rb +231 -58
  136. data/lib/carray/boolean_reduce.rb +34 -42
  137. data/lib/carray/categorical.rb +242 -52
  138. data/lib/carray/categorical_iterator.rb +437 -827
  139. data/lib/carray/complex.rb +41 -34
  140. data/lib/carray/conditional.rb +150 -14
  141. data/lib/carray/const_string.rb +59 -4
  142. data/lib/carray/construct.rb +15 -11
  143. data/lib/carray/data_type_extension.rb +1 -0
  144. data/lib/carray/distance_limit.rb +77 -0
  145. data/lib/carray/fixlen_string.rb +7 -1
  146. data/lib/carray/frame/column_verbs.rb +70 -0
  147. data/lib/carray/frame/concat.rb +157 -6
  148. data/lib/carray/frame/convert.rb +82 -7
  149. data/lib/carray/frame/csv_parser.rb +688 -87
  150. data/lib/carray/frame/describe.rb +80 -0
  151. data/lib/carray/frame/frame.rb +369 -45
  152. data/lib/carray/frame/group.rb +631 -90
  153. data/lib/carray/frame/io.rb +506 -79
  154. data/lib/carray/frame/join.rb +411 -62
  155. data/lib/carray/frame/records.rb +116 -19
  156. data/lib/carray/frame/reshape.rb +455 -0
  157. data/lib/carray/frame/sort.rb +30 -18
  158. data/lib/carray/frame/verbs.rb +868 -61
  159. data/lib/carray/frame/windows.rb +117 -0
  160. data/lib/carray/frame.rb +4 -0
  161. data/lib/carray/fuse_source.rb +118 -2
  162. data/lib/carray/fusion.rb +257 -42
  163. data/lib/carray/histogram.rb +6 -1
  164. data/lib/carray/inspect.rb +54 -13
  165. data/lib/carray/iterator.rb +99 -9
  166. data/lib/carray/key_windows.rb +620 -0
  167. data/lib/carray/lazy.rb +216 -193
  168. data/lib/carray/mask_gap_fill.rb +112 -92
  169. data/lib/carray/math.rb +32 -19
  170. data/lib/carray/meld_reduce.rb +52 -21
  171. data/lib/carray/methods/align_addr.rb +8 -4
  172. data/lib/carray/methods/bin.rb +26 -18
  173. data/lib/carray/methods/bincount.rb +18 -2
  174. data/lib/carray/methods/bit_string.rb +15 -4
  175. data/lib/carray/methods/choose.rb +16 -3
  176. data/lib/carray/methods/composition.rb +18 -10
  177. data/lib/carray/methods/factorize.rb +2 -2
  178. data/lib/carray/methods/gather_nd.rb +25 -5
  179. data/lib/carray/methods/insert_block.rb +9 -14
  180. data/lib/carray/methods/is_between.rb +59 -0
  181. data/lib/carray/methods/is_in.rb +140 -10
  182. data/lib/carray/methods/join.rb +2 -7
  183. data/lib/carray/methods/lag.rb +49 -0
  184. data/lib/carray/methods/locate_addr.rb +182 -45
  185. data/lib/carray/methods/mode.rb +18 -1
  186. data/lib/carray/methods/nunique.rb +11 -6
  187. data/lib/carray/methods/repeat.rb +1 -20
  188. data/lib/carray/methods/resize.rb +21 -23
  189. data/lib/carray/methods/snap.rb +34 -12
  190. data/lib/carray/methods/unique.rb +11 -2
  191. data/lib/carray/methods/value_counts.rb +4 -11
  192. data/lib/carray/record_order.rb +208 -0
  193. data/lib/carray/rng.rb +3 -0
  194. data/lib/carray/runtime.rb +65 -17
  195. data/lib/carray/segment_iterator.rb +1221 -0
  196. data/lib/carray/serialize.rb +214 -30
  197. data/lib/carray/slab_iterator.rb +101 -27
  198. data/lib/carray/stack.rb +10 -4
  199. data/lib/carray/string.rb +105 -5
  200. data/lib/carray/string_operation_extension.rb +12 -4
  201. data/lib/carray/struct.rb +137 -58
  202. data/lib/carray/struct_builder.rb +124 -17
  203. data/lib/carray/time.rb +1222 -186
  204. data/lib/carray/window_iterator.rb +242 -81
  205. data/lib/carray.rb +9 -2
  206. data/yard-stubs/ca_obj_array.rb +4 -4
  207. data/yard-stubs/ca_obj_bitarray.rb +7 -0
  208. data/yard-stubs/ca_obj_bitfield.rb +23 -12
  209. data/yard-stubs/ca_obj_fake.rb +1 -1
  210. data/yard-stubs/ca_obj_field.rb +5 -1
  211. data/yard-stubs/ca_obj_grid.rb +21 -10
  212. data/yard-stubs/ca_obj_refer.rb +5 -4
  213. data/yard-stubs/ca_obj_roll.rb +2 -3
  214. data/yard-stubs/ca_obj_shift.rb +5 -1
  215. data/yard-stubs/ca_obj_stride.rb +10 -7
  216. data/yard-stubs/ca_obj_tile.rb +1 -1
  217. data/yard-stubs/ca_obj_transpose.rb +1 -1
  218. data/yard-stubs/ca_obj_window.rb +9 -5
  219. data/yard-stubs/carray_access.rb +11 -5
  220. data/yard-stubs/carray_attribute.rb +0 -5
  221. data/yard-stubs/carray_broadcast.rb +1 -1
  222. data/yard-stubs/carray_cast.rb +88 -43
  223. data/yard-stubs/carray_conversion.rb +15 -2
  224. data/yard-stubs/carray_copy.rb +12 -5
  225. data/yard-stubs/carray_core.rb +3 -3
  226. data/yard-stubs/carray_count.rb +9 -0
  227. data/yard-stubs/carray_element.rb +10 -4
  228. data/yard-stubs/carray_generate.rb +21 -1
  229. data/yard-stubs/carray_lazy.rb +5 -1
  230. data/yard-stubs/carray_loop.rb +4 -1
  231. data/yard-stubs/carray_mask.rb +60 -12
  232. data/yard-stubs/carray_math.rb +52 -14
  233. data/yard-stubs/carray_median_percentile.rb +15 -8
  234. data/yard-stubs/carray_memory_view.rb +40 -23
  235. data/yard-stubs/carray_order.rb +61 -29
  236. data/yard-stubs/carray_random.rb +13 -4
  237. data/yard-stubs/carray_scatter.rb +21 -9
  238. data/yard-stubs/carray_segment.rb +72 -0
  239. data/yard-stubs/carray_slab.rb +10 -2
  240. data/yard-stubs/carray_sort.rb +49 -8
  241. data/yard-stubs/carray_test.rb +15 -3
  242. data/yard-stubs/carray_utils.rb +0 -26
  243. metadata +19 -5
  244. data/ext/ca_for_each_element.h +0 -239
  245. data/ext/carray_kernels_map.c +0 -3467
  246. data/lib/carray/arrow_tensor.rb +0 -401
  247. data/lib/carray/table.rb +0 -108
@@ -63,6 +63,7 @@ class CArray::Serializer # :nodoc:
63
63
 
64
64
  # @!visibility private
65
65
  MAGIC = "_CARRAY3"
66
+ Ractor.make_shareable(MAGIC)
66
67
  # @!visibility private
67
68
  HEADER_BYTES = 256
68
69
  # @!visibility private
@@ -112,21 +113,35 @@ class CArray::Serializer # :nodoc:
112
113
  # The data region is raw contiguous bytes; attribute Hash and
113
114
  # data_class schema (if any) ride in a YAML trailer at the tail.
114
115
  # @param ca [CArray] array to write.
115
- # @param opt [Hash] `:endian` forces the output byte order
116
+ # @param endian [Symbol, Integer, nil] forces the output byte order
116
117
  # (defaults to the host's; a differing value swaps data + header).
117
118
  # @return [CArray] `ca`.
118
119
  # @raise [ArgumentError] when `ca` is a CA_OBJECT array (use
119
120
  # `Marshal.dump(ca)` instead), or carries a data_class the
120
121
  # v1.0 flat-primitive schema cannot express.
121
- def save (ca, **opt)
122
+ # @raise [TypeError] when `ca` is a Face other than a record (save
123
+ # its storage through the Face's own accessor, such as `#ticks` or
124
+ # `#codes`, or its values with `Marshal.dump(ca.to_type(:object))`).
125
+ def save (ca, endian: nil)
122
126
  if ca.data_type == :object
123
127
  raise ArgumentError,
124
128
  "CArray.save cannot serialise a CA_OBJECT array " \
125
129
  "(arbitrary Ruby objects have no portable representation); " \
126
130
  "use Marshal.dump(ca) for a Ruby-only round-trip"
127
131
  end
132
+ # A Face other than a record is refused, as Marshal refuses it: the file
133
+ # would carry its storage without what makes it the Face (a unit, the
134
+ # string buffer, the labels), and load could not put that back.
135
+ if ca.face? && !ca.is_a?(CARecord)
136
+ storage = ca.__send__(:face_storage_method)
137
+ raise TypeError,
138
+ "CArray.save cannot carry a #{ca.class} yet (its Face would be " \
139
+ "lost); " +
140
+ (storage ? "save its storage, which ##{storage} gives, or " : "") +
141
+ "keep its values with Marshal.dump(ca.to_type(:object))"
142
+ end
128
143
 
129
- file_endian = opt[:endian] || CArray.endian
144
+ file_endian = endian || CArray.endian
130
145
  swap = (file_endian != CArray.endian)
131
146
 
132
147
  elements = ca.elements
@@ -170,7 +185,7 @@ class CArray::Serializer # :nodoc:
170
185
 
171
186
  @io.write(self.class.pack_header(header, file_endian))
172
187
 
173
- data_ca = swap ? ca.swap_bytes : ca
188
+ data_ca = swap ? to_other_byte_order(ca) : ca
174
189
  data_ca.dump_binary(@io)
175
190
 
176
191
  if has_mask
@@ -186,18 +201,19 @@ class CArray::Serializer # :nodoc:
186
201
  # Reads a _CARRAY3 payload from the wrapped IO and returns the
187
202
  # reconstructed array, re-wrapping it through its recorded
188
203
  # data_class when the trailer carries one.
189
- # @param opt [Hash] `:data_type` overrides the element type for a
204
+ # @param data_type [Symbol, Integer, nil] overrides the element type for a
190
205
  # bare `CA_FIXLEN` payload.
191
206
  # @return [CArray]
192
- # @raise [RuntimeError] on a bad magic string, corrupt header, or
207
+ # @raise [ArgumentError] on a bad magic string, corrupt header, or
193
208
  # unsupported version.
194
- def load (**opt)
209
+ def load (data_type: nil)
210
+ fixlen_type = data_type
195
211
  buf = @io.read(HEADER_BYTES)
196
212
  unless buf && buf.bytesize == HEADER_BYTES
197
- raise "not a CArray binary data (truncated header)"
213
+ raise ArgumentError, "not a CArray binary data (truncated header)"
198
214
  end
199
215
  if buf[0, 8] != MAGIC
200
- raise "not a CArray binary data (bad magic)"
216
+ raise ArgumentError, "not a CArray binary data (bad magic)"
201
217
  end
202
218
 
203
219
  # The byte order is self-describing via endian_marker at offset 8:
@@ -208,22 +224,23 @@ class CArray::Serializer # :nodoc:
208
224
  when 0x01 then file_endian = CA_BIG_ENDIAN
209
225
  when 0x04 then file_endian = CA_LITTLE_ENDIAN
210
226
  else
211
- raise "corrupt CArray binary data (endian marker mismatch)"
227
+ raise ArgumentError, "corrupt CArray binary data (endian marker mismatch)"
212
228
  end
213
229
  h = self.class.unpack_header(buf, file_endian)
214
230
 
215
231
  if h[:endian_marker] != ENDIAN_MARKER
216
- raise "corrupt CArray binary data (endian marker mismatch)"
232
+ raise ArgumentError, "corrupt CArray binary data (endian marker mismatch)"
217
233
  end
218
234
  if h[:version_major] != VERSION_MAJOR
219
- raise "unsupported CArray binary version #{h[:version_major]}.#{h[:version_minor]}"
235
+ raise ArgumentError, "unsupported CArray binary version #{h[:version_major]}.#{h[:version_minor]}"
220
236
  end
221
237
  if h[:header_bytes] != HEADER_BYTES
222
- raise "unsupported CArray binary header size #{h[:header_bytes]}"
238
+ raise ArgumentError, "unsupported CArray binary header size #{h[:header_bytes]}"
223
239
  end
224
240
  if h[:data_bytes] != h[:elements] * h[:element_bytes]
225
- raise "corrupt CArray binary data (data_bytes cross-check failed)"
241
+ raise ArgumentError, "corrupt CArray binary data (data_bytes cross-check failed)"
226
242
  end
243
+ check_header(h)
227
244
 
228
245
  swap = (file_endian != CArray.endian)
229
246
  data_type = h[:data_type_code]
@@ -231,30 +248,94 @@ class CArray::Serializer # :nodoc:
231
248
  dim = h[:shape][0, ndim]
232
249
  bytes = h[:element_bytes]
233
250
 
234
- if opt[:data_type] and data_type == CArray.data_type_code(CA_FIXLEN)
235
- data_type = opt[:data_type]
251
+ if fixlen_type and data_type == CArray.data_type_code(CA_FIXLEN)
252
+ data_type = fixlen_type
236
253
  end
237
254
 
238
255
  ca = CArray.new(data_type, dim, :bytes => bytes)
239
256
  ca.load_binary(@io)
240
- ca[] = ca.swap_bytes if swap
241
257
 
258
+ mask = nil
242
259
  if h[:has_mask] != 0
243
- ca.mask = 0
244
- ca.mask.load_binary(@io)
260
+ mask = CArray.boolean(*dim)
261
+ mask.load_binary(@io)
245
262
  end
246
263
 
247
264
  if h[:trailer_bytes] > 0
248
265
  trailer_raw = @io.read(h[:trailer_bytes])
266
+ unless trailer_raw && trailer_raw.bytesize == h[:trailer_bytes]
267
+ raise ArgumentError, "corrupt CArray binary data (truncated trailer)"
268
+ end
249
269
  trailer = decode_trailer(trailer_raw)
270
+ unless trailer.is_a?(Hash)
271
+ raise ArgumentError, "corrupt CArray binary data (trailer is not a mapping)"
272
+ end
250
273
  ca = apply_trailer(ca, trailer)
251
274
  end
252
275
 
276
+ # Swap once the trailer is applied and before the mask is: a record is
277
+ # swapped member by member, as the writer swapped it, and only its
278
+ # data_class says where the members are. Swapping the bare fixlen
279
+ # first reversed the whole record.
280
+ ca[] = to_other_byte_order(ca) if swap
281
+
282
+ if mask
283
+ ca.mask = 0
284
+ ca.mask[] = mask
285
+ end
286
+
253
287
  return ca
254
288
  end
255
289
 
256
290
  private
257
291
 
292
+ # `ca` in the other byte order. A member declared with `endian:` holds its
293
+ # bytes in that order on every machine and in every file, so it is left as
294
+ # it is; only the members in machine order are swapped.
295
+ def to_other_byte_order (ca)
296
+ swapped = ca.swap_bytes
297
+ declared = ca.respond_to?(:data_class) && ca.data_class &&
298
+ ca.data_class.fields.select(&:endian)
299
+ return swapped if !declared || declared.empty?
300
+ swapped = swapped.copy
301
+ declared.each { |f| swapped[f.name][] = ca[f.name].value }
302
+ swapped
303
+ end
304
+
305
+ # The header fields that describe the same array have to agree, and a
306
+ # source whose size is known has to hold what the header declares --
307
+ # checked before the array is allocated, so a short file cannot make
308
+ # the reader allocate what it declares.
309
+ def check_header (h)
310
+ ndim = h[:ndim]
311
+ unless ndim >= 1 && ndim <= CA_RANK_MAX
312
+ raise ArgumentError, "corrupt CArray binary data (ndim #{ndim} out of 1..#{CA_RANK_MAX})"
313
+ end
314
+ dim = h[:shape][0, ndim]
315
+ if dim.any?(&:negative?) || dim.inject(:*) != h[:elements]
316
+ raise ArgumentError, "corrupt CArray binary data (shape #{dim.inspect} does not " \
317
+ "hold #{h[:elements]} elements)"
318
+ end
319
+ fixlen = CArray.data_type_code(CA_FIXLEN)
320
+ if h[:data_type_code] != fixlen &&
321
+ h[:data_type_code] <= CArray.data_type_code(CA_OBJECT) &&
322
+ h[:element_bytes] != CArray.sizeof(h[:data_type_code])
323
+ raise ArgumentError, "corrupt CArray binary data (element_bytes #{h[:element_bytes]} " \
324
+ "for data type #{CArray.data_type_name(h[:data_type_code]) rescue h[:data_type_code]})"
325
+ end
326
+ mask_bytes = (h[:has_mask] != 0) ? h[:elements] : 0
327
+ if h[:mask_bytes] != mask_bytes
328
+ raise ArgumentError, "corrupt CArray binary data (mask_bytes cross-check failed)"
329
+ end
330
+ if @io.respond_to?(:size) && @io.respond_to?(:pos)
331
+ declared = h[:data_bytes] + mask_bytes + h[:trailer_bytes]
332
+ if @io.size - @io.pos < declared
333
+ raise ArgumentError, "corrupt CArray binary data (declares #{declared} bytes after " \
334
+ "the header, the source holds #{@io.size - @io.pos})"
335
+ end
336
+ end
337
+ end
338
+
258
339
  # Build the trailer mapping (String-keyed) from the array's
259
340
  # attributes and data_class. Returns {} when there is nothing to
260
341
  # serialise.
@@ -279,7 +360,8 @@ class CArray::Serializer # :nodoc:
279
360
  raise ArgumentError,
280
361
  "CArray.save cannot serialise data_class #{klass.inspect}: " \
281
362
  "member #{f.name.inspect} is a bitfield (the v1.0 Layer 1 " \
282
- "schema is flat-primitive only); use Marshal.dump(ca)"
363
+ "schema is flat-primitive only); Marshal.dump(ca) carries it " \
364
+ "when the struct is named (assigned to a constant)"
283
365
  end
284
366
  pep = SYMBOL_TO_PEP[f.type]
285
367
  unless pep
@@ -287,9 +369,12 @@ class CArray::Serializer # :nodoc:
287
369
  "CArray.save cannot serialise data_class #{klass.inspect}: " \
288
370
  "member #{f.name.inspect} has non-primitive type #{f.type.inspect} " \
289
371
  "(nested struct / template / fixlen are not expressible in the " \
290
- "v1.0 Layer 1 schema); use Marshal.dump(ca)"
372
+ "v1.0 Layer 1 schema); Marshal.dump(ca) carries it " \
373
+ "when the struct is named (assigned to a constant)"
291
374
  end
292
- { "name" => f.name, "type" => pep, "offset" => f.offset, "bytes" => f.bytes }
375
+ m = { "name" => f.name, "type" => pep, "offset" => f.offset, "bytes" => f.bytes }
376
+ m["endian"] = f.endian.to_s if f.endian
377
+ m
293
378
  end
294
379
 
295
380
  schema = {
@@ -310,7 +395,7 @@ class CArray::Serializer # :nodoc:
310
395
  if trailer.key?("attrs")
311
396
  attrs = trailer["attrs"]
312
397
  if attrs.is_a?(Hash) and not attrs.empty?
313
- ca.instance_variable_set(:@attr, attrs)
398
+ ca.set_attrs(attrs)
314
399
  end
315
400
  end
316
401
  ca
@@ -320,6 +405,12 @@ class CArray::Serializer # :nodoc:
320
405
  # synthesise an anonymous struct from the Layer 1 members[], else
321
406
  # leave the raw CA_FIXLEN array untouched.
322
407
  def wrap_data_class (ca, schema)
408
+ members = schema.is_a?(Hash) ? schema["members"] : nil
409
+ unless schema.is_a?(Hash) &&
410
+ (members.nil? || members.is_a?(Array) && members.all?(Hash))
411
+ raise ArgumentError, "corrupt CArray binary data (data_class schema is not a " \
412
+ "mapping with a list of member mappings)"
413
+ end
323
414
  name = schema["name"]
324
415
  if name
325
416
  begin
@@ -352,9 +443,17 @@ class CArray::Serializer # :nodoc:
352
443
  members.each do |m|
353
444
  sym = PEP_TO_SYMBOL[m["type"]]
354
445
  unless sym
355
- raise "unknown data_class member type #{m["type"].inspect} in CArray binary data"
446
+ raise ArgumentError, "unknown data_class member type #{m["type"].inspect} in CArray binary data"
356
447
  end
357
- member sym, m["name"], :offset => m["offset"]
448
+ opt = { :offset => m["offset"] }
449
+ if m.key?("endian")
450
+ endian = %w[big little native preserve].find { |e| e == m["endian"] }
451
+ unless endian
452
+ raise ArgumentError, "unknown data_class member endian #{m["endian"].inspect} in CArray binary data"
453
+ end
454
+ opt[:endian] = endian.to_sym
455
+ end
456
+ member sym, m["name"], opt
358
457
  end
359
458
  end
360
459
  end
@@ -521,7 +620,8 @@ class CArray
521
620
  ["object",
522
621
  target.shape,
523
622
  target.value.to_a,
524
- (target.has_mask? ? target.mask.to_a : nil)]
623
+ (target.has_mask? ? target.mask.to_a : nil),
624
+ target.attrs.to_h]
525
625
  else
526
626
  ["portable", CArray.dump(target)]
527
627
  end
@@ -533,23 +633,107 @@ class CArray
533
633
  # @param data [Array] Marshal payload.
534
634
  # @return [void]
535
635
  def marshal_load (data)
636
+ source = CArray.send(:array_from_marshal_payload, data)
637
+ initialize_copy(source)
638
+ attrs = source.attrs
639
+ set_attrs(attrs) unless attrs.empty?
640
+ end
641
+
642
+ # The array a {#marshal_dump} payload describes.
643
+ def self.array_from_marshal_payload (data)
536
644
  tag, *rest = data
537
645
  case tag
538
646
  when "object"
539
- shape, values, mask = rest
647
+ shape, values, mask, attrs = rest
540
648
  ca = CArray.object(*shape)
541
649
  ca[] = values
542
650
  if mask
543
651
  ca.mask = 0
544
652
  ca.mask[] = mask
545
653
  end
546
- initialize_copy(ca)
654
+ ca.set_attrs(attrs) if attrs && !attrs.empty?
655
+ ca
547
656
  when "portable"
548
- ca = CArray.load(StringIO.new(rest[0]))
549
- initialize_copy(ca)
657
+ CArray.load(StringIO.new(rest[0]))
658
+ when "record"
659
+ data_class, storage, attrs = rest
660
+ ca = CARecord.wrap(array_from_marshal_payload(storage), data_class)
661
+ ca.set_attrs(attrs) if attrs && !attrs.empty?
662
+ ca
550
663
  else
551
664
  raise TypeError, "unrecognised CArray Marshal payload"
552
665
  end
553
666
  end
667
+ private_class_method :array_from_marshal_payload
668
+
669
+ end
670
+
671
+ # Marshal of a view and a scalar: the {CArray#marshal_dump} payload, loaded
672
+ # back by the class method rather than into an empty instance. A view comes
673
+ # back as a plain array (a copy owns its memory, as #copy gives); a scalar
674
+ # comes back a scalar. A wrap is refused below.
675
+ [CAView, CAWrap, CScalar].each do |klass|
676
+ klass.class_eval do
677
+ # A Face other than a record is refused: the payload would carry its
678
+ # storage without what makes it the Face (a unit, labels), and load
679
+ # could not put that back.
680
+ # @return [String]
681
+ def _dump (_level)
682
+ if face?
683
+ storage = face_storage_method
684
+ raise TypeError,
685
+ "Marshal cannot carry a #{self.class} yet (its Face would be lost); " \
686
+ "dump its values with to_type(:object)" +
687
+ (storage ? ", or its storage, which ##{storage} gives" : "")
688
+ end
689
+ Marshal.dump(CArray.instance_method(:marshal_dump).bind_call(self))
690
+ end
691
+ end
692
+ end
693
+
694
+ class CAView
695
+ # @return [CArray] the plain array a {#_dump} string describes.
696
+ def self._load (str)
697
+ CArray.send(:array_from_marshal_payload, Marshal.load(str))
698
+ end
699
+ end
700
+
701
+ class CAWrap
702
+ # A wrap borrows memory, which Marshal cannot carry to another process;
703
+ # Ruby would also try to write the objects that keep the borrowed memory
704
+ # alive. Its copy owns its memory and marshals.
705
+ def _dump (_level)
706
+ raise TypeError,
707
+ "Marshal cannot carry a #{self.class}: it borrows its memory; " \
708
+ "dump .copy instead"
709
+ end
710
+
711
+ def self._load (str)
712
+ CArray.send(:array_from_marshal_payload, Marshal.load(str))
713
+ end
714
+ end
715
+
716
+ class CScalar
717
+ # @return [CScalar] the scalar a {#_dump} string describes.
718
+ def self._load (str)
719
+ a = CArray.send(:array_from_marshal_payload, Marshal.load(str))
720
+ s = CScalar.new(a.data_type, bytes: a.bytes)
721
+ s[0] = a[0]
722
+ s
723
+ end
724
+ end
725
+
726
+ class CARecord
727
+
728
+ # A record array keeps marshal_dump: it round-trips with its class. A
729
+ # named struct is written by name with the record bytes, so every kind of
730
+ # member (nested, fixlen, bit, array) travels. An anonymous struct has no
731
+ # name to write: it goes through the binary format, whose schema
732
+ # describes flat numeric members only.
733
+ def marshal_dump
734
+ return CArray.instance_method(:marshal_dump).bind_call(self) unless data_class.name
735
+ ["record", data_class,
736
+ CArray.instance_method(:marshal_dump).bind_call(copy.parent), attrs.to_h]
737
+ end
554
738
 
555
739
  end
@@ -152,12 +152,45 @@ class CASlabIterator < CAIterator
152
152
  # @!method max_addr
153
153
  # Per-slab flat source address of the maximum.
154
154
  # @return [CArray] one flat address per slab
155
- [:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev, :all, :any,
155
+ #
156
+ # Each takes `min_count:` and `fill_value:` and passes them to the core
157
+ # reduction, as the other iterators do; `all` and `any` take neither, as
158
+ # the core ones do not.
159
+ [:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev,
156
160
  :variancep, :stddevp, :minmax, :min_index, :max_index,
157
161
  :min_addr, :max_addr].each do |op|
158
162
  class_eval <<~RUBY, __FILE__, __LINE__ + 1
159
- def #{op}
160
- @reference.#{op}(axis: @slab_axes)
163
+ def #{op} (min_count: nil, fill_value: nil)
164
+ @reference.#{op}(axis: @slab_axes, **reduce_options(min_count, fill_value))
165
+ end
166
+ RUBY
167
+ end
168
+
169
+ [:all, :any].each do |op|
170
+ class_eval <<~RUBY, __FILE__, __LINE__ + 1
171
+ def #{op} (**kw)
172
+ @reference.#{op}(axis: @slab_axes, **CAIterator.reduce_keywords(:#{op}, kw))
173
+ end
174
+ RUBY
175
+ end
176
+
177
+ # @!method first
178
+ # Per-slab first present value, in row-major order over the slab axes;
179
+ # UNDEF for a slab with none.
180
+ # @return [CArray] one value per slab
181
+ # @!method last
182
+ # Per-slab last present value; UNDEF for a slab with none.
183
+ # @return [CArray] one value per slab
184
+ # @!method nunique
185
+ # Per-slab number of distinct present values (int64); 0 for a slab with
186
+ # none.
187
+ # @return [CArray] one count per slab
188
+ #
189
+ # These take no `min_count:` or `fill_value:`, as the core ones do not.
190
+ [:first, :last, :nunique].each do |op|
191
+ class_eval <<~RUBY, __FILE__, __LINE__ + 1
192
+ def #{op} (**kw)
193
+ @reference.#{op}(axis: @slab_axes, **CAIterator.reduce_keywords(:#{op}, kw))
161
194
  end
162
195
  RUBY
163
196
  end
@@ -167,25 +200,28 @@ class CASlabIterator < CAIterator
167
200
  # No argument counts present (non-masked) cells; `count(UNDEF)` counts
168
201
  # masked cells; `count(v)` counts cells equal to `v`.
169
202
  # @return [CArray] one count per slab
170
- def count (*args)
171
- return count_not_masked if args.empty?
203
+ def count (*args, **kw)
204
+ return count_not_masked(**kw) if args.empty?
172
205
  # CABlock / CAWindow shadow #count with a geometry accessor, and @reference
173
206
  # may be such a view, so dispatch CArray#count explicitly for count(v).
174
- CArray.instance_method(:count).bind_call(@reference, *args, axis: @slab_axes)
207
+ CArray.instance_method(:count).bind_call(@reference, *args, axis: @slab_axes,
208
+ **CAIterator.reduce_keywords(:count, kw))
175
209
  end
176
210
 
177
211
  # @overload count_not_masked
178
212
  # Per-slab count of present (non-masked) cells.
179
213
  # @return [CArray]
180
- def count_not_masked
181
- @reference.count_not_masked(axis: @slab_axes)
214
+ def count_not_masked (**kw)
215
+ @reference.count_not_masked(axis: @slab_axes,
216
+ **CAIterator.reduce_keywords(:count_not_masked, kw))
182
217
  end
183
218
 
184
219
  # @overload count_masked
185
220
  # Per-slab count of masked cells.
186
221
  # @return [CArray]
187
- def count_masked
188
- @reference.count_masked(axis: @slab_axes)
222
+ def count_masked (**kw)
223
+ @reference.count_masked(axis: @slab_axes,
224
+ **CAIterator.reduce_keywords(:count_masked, kw))
189
225
  end
190
226
 
191
227
  # @overload elements
@@ -206,30 +242,31 @@ class CASlabIterator < CAIterator
206
242
  #
207
243
  # Core order statistics take a single integer axis, so a single slab axis is
208
244
  # delegated directly (fast C path) and a multi-axis slab is folded per slab
209
- # with reduce_slab (each slab flattened). As with `CArray#median(axis:)`, a
210
- # masked source raises (the known per-axis masked limitation); strip the mask
211
- # with `ca.value` first if needed.
245
+ # with reduce_slab (each slab flattened). Masked cells are left out, as in
246
+ # `CArray#median(axis:)`.
212
247
 
213
- # @overload median
248
+ # @overload median(min_count: nil, fill_value: nil)
214
249
  # Per-slab median.
215
250
  # @return [CArray]
216
- def median
251
+ def median (min_count: nil, fill_value: nil)
252
+ opts = reduce_options(min_count, fill_value)
217
253
  if @slab_axes.size == 1
218
- @reference.median(axis: @slab_axes[0])
254
+ @reference.median(axis: @slab_axes[0], **opts)
219
255
  else
220
- @reference.reduce_slab(axis: @slab_axes) { |s| s.median }
256
+ @reference.reduce_slab(axis: @slab_axes) { |s| s.median(**opts) }
221
257
  end
222
258
  end
223
259
 
224
- # @overload percentile(*pers)
260
+ # @overload percentile(*pers, min_count: nil, fill_value: nil)
225
261
  # Per-slab percentile(s). One argument returns one CArray; several return an
226
262
  # array of CArrays (as `CArray#percentile`).
227
263
  # @return [CArray, Array<CArray>]
228
- def percentile (*pers)
264
+ def percentile (*pers, min_count: nil, fill_value: nil)
265
+ opts = reduce_options(min_count, fill_value)
229
266
  if @slab_axes.size == 1
230
- @reference.percentile(*pers, axis: @slab_axes[0])
267
+ @reference.percentile(*pers, axis: @slab_axes[0], **opts)
231
268
  else
232
- rs = pers.map { |p| @reference.reduce_slab(axis: @slab_axes) { |s| s.percentile(p) } }
269
+ rs = pers.map { |p| @reference.reduce_slab(axis: @slab_axes) { |s| s.percentile(p, **opts) } }
233
270
  pers.size == 1 ? rs[0] : rs
234
271
  end
235
272
  end
@@ -279,18 +316,18 @@ class CASlabIterator < CAIterator
279
316
  # the same shape as the reference (one weight per cell). Core wsum / wmean
280
317
  # accept multi-axis, so this covers multi-axis slabs directly.
281
318
 
282
- # @overload wsum(weights)
319
+ # @overload wsum(weights, min_count: nil, fill_value: nil)
283
320
  # Per-slab weighted sum, `weights` shaped like the reference.
284
321
  # @return [CArray]
285
- def wsum (weights)
286
- @reference.wsum(weights, axis: @slab_axes)
322
+ def wsum (weights, min_count: nil, fill_value: nil)
323
+ @reference.wsum(weights, axis: @slab_axes, **reduce_options(min_count, fill_value))
287
324
  end
288
325
 
289
- # @overload wmean(weights)
326
+ # @overload wmean(weights, min_count: nil, fill_value: nil)
290
327
  # Per-slab weighted mean, `weights` shaped like the reference.
291
328
  # @return [CArray]
292
- def wmean (weights)
293
- @reference.wmean(weights, axis: @slab_axes)
329
+ def wmean (weights, min_count: nil, fill_value: nil)
330
+ @reference.wmean(weights, axis: @slab_axes, **reduce_options(min_count, fill_value))
294
331
  end
295
332
 
296
333
  # ---- segment scan: within-slab running statistics ---------------------
@@ -328,10 +365,47 @@ class CASlabIterator < CAIterator
328
365
  RUBY
329
366
  end
330
367
 
368
+ # @overload lag(n = 1)
369
+ # Per-slab value `n` steps earlier along the slab axis (UNDEF where the
370
+ # slab has none), as `reference.lag(n, axis: slab_axis)`: a view of the
371
+ # reference. A multi-axis slab has no single direction to step in, so it
372
+ # raises, as the scans do.
373
+ # @return [CArray] reference-shaped
374
+ def lag (n = 1)
375
+ @reference.lag(n, axis: single_scan_axis(:lag))
376
+ end
377
+
378
+ # @overload strip_mask(method:)
379
+ # Per-slab fill of the masked cells along the slab axis, as
380
+ # `reference.strip_mask(method:, axis: slab_axis)`: `:forward` / `:ffill`
381
+ # or `:backward` / `:bfill`. A multi-axis slab raises, as the scans do.
382
+ # @return [CArray] reference-shaped
383
+ def strip_mask (method:)
384
+ CAIterator.hold_backward?(method)
385
+ @reference.strip_mask(method: method, axis: single_scan_axis(:strip_mask))
386
+ end
387
+
388
+ # @overload order(descending: false, method: :ordinal, kind: :quick)
389
+ # Per-slab rank of each cell along the slab axis (int64), as
390
+ # `reference.order(axis: slab_axis, ...)`. A multi-axis slab has no
391
+ # single sort axis, so it raises, as sort_addr does.
392
+ # @return [CArray] reference-shaped
393
+ def order (**kw)
394
+ @reference.order(axis: single_sort_axis(:order), **CAIterator.reduce_keywords(:order, kw))
395
+ end
396
+
331
397
  private
332
398
 
333
399
  # The single slab axis for the order surface (core sort takes one integer
334
400
  # axis). A multi-axis slab has no single sort axis, so raise.
401
+ # The reduction keywords that were given, for the core call.
402
+ def reduce_options (min_count, fill_value)
403
+ opts = {}
404
+ opts[:min_count] = min_count unless min_count.nil?
405
+ opts[:fill_value] = fill_value unless fill_value.nil?
406
+ opts
407
+ end
408
+
335
409
  def single_sort_axis (op)
336
410
  return @slab_axes[0] if @slab_axes.size == 1
337
411
  raise ArgumentError,
data/lib/carray/stack.rb CHANGED
@@ -51,13 +51,15 @@ class CArray
51
51
  # `data_type` and re-wraps homogeneous Face inputs via
52
52
  # `face_lift`. Output `ndim` is one greater than each piece.
53
53
  # @param list [Array<CArray>] pieces to stack; must not be empty.
54
- # @param axis [Integer] position of the new K axis.
54
+ # @param axis [Integer, nil] position of the new K axis; `nil` is
55
+ # the same as `0`.
55
56
  # @param data_type [Symbol, Integer, nil] result `data_type`;
56
57
  # inferred when `nil`.
57
58
  # @return [CArray] CAStack view.
58
59
  # @raise [ArgumentError] when `list` is empty.
59
60
  def self.stack (list, axis: 0, data_type: nil)
60
61
  raise ArgumentError, "stack: list must not be empty" if list.empty?
62
+ axis = 0 if axis.nil?
61
63
  list = CArray.promote_list(list, data_type: data_type)
62
64
  axis = CArray.normalize_axis(axis, list[0].ndim + 1, "stack")
63
65
  CAStack.new(list, axis: axis) # CAStack.new does Face lift internally
@@ -80,8 +82,9 @@ class CArray
80
82
  # Convenience form: a single Array argument is treated as the list.
81
83
  # @param arrays [Array<CArray>] pieces to weld. A single Array
82
84
  # argument is accepted for compatibility with older callers.
83
- # @param axis [Integer] existing axis to extend (normalises negative
84
- # values against the reference ndim).
85
+ # @param axis [Integer, nil] existing axis to extend (normalises
86
+ # negative values against the reference ndim); `nil` is the same
87
+ # as `0`.
85
88
  # @return [CAMeld] view over the welded pieces.
86
89
  # @raise [ArgumentError] when the list is empty, ndim mismatch,
87
90
  # data_type mismatch, or non-axis dim mismatch across pieces
@@ -91,6 +94,7 @@ class CArray
91
94
  arrays = arrays[0]
92
95
  end
93
96
  raise ArgumentError, "meld: list must not be empty" if arrays.empty?
97
+ axis = 0 if axis.nil?
94
98
  first = arrays[0]
95
99
  unless first.is_a?(CArray)
96
100
  raise ArgumentError, "meld: entries must be CArray (got #{first.class})"
@@ -247,8 +251,10 @@ class CArray
247
251
  # entities.
248
252
  # @param axis [Integer] axis to split along.
249
253
  # @return [Array<CArray>] one slice per index along `axis`.
250
- # @raise [ArgumentError] when `axis` is not a single Integer.
254
+ # @raise [ArgumentError] when `axis` is not a single Integer
255
+ # (including `nil`).
251
256
  def split (axis:)
257
+ raise ArgumentError, "split: axis: is required" if axis.nil?
252
258
  if axis.is_a?(Array)
253
259
  raise ArgumentError, "split: axis must be a single Integer"
254
260
  end