carray 3.0.2 → 3.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +304 -0
  3. data/README.md +3 -4
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_dispatch.c +225 -0
  6. data/ext/ca_axis_group.c +177 -99
  7. data/ext/ca_binop_dispatch.c +54 -32
  8. data/ext/ca_binop_dispatch.h +1 -0
  9. data/ext/ca_categorical_iterator.c +66 -71
  10. data/ext/ca_compare.c +8 -2
  11. data/ext/ca_for_buffer.h +21 -68
  12. data/ext/ca_group_iter.c +27 -11
  13. data/ext/ca_kernel_iterator.c +663 -214
  14. data/ext/ca_kernel_iterator.h +231 -167
  15. data/ext/ca_monop_dispatch.c +4 -4
  16. data/ext/ca_monop_dispatch.h +4 -0
  17. data/ext/ca_obj_array.c +132 -22
  18. data/ext/ca_obj_bincmp.c +152 -121
  19. data/ext/ca_obj_binop.c +107 -210
  20. data/ext/ca_obj_bitarray.c +37 -24
  21. data/ext/ca_obj_bitfield.c +95 -62
  22. data/ext/ca_obj_block.c +1 -0
  23. data/ext/ca_obj_byte_swap.c +40 -40
  24. data/ext/ca_obj_const_string.c +96 -25
  25. data/ext/ca_obj_face.c +406 -64
  26. data/ext/ca_obj_face.h +92 -47
  27. data/ext/ca_obj_fake.c +21 -50
  28. data/ext/ca_obj_farray.c +1 -0
  29. data/ext/ca_obj_field.c +16 -3
  30. data/ext/ca_obj_fixlen_string.c +21 -21
  31. data/ext/ca_obj_grid.c +181 -124
  32. data/ext/ca_obj_meld.c +189 -107
  33. data/ext/ca_obj_moncmp.c +61 -61
  34. data/ext/ca_obj_monop.c +125 -107
  35. data/ext/ca_obj_object.c +543 -233
  36. data/ext/ca_obj_record.c +51 -16
  37. data/ext/ca_obj_reduce.c +5 -8
  38. data/ext/ca_obj_refer.c +117 -33
  39. data/ext/ca_obj_remap.c +57 -17
  40. data/ext/ca_obj_repeat.c +21 -5
  41. data/ext/ca_obj_roll.c +115 -11
  42. data/ext/ca_obj_select.c +47 -49
  43. data/ext/ca_obj_select_axis.c +58 -60
  44. data/ext/ca_obj_shift.c +22 -102
  45. data/ext/ca_obj_stack.c +223 -118
  46. data/ext/ca_obj_stride.c +195 -34
  47. data/ext/ca_obj_string.c +17 -18
  48. data/ext/ca_obj_tile.c +32 -16
  49. data/ext/ca_obj_time.c +23 -25
  50. data/ext/ca_obj_timedelta.c +21 -23
  51. data/ext/ca_obj_transpose.c +12 -3
  52. data/ext/ca_obj_triop.c +110 -117
  53. data/ext/ca_obj_window.c +149 -27
  54. data/ext/ca_op_ipower.c +24 -11
  55. data/ext/ca_op_powi.h +57 -30
  56. data/ext/ca_pow5_table.h +656 -0
  57. data/ext/ca_sort_kernels.h +70 -0
  58. data/ext/ca_sweep_engine.c +102 -79
  59. data/ext/ca_sweep_engine.h +7 -20
  60. data/ext/ca_triop_dispatch.c +4 -1
  61. data/ext/ca_triop_dispatch.h +5 -2
  62. data/ext/caframe_csv_reader.c +463 -0
  63. data/ext/carray.h +70 -27
  64. data/ext/carray_access.c +455 -211
  65. data/ext/carray_address_basis.c +177 -36
  66. data/ext/carray_attribute.c +45 -3
  67. data/ext/carray_bincount.c +32 -2
  68. data/ext/carray_broadcast.c +46 -14
  69. data/ext/carray_call_cfunc.c +0 -28
  70. data/ext/carray_cast.c +1444 -209
  71. data/ext/carray_cast_func.rb +34 -21
  72. data/ext/carray_conversion.c +197 -73
  73. data/ext/carray_copy.c +32 -11
  74. data/ext/carray_core.c +514 -122
  75. data/ext/carray_count.c +140 -87
  76. data/ext/carray_element.c +18 -27
  77. data/ext/carray_factorize.c +569 -277
  78. data/ext/carray_generate.c +76 -40
  79. data/ext/carray_histogram.c +152 -94
  80. data/ext/carray_hold.c +2 -1
  81. data/ext/carray_index_classifier.c +63 -96
  82. data/ext/carray_internal.h +385 -3
  83. data/ext/carray_lazy.c +834 -79
  84. data/ext/carray_loop.c +81 -65
  85. data/ext/carray_mask.c +682 -316
  86. data/ext/carray_math_kernel.h +1 -0
  87. data/ext/carray_median_percentile.c +221 -76
  88. data/ext/carray_memory_view.c +254 -248
  89. data/ext/carray_operator.c +1089 -359
  90. data/ext/carray_order.c +133 -69
  91. data/ext/carray_partition.c +102 -56
  92. data/ext/carray_random.c +273 -239
  93. data/ext/carray_scatter.c +308 -65
  94. data/ext/carray_segment.c +257 -0
  95. data/ext/carray_slab.c +170 -70
  96. data/ext/carray_sort.c +215 -218
  97. data/ext/carray_sort_kernel.c +8 -1
  98. data/ext/carray_struct.c +99 -21
  99. data/ext/carray_test.c +167 -64
  100. data/ext/carray_time_text.c +1182 -0
  101. data/ext/carray_utils.c +363 -98
  102. data/ext/extconf.rb +27 -2
  103. data/ext/mk_call_cfunc.rb +0 -2
  104. data/ext/mk_pow5_table.rb +35 -0
  105. data/ext/mkkernel.rb +1076 -846
  106. data/ext/ruby_carray.c +7 -1
  107. data/ext/version.h +4 -4
  108. data/ext/version.rb +4 -2
  109. data/lib/carray/attribute.rb +39 -0
  110. data/lib/carray/autoload_carray.rb +19 -0
  111. data/lib/carray/autoload_method_extension.rb +6 -0
  112. data/lib/carray/axis_group.rb +174 -26
  113. data/lib/carray/basics.rb +91 -25
  114. data/lib/carray/bincount_nd.rb +6 -0
  115. data/lib/carray/block_iterator.rb +118 -48
  116. data/lib/carray/boolean_reduce.rb +21 -29
  117. data/lib/carray/categorical.rb +94 -26
  118. data/lib/carray/categorical_iterator.rb +333 -826
  119. data/lib/carray/complex.rb +36 -29
  120. data/lib/carray/conditional.rb +94 -7
  121. data/lib/carray/const_string.rb +46 -0
  122. data/lib/carray/frame/concat.rb +157 -6
  123. data/lib/carray/frame/convert.rb +13 -7
  124. data/lib/carray/frame/csv_parser.rb +589 -82
  125. data/lib/carray/frame/describe.rb +79 -0
  126. data/lib/carray/frame/frame.rb +124 -25
  127. data/lib/carray/frame/group.rb +189 -63
  128. data/lib/carray/frame/io.rb +412 -72
  129. data/lib/carray/frame/join.rb +134 -46
  130. data/lib/carray/frame/records.rb +37 -16
  131. data/lib/carray/frame/reshape.rb +448 -0
  132. data/lib/carray/frame/sort.rb +26 -2
  133. data/lib/carray/frame/verbs.rb +715 -48
  134. data/lib/carray/frame.rb +2 -0
  135. data/lib/carray/fuse_source.rb +111 -2
  136. data/lib/carray/fusion.rb +229 -38
  137. data/lib/carray/inspect.rb +29 -11
  138. data/lib/carray/iterator.rb +14 -2
  139. data/lib/carray/lazy.rb +54 -30
  140. data/lib/carray/mask_gap_fill.rb +66 -52
  141. data/lib/carray/math.rb +32 -19
  142. data/lib/carray/meld_reduce.rb +40 -7
  143. data/lib/carray/methods/align_addr.rb +8 -4
  144. data/lib/carray/methods/bincount.rb +2 -2
  145. data/lib/carray/methods/bit_string.rb +14 -3
  146. data/lib/carray/methods/composition.rb +16 -10
  147. data/lib/carray/methods/factorize.rb +2 -2
  148. data/lib/carray/methods/gather_nd.rb +13 -2
  149. data/lib/carray/methods/insert_block.rb +9 -14
  150. data/lib/carray/methods/is_in.rb +117 -4
  151. data/lib/carray/methods/join.rb +1 -1
  152. data/lib/carray/methods/locate_addr.rb +177 -36
  153. data/lib/carray/methods/mode.rb +18 -1
  154. data/lib/carray/methods/repeat.rb +1 -20
  155. data/lib/carray/methods/resize.rb +21 -23
  156. data/lib/carray/methods/snap.rb +34 -12
  157. data/lib/carray/methods/unique.rb +11 -2
  158. data/lib/carray/record_order.rb +208 -0
  159. data/lib/carray/runtime.rb +14 -11
  160. data/lib/carray/segment_iterator.rb +990 -0
  161. data/lib/carray/serialize.rb +197 -27
  162. data/lib/carray/slab_iterator.rb +51 -27
  163. data/lib/carray/stack.rb +10 -4
  164. data/lib/carray/string_operation_extension.rb +4 -3
  165. data/lib/carray/struct.rb +142 -51
  166. data/lib/carray/struct_builder.rb +110 -14
  167. data/lib/carray/table.rb +2 -2
  168. data/lib/carray/time.rb +517 -103
  169. data/lib/carray/window_iterator.rb +127 -72
  170. data/lib/carray.rb +2 -2
  171. data/yard-stubs/ca_obj_array.rb +4 -4
  172. data/yard-stubs/ca_obj_bitarray.rb +7 -0
  173. data/yard-stubs/ca_obj_bitfield.rb +23 -12
  174. data/yard-stubs/ca_obj_fake.rb +1 -1
  175. data/yard-stubs/ca_obj_field.rb +1 -1
  176. data/yard-stubs/ca_obj_refer.rb +3 -3
  177. data/yard-stubs/ca_obj_roll.rb +0 -1
  178. data/yard-stubs/ca_obj_stride.rb +10 -7
  179. data/yard-stubs/ca_obj_transpose.rb +1 -1
  180. data/yard-stubs/ca_obj_window.rb +1 -2
  181. data/yard-stubs/carray_access.rb +5 -4
  182. data/yard-stubs/carray_broadcast.rb +1 -1
  183. data/yard-stubs/carray_cast.rb +78 -43
  184. data/yard-stubs/carray_conversion.rb +13 -1
  185. data/yard-stubs/carray_copy.rb +8 -3
  186. data/yard-stubs/carray_core.rb +3 -3
  187. data/yard-stubs/carray_count.rb +9 -0
  188. data/yard-stubs/carray_lazy.rb +5 -1
  189. data/yard-stubs/carray_loop.rb +4 -1
  190. data/yard-stubs/carray_mask.rb +34 -10
  191. data/yard-stubs/carray_math.rb +25 -13
  192. data/yard-stubs/carray_median_percentile.rb +10 -6
  193. data/yard-stubs/carray_order.rb +34 -13
  194. data/yard-stubs/carray_scatter.rb +10 -3
  195. data/yard-stubs/carray_segment.rb +72 -0
  196. data/yard-stubs/carray_slab.rb +2 -1
  197. data/yard-stubs/carray_sort.rb +43 -3
  198. data/yard-stubs/carray_test.rb +10 -0
  199. data/yard-stubs/carray_utils.rb +0 -26
  200. metadata +11 -19
  201. data/ext/ca_for_each_element.h +0 -239
  202. data/ext/carray_build_flags.h +0 -3
  203. data/ext/carray_config.h +0 -28
  204. data/ext/carray_kernels_bincmp.c +0 -4446
  205. data/ext/carray_kernels_binop.c +0 -11001
  206. data/ext/carray_kernels_init.c +0 -1131
  207. data/ext/carray_kernels_map.c +0 -3467
  208. data/ext/carray_kernels_moncmp.c +0 -2097
  209. data/ext/carray_kernels_monop.c +0 -18313
  210. data/ext/carray_kernels_reduce_aggregate.c +0 -26005
  211. data/ext/carray_kernels_reduce_boolean.c +0 -330
  212. data/ext/carray_kernels_reduce_cumulative.c +0 -14862
  213. data/ext/carray_kernels_reduce_extreme.c +0 -17494
  214. data/ext/carray_kernels_reduce_variance.c +0 -3910
  215. data/ext/carray_kernels_scan.c +0 -3693
  216. data/ext/carray_kernels_search.c +0 -32218
  217. data/ext/carray_kernels_sort.c +0 -10626
  218. data/ext/carray_kernels_triop.c +0 -1392
@@ -112,21 +112,31 @@ class CArray::Serializer # :nodoc:
112
112
  # The data region is raw contiguous bytes; attribute Hash and
113
113
  # data_class schema (if any) ride in a YAML trailer at the tail.
114
114
  # @param ca [CArray] array to write.
115
- # @param opt [Hash] `:endian` forces the output byte order
115
+ # @param endian [Symbol, Integer, nil] forces the output byte order
116
116
  # (defaults to the host's; a differing value swaps data + header).
117
117
  # @return [CArray] `ca`.
118
118
  # @raise [ArgumentError] when `ca` is a CA_OBJECT array (use
119
119
  # `Marshal.dump(ca)` instead), or carries a data_class the
120
120
  # v1.0 flat-primitive schema cannot express.
121
- def save (ca, **opt)
121
+ # @raise [TypeError] when `ca` is a Face other than a record (save
122
+ # its storage with `.parent`).
123
+ def save (ca, endian: nil)
122
124
  if ca.data_type == :object
123
125
  raise ArgumentError,
124
126
  "CArray.save cannot serialise a CA_OBJECT array " \
125
127
  "(arbitrary Ruby objects have no portable representation); " \
126
128
  "use Marshal.dump(ca) for a Ruby-only round-trip"
127
129
  end
130
+ # A Face other than a record is refused, as Marshal refuses it: the file
131
+ # would carry its storage without what makes it the Face (a unit, the
132
+ # string buffer, the labels), and load could not put that back.
133
+ if ca.face? && !ca.is_a?(CARecord)
134
+ raise TypeError,
135
+ "CArray.save cannot carry a #{ca.class} yet (its Face would be " \
136
+ "lost); save its storage with .parent"
137
+ end
128
138
 
129
- file_endian = opt[:endian] || CArray.endian
139
+ file_endian = endian || CArray.endian
130
140
  swap = (file_endian != CArray.endian)
131
141
 
132
142
  elements = ca.elements
@@ -170,7 +180,7 @@ class CArray::Serializer # :nodoc:
170
180
 
171
181
  @io.write(self.class.pack_header(header, file_endian))
172
182
 
173
- data_ca = swap ? ca.swap_bytes : ca
183
+ data_ca = swap ? to_other_byte_order(ca) : ca
174
184
  data_ca.dump_binary(@io)
175
185
 
176
186
  if has_mask
@@ -186,18 +196,19 @@ class CArray::Serializer # :nodoc:
186
196
  # Reads a _CARRAY3 payload from the wrapped IO and returns the
187
197
  # reconstructed array, re-wrapping it through its recorded
188
198
  # data_class when the trailer carries one.
189
- # @param opt [Hash] `:data_type` overrides the element type for a
199
+ # @param data_type [Symbol, Integer, nil] overrides the element type for a
190
200
  # bare `CA_FIXLEN` payload.
191
201
  # @return [CArray]
192
- # @raise [RuntimeError] on a bad magic string, corrupt header, or
202
+ # @raise [ArgumentError] on a bad magic string, corrupt header, or
193
203
  # unsupported version.
194
- def load (**opt)
204
+ def load (data_type: nil)
205
+ fixlen_type = data_type
195
206
  buf = @io.read(HEADER_BYTES)
196
207
  unless buf && buf.bytesize == HEADER_BYTES
197
- raise "not a CArray binary data (truncated header)"
208
+ raise ArgumentError, "not a CArray binary data (truncated header)"
198
209
  end
199
210
  if buf[0, 8] != MAGIC
200
- raise "not a CArray binary data (bad magic)"
211
+ raise ArgumentError, "not a CArray binary data (bad magic)"
201
212
  end
202
213
 
203
214
  # The byte order is self-describing via endian_marker at offset 8:
@@ -208,22 +219,23 @@ class CArray::Serializer # :nodoc:
208
219
  when 0x01 then file_endian = CA_BIG_ENDIAN
209
220
  when 0x04 then file_endian = CA_LITTLE_ENDIAN
210
221
  else
211
- raise "corrupt CArray binary data (endian marker mismatch)"
222
+ raise ArgumentError, "corrupt CArray binary data (endian marker mismatch)"
212
223
  end
213
224
  h = self.class.unpack_header(buf, file_endian)
214
225
 
215
226
  if h[:endian_marker] != ENDIAN_MARKER
216
- raise "corrupt CArray binary data (endian marker mismatch)"
227
+ raise ArgumentError, "corrupt CArray binary data (endian marker mismatch)"
217
228
  end
218
229
  if h[:version_major] != VERSION_MAJOR
219
- raise "unsupported CArray binary version #{h[:version_major]}.#{h[:version_minor]}"
230
+ raise ArgumentError, "unsupported CArray binary version #{h[:version_major]}.#{h[:version_minor]}"
220
231
  end
221
232
  if h[:header_bytes] != HEADER_BYTES
222
- raise "unsupported CArray binary header size #{h[:header_bytes]}"
233
+ raise ArgumentError, "unsupported CArray binary header size #{h[:header_bytes]}"
223
234
  end
224
235
  if h[:data_bytes] != h[:elements] * h[:element_bytes]
225
- raise "corrupt CArray binary data (data_bytes cross-check failed)"
236
+ raise ArgumentError, "corrupt CArray binary data (data_bytes cross-check failed)"
226
237
  end
238
+ check_header(h)
227
239
 
228
240
  swap = (file_endian != CArray.endian)
229
241
  data_type = h[:data_type_code]
@@ -231,30 +243,94 @@ class CArray::Serializer # :nodoc:
231
243
  dim = h[:shape][0, ndim]
232
244
  bytes = h[:element_bytes]
233
245
 
234
- if opt[:data_type] and data_type == CArray.data_type_code(CA_FIXLEN)
235
- data_type = opt[:data_type]
246
+ if fixlen_type and data_type == CArray.data_type_code(CA_FIXLEN)
247
+ data_type = fixlen_type
236
248
  end
237
249
 
238
250
  ca = CArray.new(data_type, dim, :bytes => bytes)
239
251
  ca.load_binary(@io)
240
- ca[] = ca.swap_bytes if swap
241
252
 
253
+ mask = nil
242
254
  if h[:has_mask] != 0
243
- ca.mask = 0
244
- ca.mask.load_binary(@io)
255
+ mask = CArray.boolean(*dim)
256
+ mask.load_binary(@io)
245
257
  end
246
258
 
247
259
  if h[:trailer_bytes] > 0
248
260
  trailer_raw = @io.read(h[:trailer_bytes])
261
+ unless trailer_raw && trailer_raw.bytesize == h[:trailer_bytes]
262
+ raise ArgumentError, "corrupt CArray binary data (truncated trailer)"
263
+ end
249
264
  trailer = decode_trailer(trailer_raw)
265
+ unless trailer.is_a?(Hash)
266
+ raise ArgumentError, "corrupt CArray binary data (trailer is not a mapping)"
267
+ end
250
268
  ca = apply_trailer(ca, trailer)
251
269
  end
252
270
 
271
+ # Swap once the trailer is applied and before the mask is: a record is
272
+ # swapped member by member, as the writer swapped it, and only its
273
+ # data_class says where the members are. Swapping the bare fixlen
274
+ # first reversed the whole record.
275
+ ca[] = to_other_byte_order(ca) if swap
276
+
277
+ if mask
278
+ ca.mask = 0
279
+ ca.mask[] = mask
280
+ end
281
+
253
282
  return ca
254
283
  end
255
284
 
256
285
  private
257
286
 
287
+ # `ca` in the other byte order. A member declared with `endian:` holds its
288
+ # bytes in that order on every machine and in every file, so it is left as
289
+ # it is; only the members in machine order are swapped.
290
+ def to_other_byte_order (ca)
291
+ swapped = ca.swap_bytes
292
+ declared = ca.respond_to?(:data_class) && ca.data_class &&
293
+ ca.data_class.fields.select(&:endian)
294
+ return swapped if !declared || declared.empty?
295
+ swapped = swapped.copy
296
+ declared.each { |f| swapped[f.name][] = ca[f.name].value }
297
+ swapped
298
+ end
299
+
300
+ # The header fields that describe the same array have to agree, and a
301
+ # source whose size is known has to hold what the header declares --
302
+ # checked before the array is allocated, so a short file cannot make
303
+ # the reader allocate what it declares.
304
+ def check_header (h)
305
+ ndim = h[:ndim]
306
+ unless ndim >= 1 && ndim <= CA_RANK_MAX
307
+ raise ArgumentError, "corrupt CArray binary data (ndim #{ndim} out of 1..#{CA_RANK_MAX})"
308
+ end
309
+ dim = h[:shape][0, ndim]
310
+ if dim.any?(&:negative?) || dim.inject(:*) != h[:elements]
311
+ raise ArgumentError, "corrupt CArray binary data (shape #{dim.inspect} does not " \
312
+ "hold #{h[:elements]} elements)"
313
+ end
314
+ fixlen = CArray.data_type_code(CA_FIXLEN)
315
+ if h[:data_type_code] != fixlen &&
316
+ h[:data_type_code] <= CArray.data_type_code(CA_OBJECT) &&
317
+ h[:element_bytes] != CArray.sizeof(h[:data_type_code])
318
+ raise ArgumentError, "corrupt CArray binary data (element_bytes #{h[:element_bytes]} " \
319
+ "for data type #{CArray.data_type_name(h[:data_type_code]) rescue h[:data_type_code]})"
320
+ end
321
+ mask_bytes = (h[:has_mask] != 0) ? h[:elements] : 0
322
+ if h[:mask_bytes] != mask_bytes
323
+ raise ArgumentError, "corrupt CArray binary data (mask_bytes cross-check failed)"
324
+ end
325
+ if @io.respond_to?(:size) && @io.respond_to?(:pos)
326
+ declared = h[:data_bytes] + mask_bytes + h[:trailer_bytes]
327
+ if @io.size - @io.pos < declared
328
+ raise ArgumentError, "corrupt CArray binary data (declares #{declared} bytes after " \
329
+ "the header, the source holds #{@io.size - @io.pos})"
330
+ end
331
+ end
332
+ end
333
+
258
334
  # Build the trailer mapping (String-keyed) from the array's
259
335
  # attributes and data_class. Returns {} when there is nothing to
260
336
  # serialise.
@@ -279,7 +355,8 @@ class CArray::Serializer # :nodoc:
279
355
  raise ArgumentError,
280
356
  "CArray.save cannot serialise data_class #{klass.inspect}: " \
281
357
  "member #{f.name.inspect} is a bitfield (the v1.0 Layer 1 " \
282
- "schema is flat-primitive only); use Marshal.dump(ca)"
358
+ "schema is flat-primitive only); Marshal.dump(ca) carries it " \
359
+ "when the struct is named (assigned to a constant)"
283
360
  end
284
361
  pep = SYMBOL_TO_PEP[f.type]
285
362
  unless pep
@@ -287,9 +364,12 @@ class CArray::Serializer # :nodoc:
287
364
  "CArray.save cannot serialise data_class #{klass.inspect}: " \
288
365
  "member #{f.name.inspect} has non-primitive type #{f.type.inspect} " \
289
366
  "(nested struct / template / fixlen are not expressible in the " \
290
- "v1.0 Layer 1 schema); use Marshal.dump(ca)"
367
+ "v1.0 Layer 1 schema); Marshal.dump(ca) carries it " \
368
+ "when the struct is named (assigned to a constant)"
291
369
  end
292
- { "name" => f.name, "type" => pep, "offset" => f.offset, "bytes" => f.bytes }
370
+ m = { "name" => f.name, "type" => pep, "offset" => f.offset, "bytes" => f.bytes }
371
+ m["endian"] = f.endian.to_s if f.endian
372
+ m
293
373
  end
294
374
 
295
375
  schema = {
@@ -320,6 +400,12 @@ class CArray::Serializer # :nodoc:
320
400
  # synthesise an anonymous struct from the Layer 1 members[], else
321
401
  # leave the raw CA_FIXLEN array untouched.
322
402
  def wrap_data_class (ca, schema)
403
+ members = schema.is_a?(Hash) ? schema["members"] : nil
404
+ unless schema.is_a?(Hash) &&
405
+ (members.nil? || members.is_a?(Array) && members.all?(Hash))
406
+ raise ArgumentError, "corrupt CArray binary data (data_class schema is not a " \
407
+ "mapping with a list of member mappings)"
408
+ end
323
409
  name = schema["name"]
324
410
  if name
325
411
  begin
@@ -352,9 +438,17 @@ class CArray::Serializer # :nodoc:
352
438
  members.each do |m|
353
439
  sym = PEP_TO_SYMBOL[m["type"]]
354
440
  unless sym
355
- raise "unknown data_class member type #{m["type"].inspect} in CArray binary data"
441
+ raise ArgumentError, "unknown data_class member type #{m["type"].inspect} in CArray binary data"
356
442
  end
357
- member sym, m["name"], :offset => m["offset"]
443
+ opt = { :offset => m["offset"] }
444
+ if m.key?("endian")
445
+ endian = %w[big little native preserve].find { |e| e == m["endian"] }
446
+ unless endian
447
+ raise ArgumentError, "unknown data_class member endian #{m["endian"].inspect} in CArray binary data"
448
+ end
449
+ opt[:endian] = endian.to_sym
450
+ end
451
+ member sym, m["name"], opt
358
452
  end
359
453
  end
360
454
  end
@@ -533,6 +627,11 @@ class CArray
533
627
  # @param data [Array] Marshal payload.
534
628
  # @return [void]
535
629
  def marshal_load (data)
630
+ initialize_copy(CArray.send(:array_from_marshal_payload, data))
631
+ end
632
+
633
+ # The array a {#marshal_dump} payload describes.
634
+ def self.array_from_marshal_payload (data)
536
635
  tag, *rest = data
537
636
  case tag
538
637
  when "object"
@@ -543,13 +642,84 @@ class CArray
543
642
  ca.mask = 0
544
643
  ca.mask[] = mask
545
644
  end
546
- initialize_copy(ca)
645
+ ca
547
646
  when "portable"
548
- ca = CArray.load(StringIO.new(rest[0]))
549
- initialize_copy(ca)
647
+ CArray.load(StringIO.new(rest[0]))
648
+ when "record"
649
+ data_class, storage = rest
650
+ CARecord.wrap(array_from_marshal_payload(storage), data_class)
550
651
  else
551
652
  raise TypeError, "unrecognised CArray Marshal payload"
552
653
  end
553
654
  end
655
+ private_class_method :array_from_marshal_payload
656
+
657
+ end
658
+
659
+ # Marshal of a view and a scalar: the {CArray#marshal_dump} payload, loaded
660
+ # back by the class method rather than into an empty instance. A view comes
661
+ # back as a plain array (a copy owns its memory, as #copy gives); a scalar
662
+ # comes back a scalar. A wrap is refused below.
663
+ [CAView, CAWrap, CScalar].each do |klass|
664
+ klass.class_eval do
665
+ # A Face other than a record is refused: the payload would carry its
666
+ # storage without what makes it the Face (a unit, labels), and load
667
+ # could not put that back.
668
+ # @return [String]
669
+ def _dump (_level)
670
+ if face?
671
+ raise TypeError,
672
+ "Marshal cannot carry a #{self.class} yet (its Face would be lost); " \
673
+ "dump its storage with .parent"
674
+ end
675
+ Marshal.dump(CArray.instance_method(:marshal_dump).bind_call(self))
676
+ end
677
+ end
678
+ end
679
+
680
+ class CAView
681
+ # @return [CArray] the plain array a {#_dump} string describes.
682
+ def self._load (str)
683
+ CArray.send(:array_from_marshal_payload, Marshal.load(str))
684
+ end
685
+ end
686
+
687
+ class CAWrap
688
+ # A wrap borrows memory, which Marshal cannot carry to another process;
689
+ # Ruby would also try to write the objects that keep the borrowed memory
690
+ # alive. Its copy owns its memory and marshals.
691
+ def _dump (_level)
692
+ raise TypeError,
693
+ "Marshal cannot carry a #{self.class}: it borrows its memory; " \
694
+ "dump .copy instead"
695
+ end
696
+
697
+ def self._load (str)
698
+ CArray.send(:array_from_marshal_payload, Marshal.load(str))
699
+ end
700
+ end
701
+
702
+ class CScalar
703
+ # @return [CScalar] the scalar a {#_dump} string describes.
704
+ def self._load (str)
705
+ a = CArray.send(:array_from_marshal_payload, Marshal.load(str))
706
+ s = CScalar.new(a.data_type, bytes: a.bytes)
707
+ s[0] = a[0]
708
+ s
709
+ end
710
+ end
711
+
712
+ class CARecord
713
+
714
+ # A record array keeps marshal_dump: it round-trips with its class. A
715
+ # named struct is written by name with the record bytes, so every kind of
716
+ # member (nested, fixlen, bit, array) travels. An anonymous struct has no
717
+ # name to write: it goes through the binary format, whose schema
718
+ # describes flat numeric members only.
719
+ def marshal_dump
720
+ return CArray.instance_method(:marshal_dump).bind_call(self) unless data_class.name
721
+ ["record", data_class,
722
+ CArray.instance_method(:marshal_dump).bind_call(copy.parent)]
723
+ end
554
724
 
555
725
  end
@@ -152,12 +152,24 @@ class CASlabIterator < CAIterator
152
152
  # @!method max_addr
153
153
  # Per-slab flat source address of the maximum.
154
154
  # @return [CArray] one flat address per slab
155
- [:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev, :all, :any,
155
+ #
156
+ # Each takes `min_count:` and `fill_value:` and passes them to the core
157
+ # reduction, as the other iterators do; `all` and `any` take neither, as
158
+ # the core ones do not.
159
+ [:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev,
156
160
  :variancep, :stddevp, :minmax, :min_index, :max_index,
157
161
  :min_addr, :max_addr].each do |op|
158
162
  class_eval <<~RUBY, __FILE__, __LINE__ + 1
159
- def #{op}
160
- @reference.#{op}(axis: @slab_axes)
163
+ def #{op} (min_count: nil, fill_value: nil)
164
+ @reference.#{op}(axis: @slab_axes, **reduce_options(min_count, fill_value))
165
+ end
166
+ RUBY
167
+ end
168
+
169
+ [:all, :any].each do |op|
170
+ class_eval <<~RUBY, __FILE__, __LINE__ + 1
171
+ def #{op} (**kw)
172
+ @reference.#{op}(axis: @slab_axes, **CAIterator.reduce_keywords(:#{op}, kw))
161
173
  end
162
174
  RUBY
163
175
  end
@@ -167,25 +179,28 @@ class CASlabIterator < CAIterator
167
179
  # No argument counts present (non-masked) cells; `count(UNDEF)` counts
168
180
  # masked cells; `count(v)` counts cells equal to `v`.
169
181
  # @return [CArray] one count per slab
170
- def count (*args)
171
- return count_not_masked if args.empty?
182
+ def count (*args, **kw)
183
+ return count_not_masked(**kw) if args.empty?
172
184
  # CABlock / CAWindow shadow #count with a geometry accessor, and @reference
173
185
  # may be such a view, so dispatch CArray#count explicitly for count(v).
174
- CArray.instance_method(:count).bind_call(@reference, *args, axis: @slab_axes)
186
+ CArray.instance_method(:count).bind_call(@reference, *args, axis: @slab_axes,
187
+ **CAIterator.reduce_keywords(:count, kw))
175
188
  end
176
189
 
177
190
  # @overload count_not_masked
178
191
  # Per-slab count of present (non-masked) cells.
179
192
  # @return [CArray]
180
- def count_not_masked
181
- @reference.count_not_masked(axis: @slab_axes)
193
+ def count_not_masked (**kw)
194
+ @reference.count_not_masked(axis: @slab_axes,
195
+ **CAIterator.reduce_keywords(:count_not_masked, kw))
182
196
  end
183
197
 
184
198
  # @overload count_masked
185
199
  # Per-slab count of masked cells.
186
200
  # @return [CArray]
187
- def count_masked
188
- @reference.count_masked(axis: @slab_axes)
201
+ def count_masked (**kw)
202
+ @reference.count_masked(axis: @slab_axes,
203
+ **CAIterator.reduce_keywords(:count_masked, kw))
189
204
  end
190
205
 
191
206
  # @overload elements
@@ -206,30 +221,31 @@ class CASlabIterator < CAIterator
206
221
  #
207
222
  # Core order statistics take a single integer axis, so a single slab axis is
208
223
  # delegated directly (fast C path) and a multi-axis slab is folded per slab
209
- # with reduce_slab (each slab flattened). As with `CArray#median(axis:)`, a
210
- # masked source raises (the known per-axis masked limitation); strip the mask
211
- # with `ca.value` first if needed.
224
+ # with reduce_slab (each slab flattened). Masked cells are left out, as in
225
+ # `CArray#median(axis:)`.
212
226
 
213
- # @overload median
227
+ # @overload median(min_count: nil, fill_value: nil)
214
228
  # Per-slab median.
215
229
  # @return [CArray]
216
- def median
230
+ def median (min_count: nil, fill_value: nil)
231
+ opts = reduce_options(min_count, fill_value)
217
232
  if @slab_axes.size == 1
218
- @reference.median(axis: @slab_axes[0])
233
+ @reference.median(axis: @slab_axes[0], **opts)
219
234
  else
220
- @reference.reduce_slab(axis: @slab_axes) { |s| s.median }
235
+ @reference.reduce_slab(axis: @slab_axes) { |s| s.median(**opts) }
221
236
  end
222
237
  end
223
238
 
224
- # @overload percentile(*pers)
239
+ # @overload percentile(*pers, min_count: nil, fill_value: nil)
225
240
  # Per-slab percentile(s). One argument returns one CArray; several return an
226
241
  # array of CArrays (as `CArray#percentile`).
227
242
  # @return [CArray, Array<CArray>]
228
- def percentile (*pers)
243
+ def percentile (*pers, min_count: nil, fill_value: nil)
244
+ opts = reduce_options(min_count, fill_value)
229
245
  if @slab_axes.size == 1
230
- @reference.percentile(*pers, axis: @slab_axes[0])
246
+ @reference.percentile(*pers, axis: @slab_axes[0], **opts)
231
247
  else
232
- rs = pers.map { |p| @reference.reduce_slab(axis: @slab_axes) { |s| s.percentile(p) } }
248
+ rs = pers.map { |p| @reference.reduce_slab(axis: @slab_axes) { |s| s.percentile(p, **opts) } }
233
249
  pers.size == 1 ? rs[0] : rs
234
250
  end
235
251
  end
@@ -279,18 +295,18 @@ class CASlabIterator < CAIterator
279
295
  # the same shape as the reference (one weight per cell). Core wsum / wmean
280
296
  # accept multi-axis, so this covers multi-axis slabs directly.
281
297
 
282
- # @overload wsum(weights)
298
+ # @overload wsum(weights, min_count: nil, fill_value: nil)
283
299
  # Per-slab weighted sum, `weights` shaped like the reference.
284
300
  # @return [CArray]
285
- def wsum (weights)
286
- @reference.wsum(weights, axis: @slab_axes)
301
+ def wsum (weights, min_count: nil, fill_value: nil)
302
+ @reference.wsum(weights, axis: @slab_axes, **reduce_options(min_count, fill_value))
287
303
  end
288
304
 
289
- # @overload wmean(weights)
305
+ # @overload wmean(weights, min_count: nil, fill_value: nil)
290
306
  # Per-slab weighted mean, `weights` shaped like the reference.
291
307
  # @return [CArray]
292
- def wmean (weights)
293
- @reference.wmean(weights, axis: @slab_axes)
308
+ def wmean (weights, min_count: nil, fill_value: nil)
309
+ @reference.wmean(weights, axis: @slab_axes, **reduce_options(min_count, fill_value))
294
310
  end
295
311
 
296
312
  # ---- segment scan: within-slab running statistics ---------------------
@@ -332,6 +348,14 @@ class CASlabIterator < CAIterator
332
348
 
333
349
  # The single slab axis for the order surface (core sort takes one integer
334
350
  # axis). A multi-axis slab has no single sort axis, so raise.
351
+ # The reduction keywords that were given, for the core call.
352
+ def reduce_options (min_count, fill_value)
353
+ opts = {}
354
+ opts[:min_count] = min_count unless min_count.nil?
355
+ opts[:fill_value] = fill_value unless fill_value.nil?
356
+ opts
357
+ end
358
+
335
359
  def single_sort_axis (op)
336
360
  return @slab_axes[0] if @slab_axes.size == 1
337
361
  raise ArgumentError,
data/lib/carray/stack.rb CHANGED
@@ -51,13 +51,15 @@ class CArray
51
51
  # `data_type` and re-wraps homogeneous Face inputs via
52
52
  # `face_lift`. Output `ndim` is one greater than each piece.
53
53
  # @param list [Array<CArray>] pieces to stack; must not be empty.
54
- # @param axis [Integer] position of the new K axis.
54
+ # @param axis [Integer, nil] position of the new K axis; `nil` is
55
+ # the same as `0`.
55
56
  # @param data_type [Symbol, Integer, nil] result `data_type`;
56
57
  # inferred when `nil`.
57
58
  # @return [CArray] CAStack view.
58
59
  # @raise [ArgumentError] when `list` is empty.
59
60
  def self.stack (list, axis: 0, data_type: nil)
60
61
  raise ArgumentError, "stack: list must not be empty" if list.empty?
62
+ axis = 0 if axis.nil?
61
63
  list = CArray.promote_list(list, data_type: data_type)
62
64
  axis = CArray.normalize_axis(axis, list[0].ndim + 1, "stack")
63
65
  CAStack.new(list, axis: axis) # CAStack.new does Face lift internally
@@ -80,8 +82,9 @@ class CArray
80
82
  # Convenience form: a single Array argument is treated as the list.
81
83
  # @param arrays [Array<CArray>] pieces to weld. A single Array
82
84
  # argument is accepted for compatibility with older callers.
83
- # @param axis [Integer] existing axis to extend (normalises negative
84
- # values against the reference ndim).
85
+ # @param axis [Integer, nil] existing axis to extend (normalises
86
+ # negative values against the reference ndim); `nil` is the same
87
+ # as `0`.
85
88
  # @return [CAMeld] view over the welded pieces.
86
89
  # @raise [ArgumentError] when the list is empty, ndim mismatch,
87
90
  # data_type mismatch, or non-axis dim mismatch across pieces
@@ -91,6 +94,7 @@ class CArray
91
94
  arrays = arrays[0]
92
95
  end
93
96
  raise ArgumentError, "meld: list must not be empty" if arrays.empty?
97
+ axis = 0 if axis.nil?
94
98
  first = arrays[0]
95
99
  unless first.is_a?(CArray)
96
100
  raise ArgumentError, "meld: entries must be CArray (got #{first.class})"
@@ -247,8 +251,10 @@ class CArray
247
251
  # entities.
248
252
  # @param axis [Integer] axis to split along.
249
253
  # @return [Array<CArray>] one slice per index along `axis`.
250
- # @raise [ArgumentError] when `axis` is not a single Integer.
254
+ # @raise [ArgumentError] when `axis` is not a single Integer
255
+ # (including `nil`).
251
256
  def split (axis:)
257
+ raise ArgumentError, "split: axis: is required" if axis.nil?
252
258
  if axis.is_a?(Array)
253
259
  raise ArgumentError, "split: axis must be a single Integer"
254
260
  end
@@ -153,13 +153,14 @@ class CArray
153
153
 
154
154
  # @overload extract(regexp, replace = '\0')
155
155
  # Returns a CArray whose cells are each cell's first match of `regexp`
156
- # transformed by `String#sub(regexp, replace)`; non-matching cells
157
- # become `""`.
156
+ # transformed by `String#sub(regexp, replace)`. A cell that does not
157
+ # match is masked (`UNDEF`), so it stays apart from a match of the empty
158
+ # string, and a later `cast` to a number does not read it as 0.
158
159
  # @param regexp [Regexp]
159
160
  # @param replace [String]
160
161
  # @return [CAString]
161
162
  def extract (regexp, replace = '\0')
162
- string_map { |s| regexp.match(s) { |m| m[0].sub(regexp, replace) } || "" }
163
+ string_map { |s| regexp.match(s) { |m| m[0].sub(regexp, replace) } || UNDEF }
163
164
  end
164
165
 
165
166
  # --- inter-Face conversions (String Faces only) ----------------------