carray 3.0.2 → 3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +304 -0
- data/README.md +3 -4
- data/carray.gemspec +1 -1
- data/ext/ca_axis_dispatch.c +225 -0
- data/ext/ca_axis_group.c +177 -99
- data/ext/ca_binop_dispatch.c +54 -32
- data/ext/ca_binop_dispatch.h +1 -0
- data/ext/ca_categorical_iterator.c +66 -71
- data/ext/ca_compare.c +8 -2
- data/ext/ca_for_buffer.h +21 -68
- data/ext/ca_group_iter.c +27 -11
- data/ext/ca_kernel_iterator.c +663 -214
- data/ext/ca_kernel_iterator.h +231 -167
- data/ext/ca_monop_dispatch.c +4 -4
- data/ext/ca_monop_dispatch.h +4 -0
- data/ext/ca_obj_array.c +132 -22
- data/ext/ca_obj_bincmp.c +152 -121
- data/ext/ca_obj_binop.c +107 -210
- data/ext/ca_obj_bitarray.c +37 -24
- data/ext/ca_obj_bitfield.c +95 -62
- data/ext/ca_obj_block.c +1 -0
- data/ext/ca_obj_byte_swap.c +40 -40
- data/ext/ca_obj_const_string.c +96 -25
- data/ext/ca_obj_face.c +406 -64
- data/ext/ca_obj_face.h +92 -47
- data/ext/ca_obj_fake.c +21 -50
- data/ext/ca_obj_farray.c +1 -0
- data/ext/ca_obj_field.c +16 -3
- data/ext/ca_obj_fixlen_string.c +21 -21
- data/ext/ca_obj_grid.c +181 -124
- data/ext/ca_obj_meld.c +189 -107
- data/ext/ca_obj_moncmp.c +61 -61
- data/ext/ca_obj_monop.c +125 -107
- data/ext/ca_obj_object.c +543 -233
- data/ext/ca_obj_record.c +51 -16
- data/ext/ca_obj_reduce.c +5 -8
- data/ext/ca_obj_refer.c +117 -33
- data/ext/ca_obj_remap.c +57 -17
- data/ext/ca_obj_repeat.c +21 -5
- data/ext/ca_obj_roll.c +115 -11
- data/ext/ca_obj_select.c +47 -49
- data/ext/ca_obj_select_axis.c +58 -60
- data/ext/ca_obj_shift.c +22 -102
- data/ext/ca_obj_stack.c +223 -118
- data/ext/ca_obj_stride.c +195 -34
- data/ext/ca_obj_string.c +17 -18
- data/ext/ca_obj_tile.c +32 -16
- data/ext/ca_obj_time.c +23 -25
- data/ext/ca_obj_timedelta.c +21 -23
- data/ext/ca_obj_transpose.c +12 -3
- data/ext/ca_obj_triop.c +110 -117
- data/ext/ca_obj_window.c +149 -27
- data/ext/ca_op_ipower.c +24 -11
- data/ext/ca_op_powi.h +57 -30
- data/ext/ca_pow5_table.h +656 -0
- data/ext/ca_sort_kernels.h +70 -0
- data/ext/ca_sweep_engine.c +102 -79
- data/ext/ca_sweep_engine.h +7 -20
- data/ext/ca_triop_dispatch.c +4 -1
- data/ext/ca_triop_dispatch.h +5 -2
- data/ext/caframe_csv_reader.c +463 -0
- data/ext/carray.h +70 -27
- data/ext/carray_access.c +455 -211
- data/ext/carray_address_basis.c +177 -36
- data/ext/carray_attribute.c +45 -3
- data/ext/carray_bincount.c +32 -2
- data/ext/carray_broadcast.c +46 -14
- data/ext/carray_call_cfunc.c +0 -28
- data/ext/carray_cast.c +1444 -209
- data/ext/carray_cast_func.rb +34 -21
- data/ext/carray_conversion.c +197 -73
- data/ext/carray_copy.c +32 -11
- data/ext/carray_core.c +514 -122
- data/ext/carray_count.c +140 -87
- data/ext/carray_element.c +18 -27
- data/ext/carray_factorize.c +569 -277
- data/ext/carray_generate.c +76 -40
- data/ext/carray_histogram.c +152 -94
- data/ext/carray_hold.c +2 -1
- data/ext/carray_index_classifier.c +63 -96
- data/ext/carray_internal.h +385 -3
- data/ext/carray_lazy.c +834 -79
- data/ext/carray_loop.c +81 -65
- data/ext/carray_mask.c +682 -316
- data/ext/carray_math_kernel.h +1 -0
- data/ext/carray_median_percentile.c +221 -76
- data/ext/carray_memory_view.c +254 -248
- data/ext/carray_operator.c +1089 -359
- data/ext/carray_order.c +133 -69
- data/ext/carray_partition.c +102 -56
- data/ext/carray_random.c +273 -239
- data/ext/carray_scatter.c +308 -65
- data/ext/carray_segment.c +257 -0
- data/ext/carray_slab.c +170 -70
- data/ext/carray_sort.c +215 -218
- data/ext/carray_sort_kernel.c +8 -1
- data/ext/carray_struct.c +99 -21
- data/ext/carray_test.c +167 -64
- data/ext/carray_time_text.c +1182 -0
- data/ext/carray_utils.c +363 -98
- data/ext/extconf.rb +27 -2
- data/ext/mk_call_cfunc.rb +0 -2
- data/ext/mk_pow5_table.rb +35 -0
- data/ext/mkkernel.rb +1076 -846
- data/ext/ruby_carray.c +7 -1
- data/ext/version.h +4 -4
- data/ext/version.rb +4 -2
- data/lib/carray/attribute.rb +39 -0
- data/lib/carray/autoload_carray.rb +19 -0
- data/lib/carray/autoload_method_extension.rb +6 -0
- data/lib/carray/axis_group.rb +174 -26
- data/lib/carray/basics.rb +91 -25
- data/lib/carray/bincount_nd.rb +6 -0
- data/lib/carray/block_iterator.rb +118 -48
- data/lib/carray/boolean_reduce.rb +21 -29
- data/lib/carray/categorical.rb +94 -26
- data/lib/carray/categorical_iterator.rb +333 -826
- data/lib/carray/complex.rb +36 -29
- data/lib/carray/conditional.rb +94 -7
- data/lib/carray/const_string.rb +46 -0
- data/lib/carray/frame/concat.rb +157 -6
- data/lib/carray/frame/convert.rb +13 -7
- data/lib/carray/frame/csv_parser.rb +589 -82
- data/lib/carray/frame/describe.rb +79 -0
- data/lib/carray/frame/frame.rb +124 -25
- data/lib/carray/frame/group.rb +189 -63
- data/lib/carray/frame/io.rb +412 -72
- data/lib/carray/frame/join.rb +134 -46
- data/lib/carray/frame/records.rb +37 -16
- data/lib/carray/frame/reshape.rb +448 -0
- data/lib/carray/frame/sort.rb +26 -2
- data/lib/carray/frame/verbs.rb +715 -48
- data/lib/carray/frame.rb +2 -0
- data/lib/carray/fuse_source.rb +111 -2
- data/lib/carray/fusion.rb +229 -38
- data/lib/carray/inspect.rb +29 -11
- data/lib/carray/iterator.rb +14 -2
- data/lib/carray/lazy.rb +54 -30
- data/lib/carray/mask_gap_fill.rb +66 -52
- data/lib/carray/math.rb +32 -19
- data/lib/carray/meld_reduce.rb +40 -7
- data/lib/carray/methods/align_addr.rb +8 -4
- data/lib/carray/methods/bincount.rb +2 -2
- data/lib/carray/methods/bit_string.rb +14 -3
- data/lib/carray/methods/composition.rb +16 -10
- data/lib/carray/methods/factorize.rb +2 -2
- data/lib/carray/methods/gather_nd.rb +13 -2
- data/lib/carray/methods/insert_block.rb +9 -14
- data/lib/carray/methods/is_in.rb +117 -4
- data/lib/carray/methods/join.rb +1 -1
- data/lib/carray/methods/locate_addr.rb +177 -36
- data/lib/carray/methods/mode.rb +18 -1
- data/lib/carray/methods/repeat.rb +1 -20
- data/lib/carray/methods/resize.rb +21 -23
- data/lib/carray/methods/snap.rb +34 -12
- data/lib/carray/methods/unique.rb +11 -2
- data/lib/carray/record_order.rb +208 -0
- data/lib/carray/runtime.rb +14 -11
- data/lib/carray/segment_iterator.rb +990 -0
- data/lib/carray/serialize.rb +197 -27
- data/lib/carray/slab_iterator.rb +51 -27
- data/lib/carray/stack.rb +10 -4
- data/lib/carray/string_operation_extension.rb +4 -3
- data/lib/carray/struct.rb +142 -51
- data/lib/carray/struct_builder.rb +110 -14
- data/lib/carray/table.rb +2 -2
- data/lib/carray/time.rb +517 -103
- data/lib/carray/window_iterator.rb +127 -72
- data/lib/carray.rb +2 -2
- data/yard-stubs/ca_obj_array.rb +4 -4
- data/yard-stubs/ca_obj_bitarray.rb +7 -0
- data/yard-stubs/ca_obj_bitfield.rb +23 -12
- data/yard-stubs/ca_obj_fake.rb +1 -1
- data/yard-stubs/ca_obj_field.rb +1 -1
- data/yard-stubs/ca_obj_refer.rb +3 -3
- data/yard-stubs/ca_obj_roll.rb +0 -1
- data/yard-stubs/ca_obj_stride.rb +10 -7
- data/yard-stubs/ca_obj_transpose.rb +1 -1
- data/yard-stubs/ca_obj_window.rb +1 -2
- data/yard-stubs/carray_access.rb +5 -4
- data/yard-stubs/carray_broadcast.rb +1 -1
- data/yard-stubs/carray_cast.rb +78 -43
- data/yard-stubs/carray_conversion.rb +13 -1
- data/yard-stubs/carray_copy.rb +8 -3
- data/yard-stubs/carray_core.rb +3 -3
- data/yard-stubs/carray_count.rb +9 -0
- data/yard-stubs/carray_lazy.rb +5 -1
- data/yard-stubs/carray_loop.rb +4 -1
- data/yard-stubs/carray_mask.rb +34 -10
- data/yard-stubs/carray_math.rb +25 -13
- data/yard-stubs/carray_median_percentile.rb +10 -6
- data/yard-stubs/carray_order.rb +34 -13
- data/yard-stubs/carray_scatter.rb +10 -3
- data/yard-stubs/carray_segment.rb +72 -0
- data/yard-stubs/carray_slab.rb +2 -1
- data/yard-stubs/carray_sort.rb +43 -3
- data/yard-stubs/carray_test.rb +10 -0
- data/yard-stubs/carray_utils.rb +0 -26
- metadata +11 -19
- data/ext/ca_for_each_element.h +0 -239
- data/ext/carray_build_flags.h +0 -3
- data/ext/carray_config.h +0 -28
- data/ext/carray_kernels_bincmp.c +0 -4446
- data/ext/carray_kernels_binop.c +0 -11001
- data/ext/carray_kernels_init.c +0 -1131
- data/ext/carray_kernels_map.c +0 -3467
- data/ext/carray_kernels_moncmp.c +0 -2097
- data/ext/carray_kernels_monop.c +0 -18313
- data/ext/carray_kernels_reduce_aggregate.c +0 -26005
- data/ext/carray_kernels_reduce_boolean.c +0 -330
- data/ext/carray_kernels_reduce_cumulative.c +0 -14862
- data/ext/carray_kernels_reduce_extreme.c +0 -17494
- data/ext/carray_kernels_reduce_variance.c +0 -3910
- data/ext/carray_kernels_scan.c +0 -3693
- data/ext/carray_kernels_search.c +0 -32218
- data/ext/carray_kernels_sort.c +0 -10626
- data/ext/carray_kernels_triop.c +0 -1392
data/lib/carray/serialize.rb
CHANGED
|
@@ -112,21 +112,31 @@ class CArray::Serializer # :nodoc:
|
|
|
112
112
|
# The data region is raw contiguous bytes; attribute Hash and
|
|
113
113
|
# data_class schema (if any) ride in a YAML trailer at the tail.
|
|
114
114
|
# @param ca [CArray] array to write.
|
|
115
|
-
# @param
|
|
115
|
+
# @param endian [Symbol, Integer, nil] forces the output byte order
|
|
116
116
|
# (defaults to the host's; a differing value swaps data + header).
|
|
117
117
|
# @return [CArray] `ca`.
|
|
118
118
|
# @raise [ArgumentError] when `ca` is a CA_OBJECT array (use
|
|
119
119
|
# `Marshal.dump(ca)` instead), or carries a data_class the
|
|
120
120
|
# v1.0 flat-primitive schema cannot express.
|
|
121
|
-
|
|
121
|
+
# @raise [TypeError] when `ca` is a Face other than a record (save
|
|
122
|
+
# its storage with `.parent`).
|
|
123
|
+
def save (ca, endian: nil)
|
|
122
124
|
if ca.data_type == :object
|
|
123
125
|
raise ArgumentError,
|
|
124
126
|
"CArray.save cannot serialise a CA_OBJECT array " \
|
|
125
127
|
"(arbitrary Ruby objects have no portable representation); " \
|
|
126
128
|
"use Marshal.dump(ca) for a Ruby-only round-trip"
|
|
127
129
|
end
|
|
130
|
+
# A Face other than a record is refused, as Marshal refuses it: the file
|
|
131
|
+
# would carry its storage without what makes it the Face (a unit, the
|
|
132
|
+
# string buffer, the labels), and load could not put that back.
|
|
133
|
+
if ca.face? && !ca.is_a?(CARecord)
|
|
134
|
+
raise TypeError,
|
|
135
|
+
"CArray.save cannot carry a #{ca.class} yet (its Face would be " \
|
|
136
|
+
"lost); save its storage with .parent"
|
|
137
|
+
end
|
|
128
138
|
|
|
129
|
-
file_endian =
|
|
139
|
+
file_endian = endian || CArray.endian
|
|
130
140
|
swap = (file_endian != CArray.endian)
|
|
131
141
|
|
|
132
142
|
elements = ca.elements
|
|
@@ -170,7 +180,7 @@ class CArray::Serializer # :nodoc:
|
|
|
170
180
|
|
|
171
181
|
@io.write(self.class.pack_header(header, file_endian))
|
|
172
182
|
|
|
173
|
-
data_ca = swap ? ca
|
|
183
|
+
data_ca = swap ? to_other_byte_order(ca) : ca
|
|
174
184
|
data_ca.dump_binary(@io)
|
|
175
185
|
|
|
176
186
|
if has_mask
|
|
@@ -186,18 +196,19 @@ class CArray::Serializer # :nodoc:
|
|
|
186
196
|
# Reads a _CARRAY3 payload from the wrapped IO and returns the
|
|
187
197
|
# reconstructed array, re-wrapping it through its recorded
|
|
188
198
|
# data_class when the trailer carries one.
|
|
189
|
-
# @param
|
|
199
|
+
# @param data_type [Symbol, Integer, nil] overrides the element type for a
|
|
190
200
|
# bare `CA_FIXLEN` payload.
|
|
191
201
|
# @return [CArray]
|
|
192
|
-
# @raise [
|
|
202
|
+
# @raise [ArgumentError] on a bad magic string, corrupt header, or
|
|
193
203
|
# unsupported version.
|
|
194
|
-
def load (
|
|
204
|
+
def load (data_type: nil)
|
|
205
|
+
fixlen_type = data_type
|
|
195
206
|
buf = @io.read(HEADER_BYTES)
|
|
196
207
|
unless buf && buf.bytesize == HEADER_BYTES
|
|
197
|
-
raise "not a CArray binary data (truncated header)"
|
|
208
|
+
raise ArgumentError, "not a CArray binary data (truncated header)"
|
|
198
209
|
end
|
|
199
210
|
if buf[0, 8] != MAGIC
|
|
200
|
-
raise "not a CArray binary data (bad magic)"
|
|
211
|
+
raise ArgumentError, "not a CArray binary data (bad magic)"
|
|
201
212
|
end
|
|
202
213
|
|
|
203
214
|
# The byte order is self-describing via endian_marker at offset 8:
|
|
@@ -208,22 +219,23 @@ class CArray::Serializer # :nodoc:
|
|
|
208
219
|
when 0x01 then file_endian = CA_BIG_ENDIAN
|
|
209
220
|
when 0x04 then file_endian = CA_LITTLE_ENDIAN
|
|
210
221
|
else
|
|
211
|
-
raise "corrupt CArray binary data (endian marker mismatch)"
|
|
222
|
+
raise ArgumentError, "corrupt CArray binary data (endian marker mismatch)"
|
|
212
223
|
end
|
|
213
224
|
h = self.class.unpack_header(buf, file_endian)
|
|
214
225
|
|
|
215
226
|
if h[:endian_marker] != ENDIAN_MARKER
|
|
216
|
-
raise "corrupt CArray binary data (endian marker mismatch)"
|
|
227
|
+
raise ArgumentError, "corrupt CArray binary data (endian marker mismatch)"
|
|
217
228
|
end
|
|
218
229
|
if h[:version_major] != VERSION_MAJOR
|
|
219
|
-
raise "unsupported CArray binary version #{h[:version_major]}.#{h[:version_minor]}"
|
|
230
|
+
raise ArgumentError, "unsupported CArray binary version #{h[:version_major]}.#{h[:version_minor]}"
|
|
220
231
|
end
|
|
221
232
|
if h[:header_bytes] != HEADER_BYTES
|
|
222
|
-
raise "unsupported CArray binary header size #{h[:header_bytes]}"
|
|
233
|
+
raise ArgumentError, "unsupported CArray binary header size #{h[:header_bytes]}"
|
|
223
234
|
end
|
|
224
235
|
if h[:data_bytes] != h[:elements] * h[:element_bytes]
|
|
225
|
-
raise "corrupt CArray binary data (data_bytes cross-check failed)"
|
|
236
|
+
raise ArgumentError, "corrupt CArray binary data (data_bytes cross-check failed)"
|
|
226
237
|
end
|
|
238
|
+
check_header(h)
|
|
227
239
|
|
|
228
240
|
swap = (file_endian != CArray.endian)
|
|
229
241
|
data_type = h[:data_type_code]
|
|
@@ -231,30 +243,94 @@ class CArray::Serializer # :nodoc:
|
|
|
231
243
|
dim = h[:shape][0, ndim]
|
|
232
244
|
bytes = h[:element_bytes]
|
|
233
245
|
|
|
234
|
-
if
|
|
235
|
-
data_type =
|
|
246
|
+
if fixlen_type and data_type == CArray.data_type_code(CA_FIXLEN)
|
|
247
|
+
data_type = fixlen_type
|
|
236
248
|
end
|
|
237
249
|
|
|
238
250
|
ca = CArray.new(data_type, dim, :bytes => bytes)
|
|
239
251
|
ca.load_binary(@io)
|
|
240
|
-
ca[] = ca.swap_bytes if swap
|
|
241
252
|
|
|
253
|
+
mask = nil
|
|
242
254
|
if h[:has_mask] != 0
|
|
243
|
-
|
|
244
|
-
|
|
255
|
+
mask = CArray.boolean(*dim)
|
|
256
|
+
mask.load_binary(@io)
|
|
245
257
|
end
|
|
246
258
|
|
|
247
259
|
if h[:trailer_bytes] > 0
|
|
248
260
|
trailer_raw = @io.read(h[:trailer_bytes])
|
|
261
|
+
unless trailer_raw && trailer_raw.bytesize == h[:trailer_bytes]
|
|
262
|
+
raise ArgumentError, "corrupt CArray binary data (truncated trailer)"
|
|
263
|
+
end
|
|
249
264
|
trailer = decode_trailer(trailer_raw)
|
|
265
|
+
unless trailer.is_a?(Hash)
|
|
266
|
+
raise ArgumentError, "corrupt CArray binary data (trailer is not a mapping)"
|
|
267
|
+
end
|
|
250
268
|
ca = apply_trailer(ca, trailer)
|
|
251
269
|
end
|
|
252
270
|
|
|
271
|
+
# Swap once the trailer is applied and before the mask is: a record is
|
|
272
|
+
# swapped member by member, as the writer swapped it, and only its
|
|
273
|
+
# data_class says where the members are. Swapping the bare fixlen
|
|
274
|
+
# first reversed the whole record.
|
|
275
|
+
ca[] = to_other_byte_order(ca) if swap
|
|
276
|
+
|
|
277
|
+
if mask
|
|
278
|
+
ca.mask = 0
|
|
279
|
+
ca.mask[] = mask
|
|
280
|
+
end
|
|
281
|
+
|
|
253
282
|
return ca
|
|
254
283
|
end
|
|
255
284
|
|
|
256
285
|
private
|
|
257
286
|
|
|
287
|
+
# `ca` in the other byte order. A member declared with `endian:` holds its
|
|
288
|
+
# bytes in that order on every machine and in every file, so it is left as
|
|
289
|
+
# it is; only the members in machine order are swapped.
|
|
290
|
+
def to_other_byte_order (ca)
|
|
291
|
+
swapped = ca.swap_bytes
|
|
292
|
+
declared = ca.respond_to?(:data_class) && ca.data_class &&
|
|
293
|
+
ca.data_class.fields.select(&:endian)
|
|
294
|
+
return swapped if !declared || declared.empty?
|
|
295
|
+
swapped = swapped.copy
|
|
296
|
+
declared.each { |f| swapped[f.name][] = ca[f.name].value }
|
|
297
|
+
swapped
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# The header fields that describe the same array have to agree, and a
|
|
301
|
+
# source whose size is known has to hold what the header declares --
|
|
302
|
+
# checked before the array is allocated, so a short file cannot make
|
|
303
|
+
# the reader allocate what it declares.
|
|
304
|
+
def check_header (h)
|
|
305
|
+
ndim = h[:ndim]
|
|
306
|
+
unless ndim >= 1 && ndim <= CA_RANK_MAX
|
|
307
|
+
raise ArgumentError, "corrupt CArray binary data (ndim #{ndim} out of 1..#{CA_RANK_MAX})"
|
|
308
|
+
end
|
|
309
|
+
dim = h[:shape][0, ndim]
|
|
310
|
+
if dim.any?(&:negative?) || dim.inject(:*) != h[:elements]
|
|
311
|
+
raise ArgumentError, "corrupt CArray binary data (shape #{dim.inspect} does not " \
|
|
312
|
+
"hold #{h[:elements]} elements)"
|
|
313
|
+
end
|
|
314
|
+
fixlen = CArray.data_type_code(CA_FIXLEN)
|
|
315
|
+
if h[:data_type_code] != fixlen &&
|
|
316
|
+
h[:data_type_code] <= CArray.data_type_code(CA_OBJECT) &&
|
|
317
|
+
h[:element_bytes] != CArray.sizeof(h[:data_type_code])
|
|
318
|
+
raise ArgumentError, "corrupt CArray binary data (element_bytes #{h[:element_bytes]} " \
|
|
319
|
+
"for data type #{CArray.data_type_name(h[:data_type_code]) rescue h[:data_type_code]})"
|
|
320
|
+
end
|
|
321
|
+
mask_bytes = (h[:has_mask] != 0) ? h[:elements] : 0
|
|
322
|
+
if h[:mask_bytes] != mask_bytes
|
|
323
|
+
raise ArgumentError, "corrupt CArray binary data (mask_bytes cross-check failed)"
|
|
324
|
+
end
|
|
325
|
+
if @io.respond_to?(:size) && @io.respond_to?(:pos)
|
|
326
|
+
declared = h[:data_bytes] + mask_bytes + h[:trailer_bytes]
|
|
327
|
+
if @io.size - @io.pos < declared
|
|
328
|
+
raise ArgumentError, "corrupt CArray binary data (declares #{declared} bytes after " \
|
|
329
|
+
"the header, the source holds #{@io.size - @io.pos})"
|
|
330
|
+
end
|
|
331
|
+
end
|
|
332
|
+
end
|
|
333
|
+
|
|
258
334
|
# Build the trailer mapping (String-keyed) from the array's
|
|
259
335
|
# attributes and data_class. Returns {} when there is nothing to
|
|
260
336
|
# serialise.
|
|
@@ -279,7 +355,8 @@ class CArray::Serializer # :nodoc:
|
|
|
279
355
|
raise ArgumentError,
|
|
280
356
|
"CArray.save cannot serialise data_class #{klass.inspect}: " \
|
|
281
357
|
"member #{f.name.inspect} is a bitfield (the v1.0 Layer 1 " \
|
|
282
|
-
"schema is flat-primitive only);
|
|
358
|
+
"schema is flat-primitive only); Marshal.dump(ca) carries it " \
|
|
359
|
+
"when the struct is named (assigned to a constant)"
|
|
283
360
|
end
|
|
284
361
|
pep = SYMBOL_TO_PEP[f.type]
|
|
285
362
|
unless pep
|
|
@@ -287,9 +364,12 @@ class CArray::Serializer # :nodoc:
|
|
|
287
364
|
"CArray.save cannot serialise data_class #{klass.inspect}: " \
|
|
288
365
|
"member #{f.name.inspect} has non-primitive type #{f.type.inspect} " \
|
|
289
366
|
"(nested struct / template / fixlen are not expressible in the " \
|
|
290
|
-
"v1.0 Layer 1 schema);
|
|
367
|
+
"v1.0 Layer 1 schema); Marshal.dump(ca) carries it " \
|
|
368
|
+
"when the struct is named (assigned to a constant)"
|
|
291
369
|
end
|
|
292
|
-
{ "name" => f.name, "type" => pep, "offset" => f.offset, "bytes" => f.bytes }
|
|
370
|
+
m = { "name" => f.name, "type" => pep, "offset" => f.offset, "bytes" => f.bytes }
|
|
371
|
+
m["endian"] = f.endian.to_s if f.endian
|
|
372
|
+
m
|
|
293
373
|
end
|
|
294
374
|
|
|
295
375
|
schema = {
|
|
@@ -320,6 +400,12 @@ class CArray::Serializer # :nodoc:
|
|
|
320
400
|
# synthesise an anonymous struct from the Layer 1 members[], else
|
|
321
401
|
# leave the raw CA_FIXLEN array untouched.
|
|
322
402
|
def wrap_data_class (ca, schema)
|
|
403
|
+
members = schema.is_a?(Hash) ? schema["members"] : nil
|
|
404
|
+
unless schema.is_a?(Hash) &&
|
|
405
|
+
(members.nil? || members.is_a?(Array) && members.all?(Hash))
|
|
406
|
+
raise ArgumentError, "corrupt CArray binary data (data_class schema is not a " \
|
|
407
|
+
"mapping with a list of member mappings)"
|
|
408
|
+
end
|
|
323
409
|
name = schema["name"]
|
|
324
410
|
if name
|
|
325
411
|
begin
|
|
@@ -352,9 +438,17 @@ class CArray::Serializer # :nodoc:
|
|
|
352
438
|
members.each do |m|
|
|
353
439
|
sym = PEP_TO_SYMBOL[m["type"]]
|
|
354
440
|
unless sym
|
|
355
|
-
raise "unknown data_class member type #{m["type"].inspect} in CArray binary data"
|
|
441
|
+
raise ArgumentError, "unknown data_class member type #{m["type"].inspect} in CArray binary data"
|
|
356
442
|
end
|
|
357
|
-
|
|
443
|
+
opt = { :offset => m["offset"] }
|
|
444
|
+
if m.key?("endian")
|
|
445
|
+
endian = %w[big little native preserve].find { |e| e == m["endian"] }
|
|
446
|
+
unless endian
|
|
447
|
+
raise ArgumentError, "unknown data_class member endian #{m["endian"].inspect} in CArray binary data"
|
|
448
|
+
end
|
|
449
|
+
opt[:endian] = endian.to_sym
|
|
450
|
+
end
|
|
451
|
+
member sym, m["name"], opt
|
|
358
452
|
end
|
|
359
453
|
end
|
|
360
454
|
end
|
|
@@ -533,6 +627,11 @@ class CArray
|
|
|
533
627
|
# @param data [Array] Marshal payload.
|
|
534
628
|
# @return [void]
|
|
535
629
|
def marshal_load (data)
|
|
630
|
+
initialize_copy(CArray.send(:array_from_marshal_payload, data))
|
|
631
|
+
end
|
|
632
|
+
|
|
633
|
+
# The array a {#marshal_dump} payload describes.
|
|
634
|
+
def self.array_from_marshal_payload (data)
|
|
536
635
|
tag, *rest = data
|
|
537
636
|
case tag
|
|
538
637
|
when "object"
|
|
@@ -543,13 +642,84 @@ class CArray
|
|
|
543
642
|
ca.mask = 0
|
|
544
643
|
ca.mask[] = mask
|
|
545
644
|
end
|
|
546
|
-
|
|
645
|
+
ca
|
|
547
646
|
when "portable"
|
|
548
|
-
|
|
549
|
-
|
|
647
|
+
CArray.load(StringIO.new(rest[0]))
|
|
648
|
+
when "record"
|
|
649
|
+
data_class, storage = rest
|
|
650
|
+
CARecord.wrap(array_from_marshal_payload(storage), data_class)
|
|
550
651
|
else
|
|
551
652
|
raise TypeError, "unrecognised CArray Marshal payload"
|
|
552
653
|
end
|
|
553
654
|
end
|
|
655
|
+
private_class_method :array_from_marshal_payload
|
|
656
|
+
|
|
657
|
+
end
|
|
658
|
+
|
|
659
|
+
# Marshal of a view and a scalar: the {CArray#marshal_dump} payload, loaded
|
|
660
|
+
# back by the class method rather than into an empty instance. A view comes
|
|
661
|
+
# back as a plain array (a copy owns its memory, as #copy gives); a scalar
|
|
662
|
+
# comes back a scalar. A wrap is refused below.
|
|
663
|
+
[CAView, CAWrap, CScalar].each do |klass|
|
|
664
|
+
klass.class_eval do
|
|
665
|
+
# A Face other than a record is refused: the payload would carry its
|
|
666
|
+
# storage without what makes it the Face (a unit, labels), and load
|
|
667
|
+
# could not put that back.
|
|
668
|
+
# @return [String]
|
|
669
|
+
def _dump (_level)
|
|
670
|
+
if face?
|
|
671
|
+
raise TypeError,
|
|
672
|
+
"Marshal cannot carry a #{self.class} yet (its Face would be lost); " \
|
|
673
|
+
"dump its storage with .parent"
|
|
674
|
+
end
|
|
675
|
+
Marshal.dump(CArray.instance_method(:marshal_dump).bind_call(self))
|
|
676
|
+
end
|
|
677
|
+
end
|
|
678
|
+
end
|
|
679
|
+
|
|
680
|
+
class CAView
|
|
681
|
+
# @return [CArray] the plain array a {#_dump} string describes.
|
|
682
|
+
def self._load (str)
|
|
683
|
+
CArray.send(:array_from_marshal_payload, Marshal.load(str))
|
|
684
|
+
end
|
|
685
|
+
end
|
|
686
|
+
|
|
687
|
+
class CAWrap
|
|
688
|
+
# A wrap borrows memory, which Marshal cannot carry to another process;
|
|
689
|
+
# Ruby would also try to write the objects that keep the borrowed memory
|
|
690
|
+
# alive. Its copy owns its memory and marshals.
|
|
691
|
+
def _dump (_level)
|
|
692
|
+
raise TypeError,
|
|
693
|
+
"Marshal cannot carry a #{self.class}: it borrows its memory; " \
|
|
694
|
+
"dump .copy instead"
|
|
695
|
+
end
|
|
696
|
+
|
|
697
|
+
def self._load (str)
|
|
698
|
+
CArray.send(:array_from_marshal_payload, Marshal.load(str))
|
|
699
|
+
end
|
|
700
|
+
end
|
|
701
|
+
|
|
702
|
+
class CScalar
|
|
703
|
+
# @return [CScalar] the scalar a {#_dump} string describes.
|
|
704
|
+
def self._load (str)
|
|
705
|
+
a = CArray.send(:array_from_marshal_payload, Marshal.load(str))
|
|
706
|
+
s = CScalar.new(a.data_type, bytes: a.bytes)
|
|
707
|
+
s[0] = a[0]
|
|
708
|
+
s
|
|
709
|
+
end
|
|
710
|
+
end
|
|
711
|
+
|
|
712
|
+
class CARecord
|
|
713
|
+
|
|
714
|
+
# A record array keeps marshal_dump: it round-trips with its class. A
|
|
715
|
+
# named struct is written by name with the record bytes, so every kind of
|
|
716
|
+
# member (nested, fixlen, bit, array) travels. An anonymous struct has no
|
|
717
|
+
# name to write: it goes through the binary format, whose schema
|
|
718
|
+
# describes flat numeric members only.
|
|
719
|
+
def marshal_dump
|
|
720
|
+
return CArray.instance_method(:marshal_dump).bind_call(self) unless data_class.name
|
|
721
|
+
["record", data_class,
|
|
722
|
+
CArray.instance_method(:marshal_dump).bind_call(copy.parent)]
|
|
723
|
+
end
|
|
554
724
|
|
|
555
725
|
end
|
data/lib/carray/slab_iterator.rb
CHANGED
|
@@ -152,12 +152,24 @@ class CASlabIterator < CAIterator
|
|
|
152
152
|
# @!method max_addr
|
|
153
153
|
# Per-slab flat source address of the maximum.
|
|
154
154
|
# @return [CArray] one flat address per slab
|
|
155
|
-
|
|
155
|
+
#
|
|
156
|
+
# Each takes `min_count:` and `fill_value:` and passes them to the core
|
|
157
|
+
# reduction, as the other iterators do; `all` and `any` take neither, as
|
|
158
|
+
# the core ones do not.
|
|
159
|
+
[:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev,
|
|
156
160
|
:variancep, :stddevp, :minmax, :min_index, :max_index,
|
|
157
161
|
:min_addr, :max_addr].each do |op|
|
|
158
162
|
class_eval <<~RUBY, __FILE__, __LINE__ + 1
|
|
159
|
-
def #{op}
|
|
160
|
-
@reference.#{op}(axis: @slab_axes)
|
|
163
|
+
def #{op} (min_count: nil, fill_value: nil)
|
|
164
|
+
@reference.#{op}(axis: @slab_axes, **reduce_options(min_count, fill_value))
|
|
165
|
+
end
|
|
166
|
+
RUBY
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
[:all, :any].each do |op|
|
|
170
|
+
class_eval <<~RUBY, __FILE__, __LINE__ + 1
|
|
171
|
+
def #{op} (**kw)
|
|
172
|
+
@reference.#{op}(axis: @slab_axes, **CAIterator.reduce_keywords(:#{op}, kw))
|
|
161
173
|
end
|
|
162
174
|
RUBY
|
|
163
175
|
end
|
|
@@ -167,25 +179,28 @@ class CASlabIterator < CAIterator
|
|
|
167
179
|
# No argument counts present (non-masked) cells; `count(UNDEF)` counts
|
|
168
180
|
# masked cells; `count(v)` counts cells equal to `v`.
|
|
169
181
|
# @return [CArray] one count per slab
|
|
170
|
-
def count (*args)
|
|
171
|
-
return count_not_masked if args.empty?
|
|
182
|
+
def count (*args, **kw)
|
|
183
|
+
return count_not_masked(**kw) if args.empty?
|
|
172
184
|
# CABlock / CAWindow shadow #count with a geometry accessor, and @reference
|
|
173
185
|
# may be such a view, so dispatch CArray#count explicitly for count(v).
|
|
174
|
-
CArray.instance_method(:count).bind_call(@reference, *args, axis: @slab_axes
|
|
186
|
+
CArray.instance_method(:count).bind_call(@reference, *args, axis: @slab_axes,
|
|
187
|
+
**CAIterator.reduce_keywords(:count, kw))
|
|
175
188
|
end
|
|
176
189
|
|
|
177
190
|
# @overload count_not_masked
|
|
178
191
|
# Per-slab count of present (non-masked) cells.
|
|
179
192
|
# @return [CArray]
|
|
180
|
-
def count_not_masked
|
|
181
|
-
@reference.count_not_masked(axis: @slab_axes
|
|
193
|
+
def count_not_masked (**kw)
|
|
194
|
+
@reference.count_not_masked(axis: @slab_axes,
|
|
195
|
+
**CAIterator.reduce_keywords(:count_not_masked, kw))
|
|
182
196
|
end
|
|
183
197
|
|
|
184
198
|
# @overload count_masked
|
|
185
199
|
# Per-slab count of masked cells.
|
|
186
200
|
# @return [CArray]
|
|
187
|
-
def count_masked
|
|
188
|
-
@reference.count_masked(axis: @slab_axes
|
|
201
|
+
def count_masked (**kw)
|
|
202
|
+
@reference.count_masked(axis: @slab_axes,
|
|
203
|
+
**CAIterator.reduce_keywords(:count_masked, kw))
|
|
189
204
|
end
|
|
190
205
|
|
|
191
206
|
# @overload elements
|
|
@@ -206,30 +221,31 @@ class CASlabIterator < CAIterator
|
|
|
206
221
|
#
|
|
207
222
|
# Core order statistics take a single integer axis, so a single slab axis is
|
|
208
223
|
# delegated directly (fast C path) and a multi-axis slab is folded per slab
|
|
209
|
-
# with reduce_slab (each slab flattened).
|
|
210
|
-
#
|
|
211
|
-
# with `ca.value` first if needed.
|
|
224
|
+
# with reduce_slab (each slab flattened). Masked cells are left out, as in
|
|
225
|
+
# `CArray#median(axis:)`.
|
|
212
226
|
|
|
213
|
-
# @overload median
|
|
227
|
+
# @overload median(min_count: nil, fill_value: nil)
|
|
214
228
|
# Per-slab median.
|
|
215
229
|
# @return [CArray]
|
|
216
|
-
def median
|
|
230
|
+
def median (min_count: nil, fill_value: nil)
|
|
231
|
+
opts = reduce_options(min_count, fill_value)
|
|
217
232
|
if @slab_axes.size == 1
|
|
218
|
-
@reference.median(axis: @slab_axes[0])
|
|
233
|
+
@reference.median(axis: @slab_axes[0], **opts)
|
|
219
234
|
else
|
|
220
|
-
@reference.reduce_slab(axis: @slab_axes) { |s| s.median }
|
|
235
|
+
@reference.reduce_slab(axis: @slab_axes) { |s| s.median(**opts) }
|
|
221
236
|
end
|
|
222
237
|
end
|
|
223
238
|
|
|
224
|
-
# @overload percentile(*pers)
|
|
239
|
+
# @overload percentile(*pers, min_count: nil, fill_value: nil)
|
|
225
240
|
# Per-slab percentile(s). One argument returns one CArray; several return an
|
|
226
241
|
# array of CArrays (as `CArray#percentile`).
|
|
227
242
|
# @return [CArray, Array<CArray>]
|
|
228
|
-
def percentile (*pers)
|
|
243
|
+
def percentile (*pers, min_count: nil, fill_value: nil)
|
|
244
|
+
opts = reduce_options(min_count, fill_value)
|
|
229
245
|
if @slab_axes.size == 1
|
|
230
|
-
@reference.percentile(*pers, axis: @slab_axes[0])
|
|
246
|
+
@reference.percentile(*pers, axis: @slab_axes[0], **opts)
|
|
231
247
|
else
|
|
232
|
-
rs = pers.map { |p| @reference.reduce_slab(axis: @slab_axes) { |s| s.percentile(p) } }
|
|
248
|
+
rs = pers.map { |p| @reference.reduce_slab(axis: @slab_axes) { |s| s.percentile(p, **opts) } }
|
|
233
249
|
pers.size == 1 ? rs[0] : rs
|
|
234
250
|
end
|
|
235
251
|
end
|
|
@@ -279,18 +295,18 @@ class CASlabIterator < CAIterator
|
|
|
279
295
|
# the same shape as the reference (one weight per cell). Core wsum / wmean
|
|
280
296
|
# accept multi-axis, so this covers multi-axis slabs directly.
|
|
281
297
|
|
|
282
|
-
# @overload wsum(weights)
|
|
298
|
+
# @overload wsum(weights, min_count: nil, fill_value: nil)
|
|
283
299
|
# Per-slab weighted sum, `weights` shaped like the reference.
|
|
284
300
|
# @return [CArray]
|
|
285
|
-
def wsum (weights)
|
|
286
|
-
@reference.wsum(weights, axis: @slab_axes)
|
|
301
|
+
def wsum (weights, min_count: nil, fill_value: nil)
|
|
302
|
+
@reference.wsum(weights, axis: @slab_axes, **reduce_options(min_count, fill_value))
|
|
287
303
|
end
|
|
288
304
|
|
|
289
|
-
# @overload wmean(weights)
|
|
305
|
+
# @overload wmean(weights, min_count: nil, fill_value: nil)
|
|
290
306
|
# Per-slab weighted mean, `weights` shaped like the reference.
|
|
291
307
|
# @return [CArray]
|
|
292
|
-
def wmean (weights)
|
|
293
|
-
@reference.wmean(weights, axis: @slab_axes)
|
|
308
|
+
def wmean (weights, min_count: nil, fill_value: nil)
|
|
309
|
+
@reference.wmean(weights, axis: @slab_axes, **reduce_options(min_count, fill_value))
|
|
294
310
|
end
|
|
295
311
|
|
|
296
312
|
# ---- segment scan: within-slab running statistics ---------------------
|
|
@@ -332,6 +348,14 @@ class CASlabIterator < CAIterator
|
|
|
332
348
|
|
|
333
349
|
# The single slab axis for the order surface (core sort takes one integer
|
|
334
350
|
# axis). A multi-axis slab has no single sort axis, so raise.
|
|
351
|
+
# The reduction keywords that were given, for the core call.
|
|
352
|
+
def reduce_options (min_count, fill_value)
|
|
353
|
+
opts = {}
|
|
354
|
+
opts[:min_count] = min_count unless min_count.nil?
|
|
355
|
+
opts[:fill_value] = fill_value unless fill_value.nil?
|
|
356
|
+
opts
|
|
357
|
+
end
|
|
358
|
+
|
|
335
359
|
def single_sort_axis (op)
|
|
336
360
|
return @slab_axes[0] if @slab_axes.size == 1
|
|
337
361
|
raise ArgumentError,
|
data/lib/carray/stack.rb
CHANGED
|
@@ -51,13 +51,15 @@ class CArray
|
|
|
51
51
|
# `data_type` and re-wraps homogeneous Face inputs via
|
|
52
52
|
# `face_lift`. Output `ndim` is one greater than each piece.
|
|
53
53
|
# @param list [Array<CArray>] pieces to stack; must not be empty.
|
|
54
|
-
# @param axis [Integer] position of the new K axis
|
|
54
|
+
# @param axis [Integer, nil] position of the new K axis; `nil` is
|
|
55
|
+
# the same as `0`.
|
|
55
56
|
# @param data_type [Symbol, Integer, nil] result `data_type`;
|
|
56
57
|
# inferred when `nil`.
|
|
57
58
|
# @return [CArray] CAStack view.
|
|
58
59
|
# @raise [ArgumentError] when `list` is empty.
|
|
59
60
|
def self.stack (list, axis: 0, data_type: nil)
|
|
60
61
|
raise ArgumentError, "stack: list must not be empty" if list.empty?
|
|
62
|
+
axis = 0 if axis.nil?
|
|
61
63
|
list = CArray.promote_list(list, data_type: data_type)
|
|
62
64
|
axis = CArray.normalize_axis(axis, list[0].ndim + 1, "stack")
|
|
63
65
|
CAStack.new(list, axis: axis) # CAStack.new does Face lift internally
|
|
@@ -80,8 +82,9 @@ class CArray
|
|
|
80
82
|
# Convenience form: a single Array argument is treated as the list.
|
|
81
83
|
# @param arrays [Array<CArray>] pieces to weld. A single Array
|
|
82
84
|
# argument is accepted for compatibility with older callers.
|
|
83
|
-
# @param axis [Integer] existing axis to extend (normalises
|
|
84
|
-
# values against the reference ndim)
|
|
85
|
+
# @param axis [Integer, nil] existing axis to extend (normalises
|
|
86
|
+
# negative values against the reference ndim); `nil` is the same
|
|
87
|
+
# as `0`.
|
|
85
88
|
# @return [CAMeld] view over the welded pieces.
|
|
86
89
|
# @raise [ArgumentError] when the list is empty, ndim mismatch,
|
|
87
90
|
# data_type mismatch, or non-axis dim mismatch across pieces
|
|
@@ -91,6 +94,7 @@ class CArray
|
|
|
91
94
|
arrays = arrays[0]
|
|
92
95
|
end
|
|
93
96
|
raise ArgumentError, "meld: list must not be empty" if arrays.empty?
|
|
97
|
+
axis = 0 if axis.nil?
|
|
94
98
|
first = arrays[0]
|
|
95
99
|
unless first.is_a?(CArray)
|
|
96
100
|
raise ArgumentError, "meld: entries must be CArray (got #{first.class})"
|
|
@@ -247,8 +251,10 @@ class CArray
|
|
|
247
251
|
# entities.
|
|
248
252
|
# @param axis [Integer] axis to split along.
|
|
249
253
|
# @return [Array<CArray>] one slice per index along `axis`.
|
|
250
|
-
# @raise [ArgumentError] when `axis` is not a single Integer
|
|
254
|
+
# @raise [ArgumentError] when `axis` is not a single Integer
|
|
255
|
+
# (including `nil`).
|
|
251
256
|
def split (axis:)
|
|
257
|
+
raise ArgumentError, "split: axis: is required" if axis.nil?
|
|
252
258
|
if axis.is_a?(Array)
|
|
253
259
|
raise ArgumentError, "split: axis must be a single Integer"
|
|
254
260
|
end
|
|
@@ -153,13 +153,14 @@ class CArray
|
|
|
153
153
|
|
|
154
154
|
# @overload extract(regexp, replace = '\0')
|
|
155
155
|
# Returns a CArray whose cells are each cell's first match of `regexp`
|
|
156
|
-
# transformed by `String#sub(regexp, replace)
|
|
157
|
-
#
|
|
156
|
+
# transformed by `String#sub(regexp, replace)`. A cell that does not
|
|
157
|
+
# match is masked (`UNDEF`), so it stays apart from a match of the empty
|
|
158
|
+
# string, and a later `cast` to a number does not read it as 0.
|
|
158
159
|
# @param regexp [Regexp]
|
|
159
160
|
# @param replace [String]
|
|
160
161
|
# @return [CAString]
|
|
161
162
|
def extract (regexp, replace = '\0')
|
|
162
|
-
string_map { |s| regexp.match(s) { |m| m[0].sub(regexp, replace) } ||
|
|
163
|
+
string_map { |s| regexp.match(s) { |m| m[0].sub(regexp, replace) } || UNDEF }
|
|
163
164
|
end
|
|
164
165
|
|
|
165
166
|
# --- inter-Face conversions (String Faces only) ----------------------
|