carray 3.0.3 → 3.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +469 -0
  3. data/README.md +3 -2
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_array_pool.c +25 -1
  6. data/ext/ca_axis_descriptor.h +27 -24
  7. data/ext/ca_axis_dispatch.c +14 -26
  8. data/ext/ca_axis_group.c +27 -27
  9. data/ext/ca_binop_dispatch.c +1 -29
  10. data/ext/ca_binop_dispatch.h +0 -1
  11. data/ext/ca_categorical_iterator.c +231 -153
  12. data/ext/ca_compare.c +9 -20
  13. data/ext/ca_composite_dispatch.c +9 -9
  14. data/ext/ca_group_iter.c +16 -17
  15. data/ext/ca_kernel_iterator.c +1240 -2000
  16. data/ext/ca_kernel_iterator.h +502 -519
  17. data/ext/ca_monop_dispatch.c +82 -59
  18. data/ext/ca_monop_dispatch.h +1 -0
  19. data/ext/ca_obj_array.c +152 -45
  20. data/ext/ca_obj_bincmp.c +25 -21
  21. data/ext/ca_obj_binop.c +16 -26
  22. data/ext/ca_obj_bitarray.c +8 -3
  23. data/ext/ca_obj_bitfield.c +63 -49
  24. data/ext/ca_obj_block.c +18 -9
  25. data/ext/ca_obj_byte_swap.c +7 -7
  26. data/ext/ca_obj_const_string.c +188 -22
  27. data/ext/ca_obj_face.c +280 -58
  28. data/ext/ca_obj_face.h +19 -0
  29. data/ext/ca_obj_fake.c +15 -29
  30. data/ext/ca_obj_farray.c +6 -3
  31. data/ext/ca_obj_field.c +30 -8
  32. data/ext/ca_obj_fixlen_string.c +2 -3
  33. data/ext/ca_obj_grid.c +455 -134
  34. data/ext/ca_obj_meld.c +172 -34
  35. data/ext/ca_obj_moncmp.c +23 -8
  36. data/ext/ca_obj_monop.c +113 -67
  37. data/ext/ca_obj_object.c +210 -52
  38. data/ext/ca_obj_record.c +15 -7
  39. data/ext/ca_obj_reduce.c +17 -8
  40. data/ext/ca_obj_refer.c +42 -35
  41. data/ext/ca_obj_remap.c +30 -9
  42. data/ext/ca_obj_repeat.c +9 -10
  43. data/ext/ca_obj_roll.c +31 -10
  44. data/ext/ca_obj_select.c +48 -14
  45. data/ext/ca_obj_select_axis.c +15 -47
  46. data/ext/ca_obj_shift.c +41 -7
  47. data/ext/ca_obj_stack.c +180 -34
  48. data/ext/ca_obj_stride.c +190 -134
  49. data/ext/ca_obj_string.c +80 -4
  50. data/ext/ca_obj_tile.c +32 -11
  51. data/ext/ca_obj_time.c +22 -6
  52. data/ext/ca_obj_timedelta.c +22 -6
  53. data/ext/ca_obj_transpose.c +13 -7
  54. data/ext/ca_obj_triop.c +26 -32
  55. data/ext/ca_obj_window.c +267 -74
  56. data/ext/ca_op_ipower.c +4 -4
  57. data/ext/ca_sort_kernels.h +6 -3
  58. data/ext/ca_sweep_engine.c +2 -2
  59. data/ext/caframe_csv_reader.c +80 -1
  60. data/ext/carray.h +214 -75
  61. data/ext/carray_access.c +200 -99
  62. data/ext/carray_address_basis.c +343 -87
  63. data/ext/carray_attribute.c +113 -6
  64. data/ext/carray_bincount.c +10 -8
  65. data/ext/carray_broadcast.c +13 -8
  66. data/ext/carray_build_flags.h +3 -0
  67. data/ext/carray_call_cfunc.c +108 -0
  68. data/ext/carray_cast.c +375 -96
  69. data/ext/carray_cast_func.rb +90 -20
  70. data/ext/carray_class.c +4 -3
  71. data/ext/carray_config.h +28 -0
  72. data/ext/carray_conversion.c +18 -9
  73. data/ext/carray_copy.c +51 -14
  74. data/ext/carray_core.c +150 -55
  75. data/ext/carray_count.c +56 -20
  76. data/ext/carray_element.c +81 -33
  77. data/ext/carray_factorize.c +138 -84
  78. data/ext/carray_generate.c +199 -52
  79. data/ext/carray_histogram.c +63 -18
  80. data/ext/carray_hold.c +12 -15
  81. data/ext/carray_index_classifier.c +388 -176
  82. data/ext/carray_internal.h +348 -29
  83. data/ext/carray_kernels_bincmp.c +4453 -0
  84. data/ext/carray_kernels_binop.c +11368 -0
  85. data/ext/carray_kernels_init.c +1734 -0
  86. data/ext/carray_kernels_moncmp.c +2104 -0
  87. data/ext/carray_kernels_monop.c +18369 -0
  88. data/ext/carray_kernels_reduce_aggregate.c +23818 -0
  89. data/ext/carray_kernels_reduce_boolean.c +406 -0
  90. data/ext/carray_kernels_reduce_cumulative.c +10086 -0
  91. data/ext/carray_kernels_reduce_extreme.c +17634 -0
  92. data/ext/carray_kernels_reduce_variance.c +3849 -0
  93. data/ext/carray_kernels_scan.c +4302 -0
  94. data/ext/carray_kernels_search.c +33829 -0
  95. data/ext/carray_kernels_sort.c +10897 -0
  96. data/ext/carray_kernels_triop.c +1864 -0
  97. data/ext/carray_key_windows.c +220 -0
  98. data/ext/carray_lazy.c +170 -111
  99. data/ext/carray_mask.c +337 -94
  100. data/ext/carray_median_percentile.c +82 -28
  101. data/ext/carray_memory_view.c +669 -561
  102. data/ext/carray_operator.c +129 -52
  103. data/ext/carray_order.c +604 -77
  104. data/ext/carray_partition.c +28 -27
  105. data/ext/carray_random.c +85 -31
  106. data/ext/carray_round.c +273 -0
  107. data/ext/carray_scatter.c +87 -45
  108. data/ext/carray_segment.c +3 -3
  109. data/ext/carray_slab.c +123 -37
  110. data/ext/carray_sort.c +259 -51
  111. data/ext/carray_sort_kernel.c +5 -7
  112. data/ext/carray_struct.c +54 -61
  113. data/ext/carray_test.c +26 -5
  114. data/ext/carray_time_text.c +4 -2
  115. data/ext/carray_undef.c +4 -0
  116. data/ext/carray_utils.c +309 -66
  117. data/ext/mk_call_cfunc.rb +19 -2
  118. data/ext/mkkernel.rb +403 -350
  119. data/ext/ruby_carray.c +10 -4
  120. data/ext/version.h +4 -4
  121. data/lib/carray/attribute.rb +139 -22
  122. data/lib/carray/autoload_carray.rb +8 -31
  123. data/lib/carray/autoload_method_extension.rb +14 -5
  124. data/lib/carray/axis_group.rb +255 -106
  125. data/lib/carray/basics.rb +105 -17
  126. data/lib/carray/bincount_nd.rb +12 -2
  127. data/lib/carray/block_iterator.rb +126 -23
  128. data/lib/carray/boolean_reduce.rb +17 -17
  129. data/lib/carray/categorical.rb +195 -59
  130. data/lib/carray/categorical_iterator.rb +132 -29
  131. data/lib/carray/complex.rb +5 -5
  132. data/lib/carray/conditional.rb +67 -18
  133. data/lib/carray/const_string.rb +24 -5
  134. data/lib/carray/construct.rb +15 -11
  135. data/lib/carray/data_type_extension.rb +9 -1
  136. data/lib/carray/distance_limit.rb +77 -0
  137. data/lib/carray/fixlen_string.rb +48 -3
  138. data/lib/carray/frame/column_verbs.rb +71 -0
  139. data/lib/carray/frame/concat.rb +17 -3
  140. data/lib/carray/frame/convert.rb +69 -0
  141. data/lib/carray/frame/csv_parser.rb +111 -17
  142. data/lib/carray/frame/describe.rb +3 -2
  143. data/lib/carray/frame/frame.rb +252 -28
  144. data/lib/carray/frame/group.rb +603 -101
  145. data/lib/carray/frame/io.rb +117 -28
  146. data/lib/carray/frame/join.rb +312 -51
  147. data/lib/carray/frame/records.rb +83 -7
  148. data/lib/carray/frame/reshape.rb +42 -35
  149. data/lib/carray/frame/sort.rb +7 -19
  150. data/lib/carray/frame/verbs.rb +188 -48
  151. data/lib/carray/frame/windows.rb +117 -0
  152. data/lib/carray/frame.rb +2 -0
  153. data/lib/carray/fuse_source.rb +42 -6
  154. data/lib/carray/fusion.rb +40 -16
  155. data/lib/carray/histogram.rb +6 -1
  156. data/lib/carray/inspect.rb +31 -8
  157. data/lib/carray/iterator.rb +85 -7
  158. data/lib/carray/key_windows.rb +620 -0
  159. data/lib/carray/lazy.rb +217 -183
  160. data/lib/carray/mask_gap_fill.rb +63 -57
  161. data/lib/carray/meld_reduce.rb +12 -14
  162. data/lib/carray/methods/bin.rb +26 -18
  163. data/lib/carray/methods/bincount.rb +16 -0
  164. data/lib/carray/methods/bit_string.rb +1 -1
  165. data/lib/carray/methods/choose.rb +16 -3
  166. data/lib/carray/methods/composition.rb +3 -1
  167. data/lib/carray/methods/gather_nd.rb +14 -5
  168. data/lib/carray/methods/insert_block.rb +2 -2
  169. data/lib/carray/methods/is_between.rb +59 -0
  170. data/lib/carray/methods/is_in.rb +61 -37
  171. data/lib/carray/methods/join.rb +1 -6
  172. data/lib/carray/methods/lag.rb +49 -0
  173. data/lib/carray/methods/locate_addr.rb +47 -51
  174. data/lib/carray/methods/mode.rb +1 -1
  175. data/lib/carray/methods/nunique.rb +11 -6
  176. data/lib/carray/methods/resize.rb +3 -3
  177. data/lib/carray/methods/value_counts.rb +4 -11
  178. data/lib/carray/record_order.rb +3 -3
  179. data/lib/carray/rng.rb +3 -0
  180. data/lib/carray/runtime.rb +53 -8
  181. data/lib/carray/segment_iterator.rb +323 -88
  182. data/lib/carray/serialize.rb +45 -14
  183. data/lib/carray/slab_iterator.rb +50 -0
  184. data/lib/carray/stack.rb +15 -12
  185. data/lib/carray/string.rb +113 -5
  186. data/lib/carray/string_operation_extension.rb +21 -6
  187. data/lib/carray/struct.rb +9 -21
  188. data/lib/carray/struct_builder.rb +24 -11
  189. data/lib/carray/time.rb +819 -158
  190. data/lib/carray/window_iterator.rb +122 -16
  191. data/lib/carray.rb +7 -0
  192. data/yard-stubs/ca_obj_bitfield.rb +1 -1
  193. data/yard-stubs/ca_obj_field.rb +4 -0
  194. data/yard-stubs/ca_obj_grid.rb +21 -10
  195. data/yard-stubs/ca_obj_refer.rb +2 -1
  196. data/yard-stubs/ca_obj_roll.rb +2 -2
  197. data/yard-stubs/ca_obj_shift.rb +5 -1
  198. data/yard-stubs/ca_obj_tile.rb +1 -1
  199. data/yard-stubs/ca_obj_transpose.rb +5 -4
  200. data/yard-stubs/ca_obj_window.rb +8 -3
  201. data/yard-stubs/carray_access.rb +6 -1
  202. data/yard-stubs/carray_attribute.rb +3 -6
  203. data/yard-stubs/carray_cast.rb +12 -0
  204. data/yard-stubs/carray_conversion.rb +7 -2
  205. data/yard-stubs/carray_copy.rb +4 -2
  206. data/yard-stubs/carray_count.rb +4 -4
  207. data/yard-stubs/carray_element.rb +17 -10
  208. data/yard-stubs/carray_generate.rb +21 -1
  209. data/yard-stubs/carray_mask.rb +32 -3
  210. data/yard-stubs/carray_math.rb +41 -5
  211. data/yard-stubs/carray_median_percentile.rb +5 -2
  212. data/yard-stubs/carray_memory_view.rb +40 -23
  213. data/yard-stubs/carray_order.rb +129 -22
  214. data/yard-stubs/carray_random.rb +13 -4
  215. data/yard-stubs/carray_scatter.rb +17 -10
  216. data/yard-stubs/carray_slab.rb +9 -2
  217. data/yard-stubs/carray_sort.rb +6 -5
  218. data/yard-stubs/carray_test.rb +5 -3
  219. metadata +25 -3
  220. data/lib/carray/arrow_tensor.rb +0 -401
  221. data/lib/carray/table.rb +0 -108
@@ -63,6 +63,7 @@ class CArray::Serializer # :nodoc:
63
63
 
64
64
  # @!visibility private
65
65
  MAGIC = "_CARRAY3"
66
+ Ractor.make_shareable(MAGIC)
66
67
  # @!visibility private
67
68
  HEADER_BYTES = 256
68
69
  # @!visibility private
@@ -119,7 +120,8 @@ class CArray::Serializer # :nodoc:
119
120
  # `Marshal.dump(ca)` instead), or carries a data_class the
120
121
  # v1.0 flat-primitive schema cannot express.
121
122
  # @raise [TypeError] when `ca` is a Face other than a record (save
122
- # its storage with `.parent`).
123
+ # its storage through the Face's own accessor, such as `#ticks` or
124
+ # `#codes`, or its values with `Marshal.dump(ca.to_type(:object))`).
123
125
  def save (ca, endian: nil)
124
126
  if ca.data_type == :object
125
127
  raise ArgumentError,
@@ -131,12 +133,23 @@ class CArray::Serializer # :nodoc:
131
133
  # would carry its storage without what makes it the Face (a unit, the
132
134
  # string buffer, the labels), and load could not put that back.
133
135
  if ca.face? && !ca.is_a?(CARecord)
136
+ storage = ca.__send__(:face_storage_method)
134
137
  raise TypeError,
135
138
  "CArray.save cannot carry a #{ca.class} yet (its Face would be " \
136
- "lost); save its storage with .parent"
137
- end
138
-
139
- file_endian = endian || CArray.endian
139
+ "lost); " +
140
+ (storage ? "save its storage, which ##{storage} gives, or " : "") +
141
+ "keep its values with Marshal.dump(ca.to_type(:object))"
142
+ end
143
+
144
+ file_endian = case endian
145
+ when nil, :native then CArray.endian
146
+ when :big, CA_BIG_ENDIAN then CA_BIG_ENDIAN
147
+ when :little, CA_LITTLE_ENDIAN then CA_LITTLE_ENDIAN
148
+ else
149
+ raise ArgumentError,
150
+ "endian: must be :big, :little, :native, " \
151
+ "CA_BIG_ENDIAN or CA_LITTLE_ENDIAN (got #{endian.inspect})"
152
+ end
140
153
  swap = (file_endian != CArray.endian)
141
154
 
142
155
  elements = ca.elements
@@ -378,6 +391,7 @@ class CArray::Serializer # :nodoc:
378
391
  "members" => members,
379
392
  }
380
393
  schema["name"] = klass.name if klass.name
394
+ schema["order_by"] = klass.order_by.map(&:to_s) if klass.order_by
381
395
  schema
382
396
  end
383
397
 
@@ -390,7 +404,7 @@ class CArray::Serializer # :nodoc:
390
404
  if trailer.key?("attrs")
391
405
  attrs = trailer["attrs"]
392
406
  if attrs.is_a?(Hash) and not attrs.empty?
393
- ca.instance_variable_set(:@attr, attrs)
407
+ ca.set_attrs(attrs)
394
408
  end
395
409
  end
396
410
  ca
@@ -434,7 +448,15 @@ class CArray::Serializer # :nodoc:
434
448
  members = schema["members"]
435
449
  record_bytes = schema["record_bytes"]
436
450
  builder = (schema["kind"] == "union") ? CArray.method(:union) : CArray.method(:struct)
437
- builder.call(:pack => 1, :size => record_bytes) do
451
+ order_by = schema["order_by"]
452
+ names = members.map { |m| m["name"].to_s }
453
+ if order_by && !(order_by.is_a?(Array) && order_by.all? { |n| names.include?(n.to_s) })
454
+ raise ArgumentError, "corrupt CArray binary data (order_by #{order_by.inspect} " \
455
+ "names a member the data_class does not have)"
456
+ end
457
+ opts = { :pack => 1, :size => record_bytes }
458
+ opts[:order_by] = order_by.map(&:to_sym) if order_by
459
+ builder.call(**opts) do
438
460
  members.each do |m|
439
461
  sym = PEP_TO_SYMBOL[m["type"]]
440
462
  unless sym
@@ -615,7 +637,8 @@ class CArray
615
637
  ["object",
616
638
  target.shape,
617
639
  target.value.to_a,
618
- (target.has_mask? ? target.mask.to_a : nil)]
640
+ (target.has_mask? ? target.mask.to_a : nil),
641
+ target.attrs.to_h]
619
642
  else
620
643
  ["portable", CArray.dump(target)]
621
644
  end
@@ -627,7 +650,10 @@ class CArray
627
650
  # @param data [Array] Marshal payload.
628
651
  # @return [void]
629
652
  def marshal_load (data)
630
- initialize_copy(CArray.send(:array_from_marshal_payload, data))
653
+ source = CArray.send(:array_from_marshal_payload, data)
654
+ initialize_copy(source)
655
+ attrs = source.attrs
656
+ set_attrs(attrs) unless attrs.empty?
631
657
  end
632
658
 
633
659
  # The array a {#marshal_dump} payload describes.
@@ -635,19 +661,22 @@ class CArray
635
661
  tag, *rest = data
636
662
  case tag
637
663
  when "object"
638
- shape, values, mask = rest
664
+ shape, values, mask, attrs = rest
639
665
  ca = CArray.object(*shape)
640
666
  ca[] = values
641
667
  if mask
642
668
  ca.mask = 0
643
669
  ca.mask[] = mask
644
670
  end
671
+ ca.set_attrs(attrs) if attrs && !attrs.empty?
645
672
  ca
646
673
  when "portable"
647
674
  CArray.load(StringIO.new(rest[0]))
648
675
  when "record"
649
- data_class, storage = rest
650
- CARecord.wrap(array_from_marshal_payload(storage), data_class)
676
+ data_class, storage, attrs = rest
677
+ ca = CARecord.wrap(array_from_marshal_payload(storage), data_class)
678
+ ca.set_attrs(attrs) if attrs && !attrs.empty?
679
+ ca
651
680
  else
652
681
  raise TypeError, "unrecognised CArray Marshal payload"
653
682
  end
@@ -668,9 +697,11 @@ end
668
697
  # @return [String]
669
698
  def _dump (_level)
670
699
  if face?
700
+ storage = face_storage_method
671
701
  raise TypeError,
672
702
  "Marshal cannot carry a #{self.class} yet (its Face would be lost); " \
673
- "dump its storage with .parent"
703
+ "dump its values with to_type(:object)" +
704
+ (storage ? ", or its storage, which ##{storage} gives" : "")
674
705
  end
675
706
  Marshal.dump(CArray.instance_method(:marshal_dump).bind_call(self))
676
707
  end
@@ -719,7 +750,7 @@ class CARecord
719
750
  def marshal_dump
720
751
  return CArray.instance_method(:marshal_dump).bind_call(self) unless data_class.name
721
752
  ["record", data_class,
722
- CArray.instance_method(:marshal_dump).bind_call(copy.parent)]
753
+ CArray.instance_method(:marshal_dump).bind_call(copy.parent), attrs.to_h]
723
754
  end
724
755
 
725
756
  end
@@ -174,6 +174,27 @@ class CASlabIterator < CAIterator
174
174
  RUBY
175
175
  end
176
176
 
177
+ # @!method first
178
+ # Per-slab first present value, in row-major order over the slab axes;
179
+ # UNDEF for a slab with none.
180
+ # @return [CArray] one value per slab
181
+ # @!method last
182
+ # Per-slab last present value; UNDEF for a slab with none.
183
+ # @return [CArray] one value per slab
184
+ # @!method nunique
185
+ # Per-slab number of distinct present values (int64); 0 for a slab with
186
+ # none.
187
+ # @return [CArray] one count per slab
188
+ #
189
+ # These take no `min_count:` or `fill_value:`, as the core ones do not.
190
+ [:first, :last, :nunique].each do |op|
191
+ class_eval <<~RUBY, __FILE__, __LINE__ + 1
192
+ def #{op} (**kw)
193
+ @reference.#{op}(axis: @slab_axes, **CAIterator.reduce_keywords(:#{op}, kw))
194
+ end
195
+ RUBY
196
+ end
197
+
177
198
  # @overload count(v = <none>)
178
199
  # Per-slab count, delegating to `reference.count(..., axis: slab_axes)`.
179
200
  # No argument counts present (non-masked) cells; `count(UNDEF)` counts
@@ -344,6 +365,35 @@ class CASlabIterator < CAIterator
344
365
  RUBY
345
366
  end
346
367
 
368
+ # @overload lag(n = 1)
369
+ # Per-slab value `n` steps earlier along the slab axis (UNDEF where the
370
+ # slab has none), as `reference.lag(n, axis: slab_axis)`: a view of the
371
+ # reference. A multi-axis slab has no single direction to step in, so it
372
+ # raises, as the scans do.
373
+ # @return [CArray] reference-shaped
374
+ def lag (n = 1)
375
+ @reference.lag(n, axis: single_scan_axis(:lag))
376
+ end
377
+
378
+ # @overload strip_mask(method:)
379
+ # Per-slab fill of the masked cells along the slab axis, as
380
+ # `reference.strip_mask(method:, axis: slab_axis)`: `:forward` / `:ffill`
381
+ # or `:backward` / `:bfill`. A multi-axis slab raises, as the scans do.
382
+ # @return [CArray] reference-shaped
383
+ def strip_mask (method:)
384
+ CAIterator.hold_backward?(method)
385
+ @reference.strip_mask(method: method, axis: single_scan_axis(:strip_mask))
386
+ end
387
+
388
+ # @overload order(descending: false, method: :ordinal, kind: :quick)
389
+ # Per-slab rank of each cell along the slab axis (int64), as
390
+ # `reference.order(axis: slab_axis, ...)`. A multi-axis slab has no
391
+ # single sort axis, so it raises, as sort_addr does.
392
+ # @return [CArray] reference-shaped
393
+ def order (**kw)
394
+ @reference.order(axis: single_sort_axis(:order), **CAIterator.reduce_keywords(:order, kw))
395
+ end
396
+
347
397
  private
348
398
 
349
399
  # The single slab axis for the order surface (core sort takes one integer
data/lib/carray/stack.rb CHANGED
@@ -69,27 +69,28 @@ class CArray
69
69
  # No data is copied; reads gather from parents on demand and writes flow
70
70
  # back to them (chain composability preserved).
71
71
  #
72
- # Pieces must agree on `ndim`, `data_type`, byte width, and every axis
73
- # length except `axis` (the "meld axis"). Mismatched `data_type` raises:
74
- # cast the pieces yourself (`.to_type(:float64)`) or use
75
- # {CArray.concatenate} (eager, auto-casts).
72
+ # Pieces must agree on `ndim` and every axis length except `axis` (the
73
+ # "meld axis"). Pieces of different data types are brought to a common
74
+ # one as {CArray.stack} brings them (`promote_list`), through read-only
75
+ # views of the pieces that need it.
76
76
  #
77
77
  # "meld" = melt + weld — pieces dissolve their boundaries along the named
78
78
  # axis and are regarded as one.
79
79
  #
80
- # @overload meld(*arrays, axis: 0)
81
- # @overload meld(list, axis: 0)
80
+ # @overload meld(*arrays, axis: 0, data_type: nil)
81
+ # @overload meld(list, axis: 0, data_type: nil)
82
82
  # Convenience form: a single Array argument is treated as the list.
83
83
  # @param arrays [Array<CArray>] pieces to weld. A single Array
84
84
  # argument is accepted for compatibility with older callers.
85
85
  # @param axis [Integer, nil] existing axis to extend (normalises
86
86
  # negative values against the reference ndim); `nil` is the same
87
87
  # as `0`.
88
+ # @param data_type [Symbol, Integer, nil] result `data_type`;
89
+ # inferred when `nil`.
88
90
  # @return [CAMeld] view over the welded pieces.
89
91
  # @raise [ArgumentError] when the list is empty, ndim mismatch,
90
- # data_type mismatch, or non-axis dim mismatch across pieces
91
- # (surfaced by CAMeld.new).
92
- def self.meld (*arrays, axis: 0)
92
+ # or non-axis dim mismatch across pieces (surfaced by CAMeld.new).
93
+ def self.meld (*arrays, axis: 0, data_type: nil)
93
94
  if arrays.length == 1 && arrays[0].is_a?(Array)
94
95
  arrays = arrays[0]
95
96
  end
@@ -100,6 +101,7 @@ class CArray
100
101
  raise ArgumentError, "meld: entries must be CArray (got #{first.class})"
101
102
  end
102
103
  axis_norm = CArray.normalize_axis(axis, first.ndim, "meld")
104
+ arrays = CArray.promote_list(arrays, data_type: data_type)
103
105
  # Flatten nested CAMeld inputs that share our meld axis: they already
104
106
  # describe a segment sequence, so absorbing their parents keeps chain
105
107
  # depth at 1 (avoids 2-level xfer_all / reduce chains through the
@@ -119,14 +121,15 @@ class CArray
119
121
  # Instance form of {CArray.meld}; non-destructive, see the class method
120
122
  # for full semantics.
121
123
  #
122
- # @overload meld(*others, axis: 0)
124
+ # @overload meld(*others, axis: 0, data_type: nil)
123
125
  # @param others [Array<CArray>] additional pieces.
124
126
  # @param axis [Integer] existing axis to extend.
127
+ # @param data_type [Symbol, Integer, nil] result `data_type`.
125
128
  # @return [CAMeld]
126
129
  # @raise [ArgumentError] when no `others` are given.
127
- def meld (*others, axis: 0)
130
+ def meld (*others, axis: 0, data_type: nil)
128
131
  raise ArgumentError, "meld: at least one other array required" if others.empty?
129
- CArray.meld(self, *others, axis: axis)
132
+ CArray.meld(self, *others, axis: axis, data_type: data_type)
130
133
  end
131
134
 
132
135
  # Arrange `list` of uniform-shape pieces in a `tdim`-shape grid that
data/lib/carray/string.rb CHANGED
@@ -17,20 +17,28 @@ class CArray
17
17
  # CArray.string([a, nil, b]) # nil → masked element
18
18
  # CArray.string(other_ca) # from a String Face / object / raw fixlen
19
19
  #
20
- # `nil` entries become masked cells; "" (empty) is a valid distinct value.
21
- # A CArray source is normalised through a String Face ({CArray.string_face_of}):
22
- # a String Face converts, CA_OBJECT storage wraps, a raw CA_FIXLEN reads as
23
- # NUL-stripped strings; a numeric / boolean array is rejected (stringify with
24
- # {#format} / {CArray.format}).
20
+ # A cell of a CAString is a String. In the Array and block forms `nil`
21
+ # becomes a masked cell; "" (empty) is a valid distinct value. Any other
22
+ # value (a Symbol, a number, a nested Array) is refused: the values are one
23
+ # cell each, so a CAString of more than one dimension is built from an
24
+ # object array (`CArray.string(CA_OBJECT(nested))`).
25
+ # A CArray source is normalised through a String Face: a String Face
26
+ # converts, CA_OBJECT storage wraps zero-copy (every cell that is not
27
+ # masked must be a String, `nil` included: mask it instead), a raw
28
+ # CA_FIXLEN reads as NUL-stripped strings; a numeric / boolean array is
29
+ # rejected (stringify with {#format} / {CArray.format}).
25
30
  # @overload string(values)
26
31
  # Returns a {CAString} wrapping a CA_OBJECT entity of the given values.
27
32
  # @param values [Array<String, nil>] source values.
28
33
  # @return [CAString]
34
+ # @raise [TypeError] if a value is neither a String nor `nil`.
29
35
  # @overload string(ca)
30
36
  # Returns a {CAString} of the string-bearing CArray `ca`.
31
37
  # @param ca [CArray] a String Face, CA_OBJECT, or raw CA_FIXLEN array.
32
38
  # @return [CAString]
33
39
  # @raise [CArray::DataTypeError] if `ca` is numeric / boolean.
40
+ # @raise [TypeError] if a cell of an object array that is not masked is
41
+ # not a String.
34
42
  # @overload string(n) { |i| ... }
35
43
  # Returns an `n`-element {CAString} filled by the block, following the
36
44
  # arity-0 broadcast convention.
@@ -49,12 +57,26 @@ class CArray
49
57
  else
50
58
  values = Array.new(n) { |i| block.call(i) }
51
59
  end
60
+ elsif arg.is_a?(Integer)
61
+ raise ArgumentError,
62
+ "CArray.string: a count is given with a block that makes each " \
63
+ "String (CArray.string(n) { |i| ... })"
64
+ elsif arg.is_a?(String) || ! arg.respond_to?(:to_a)
65
+ raise TypeError,
66
+ "CArray.string: expected an Array of Strings, an object array or " \
67
+ "a String array (got #{arg.class})"
52
68
  else
53
69
  values = arg.to_a
54
70
  end
55
71
 
56
72
  entity = CArray.object(values.size)
57
73
  values.each_with_index do |s, i|
74
+ if s.is_a?(Array)
75
+ raise TypeError,
76
+ "CArray.string: value #{i} is an Array; a CAString cell is a " \
77
+ "String (for a CAString of another shape, give " \
78
+ "CArray.string an object array: CArray.string(CA_OBJECT(values)))"
79
+ end
58
80
  entity[i] = s.nil? ? UNDEF : s
59
81
  end
60
82
  CAString.wrap(entity)
@@ -66,4 +88,90 @@ end
66
88
  class CAString
67
89
  include CArray::StringOperationMixin
68
90
  include CArray::StringOperationMixin::Mutable
91
+
92
+ # The String operators and the picks by order run on the object storage,
93
+ # cell by cell, as on any object array. A String's +, * and % give a
94
+ # String (or raise), and a pick gives one of the operands, so the result
95
+ # is text and comes back as a CAString, as the bang forms (add!, mul!)
96
+ # keep it. A lazy operand is computed here.
97
+
98
+ # Per-cell `String#+`.
99
+ # @return [CAString] shape and mask carried.
100
+ def + (other) ; as_text(super) ; end
101
+ # Per-cell `String#+` (the word form of {#+}).
102
+ # @return [CAString]
103
+ def add (other) ; as_text(super) ; end
104
+ # Per-cell `String#*`.
105
+ # @return [CAString] shape and mask carried.
106
+ def * (other) ; as_text(super) ; end
107
+ # Per-cell `String#*` (the word form of {#*}).
108
+ # @return [CAString]
109
+ def mul (other) ; as_text(super) ; end
110
+ # Per-cell `String#%` (format).
111
+ # @return [CAString] shape and mask carried.
112
+ def % (other) ; as_text(super) ; end
113
+ # Per-cell `String#%` (the word form of {#%}).
114
+ # @return [CAString]
115
+ def mod (other) ; as_text(super) ; end
116
+ # The later String of each cell and `other`, cell by cell.
117
+ # @return [CAString]
118
+ def pmax (other) ; as_text(super) ; end
119
+ # The earlier String of each cell and `other`, cell by cell.
120
+ # @return [CAString]
121
+ def pmin (other) ; as_text(super) ; end
122
+ # Another name of {#pmax} (a String has no NaN to let win).
123
+ # @return [CAString]
124
+ def maximum (other) ; as_text(super) ; end
125
+ # Another name of {#pmin}.
126
+ # @return [CAString]
127
+ def minimum (other) ; as_text(super) ; end
128
+ # Another name of {#pmax}.
129
+ # @return [CAString]
130
+ def fmax (other) ; as_text(super) ; end
131
+ # Another name of {#pmin}.
132
+ # @return [CAString]
133
+ def fmin (other) ; as_text(super) ; end
134
+
135
+ # The String a blank cell holds: the cell a fresh {#template} gives and
136
+ # the one {#shift} fills by default. Frozen, because every blank cell
137
+ # shares it.
138
+ BLANK = "".freeze
139
+
140
+ # A same-shape CAString. With no block every cell is {BLANK}; a block
141
+ # fills the cells as for any array, and its values must be Strings or
142
+ # UNDEF. Given a data type, the result is a plain array of that type.
143
+ # @overload template(type = nil, **opts)
144
+ # @return [CAString, CArray]
145
+ def template (*args, **opts, &block)
146
+ return super if block || ! args.empty? || ! opts.empty?
147
+ out = super
148
+ out[] = BLANK
149
+ out
150
+ end
151
+
152
+ # Shifts the cells as {CArray#shift} does. The cells shifted in hold
153
+ # {BLANK} unless `fill_value:` names a String or UNDEF.
154
+ # @return [CAString] a view over the shifted cells.
155
+ def shift (*shifts, fill_value: BLANK, **opts)
156
+ super(*shifts, fill_value: fill_value, **opts)
157
+ end
158
+
159
+ # A window as {CArray#window} gives. The cells outside the array hold
160
+ # {BLANK} unless `fill_value:` names a String or UNDEF.
161
+ # @return [CAString] a view over the window.
162
+ def window (*ranges, fill_value: BLANK, **opts)
163
+ super(*ranges, fill_value: fill_value, **opts)
164
+ end
165
+
166
+ # A resized copy as {CArray#resize} gives. The new cells hold {BLANK}
167
+ # unless `fill_value:` names a String or UNDEF.
168
+ # @return [CAString]
169
+ def resize (*newdim, fill_value: BLANK)
170
+ super(*newdim, fill_value: fill_value)
171
+ end
172
+
173
+ private def as_text (result)
174
+ result = result.copy if result.lazy_view?
175
+ CAString.wrap(result)
176
+ end
69
177
  end
@@ -31,11 +31,14 @@ class CArray
31
31
  # (integer / boolean results). Shape and mask are carried.
32
32
  def string_map (data_type = nil) # :nodoc:
33
33
  out = data_type ? template(data_type) : CArray.object(*shape)
34
+ # A typed result has no nil: a cell the String method answers nil for
35
+ # (index finds nothing) is UNDEF.
36
+ cell = data_type ? ->(s) { v = yield(s); v.nil? ? UNDEF : v } : ->(s) { yield(s) }
34
37
  if has_mask?
35
38
  m = is_masked
36
- elements.times { |i| out[i] = (m[i] ? UNDEF : yield(self[i])) }
39
+ elements.times { |i| out[i] = (m[i] ? UNDEF : cell.(self[i])) }
37
40
  else
38
- elements.times { |i| out[i] = yield(self[i]) }
41
+ elements.times { |i| out[i] = cell.(self[i]) }
39
42
  end
40
43
  data_type ? out : CAString.wrap(out)
41
44
  end
@@ -124,10 +127,12 @@ class CArray
124
127
  # @return [CArray] `:int`, shape and mask of `self` preserved.
125
128
  def bytesize () ; string_map(:int) { |s| s.bytesize } ; end
126
129
  # Per-cell `String#index`.
127
- # @return [CArray] `:int`, shape and mask of `self` preserved.
130
+ # @return [CArray] `:int`, shape and mask of `self` preserved; UNDEF
131
+ # where the cell does not hold the substring.
128
132
  def str_index (*a) ; string_map(:int) { |s| s.index(*a) } ; end
129
133
  # Per-cell `String#rindex`.
130
- # @return [CArray] `:int`, shape and mask of `self` preserved.
134
+ # @return [CArray] `:int`, shape and mask of `self` preserved; UNDEF
135
+ # where the cell does not hold the substring.
131
136
  def rindex (*a) ; string_map(:int) { |s| s.rindex(*a) } ; end
132
137
 
133
138
  # ---- numeric parse (Integer / Float result) --------------------------
@@ -191,6 +196,9 @@ class CArray
191
196
  unless [:error, :silent].include?(truncate)
192
197
  raise ArgumentError, "truncate: must be :error or :silent (got #{truncate.inspect})"
193
198
  end
199
+ unless bytes.nil? || (bytes.is_a?(Integer) && bytes >= 1)
200
+ raise ArgumentError, "to_fixlen_string: invalid bytes: #{bytes.inspect} (a cell needs bytes: of 1 or more)"
201
+ end
194
202
  return self if is_a?(CAFixlenString) && (bytes.nil? || bytes == self.bytes)
195
203
 
196
204
  if has_mask?
@@ -200,7 +208,7 @@ class CArray
200
208
  values = Array.new(elements) { |i| self[i].to_s }
201
209
  end
202
210
  width = bytes || values.compact.map(&:bytesize).max || 1
203
- width = 1 if width < 1
211
+ width = 1 if width < 1 # every string empty: the narrowest cell
204
212
  if truncate == :error
205
213
  values.each_with_index do |s, i|
206
214
  next if s.nil?
@@ -278,8 +286,15 @@ class CArray
278
286
  # boolean array has no string reading and is rejected -- stringify explicitly
279
287
  # with #format / CArray.format. (This mirrors the numeric gate that rejects
280
288
  # numeric reductions on a String Face.)
281
- def self.string_face_of (ca)
289
+ def self.string_face_of (ca, verb = "CArray.string")
282
290
  return ca if ca.is_a?(CArray::StringOperationMixin)
291
+ # Another Face's storage is not its text: a CATime's fixlen surface
292
+ # holds ticks, a categorical's codes.
293
+ if ca.face?
294
+ raise TypeError,
295
+ "#{verb}: a #{ca.class} is not text " \
296
+ "(convert its values to Strings first)"
297
+ end
283
298
  case ca.data_type
284
299
  when :object then CAString.wrap(ca)
285
300
  when :fixlen then CAFixlenString.wrap(ca)
data/lib/carray/struct.rb CHANGED
@@ -340,14 +340,6 @@ class CAStruct
340
340
  super
341
341
  end
342
342
 
343
- protected
344
-
345
- def __data__
346
- @data
347
- end
348
-
349
- public
350
-
351
343
  # Member access. A member is named by a String (`s["lat"]`) or its
352
344
  # Integer position (`s[0]`); its accessor (`s.lat`) reads it too. A
353
345
  # Symbol is not a member name here, and a name that is not a member
@@ -509,26 +501,22 @@ class CAStruct
509
501
  # @!method <=(other)
510
502
  # @!method >(other)
511
503
  # @!method >=(other)
512
- { :< => ->(c) { c < 0 }, :<= => ->(c) { c <= 0 },
513
- :> => ->(c) { c > 0 }, :>= => ->(c) { c >= 0 } }.each do |op, test|
514
- define_method(op) do |other|
504
+ [:<, :<=, :>, :>=].each do |op|
505
+ define_method(op, &Ractor.make_shareable(proc do |other|
515
506
  c = (self <=> other)
516
507
  if c.nil?
517
508
  raise ArgumentError,
518
509
  "comparison of #{self.class.inspect} with #{other.class.inspect} failed" +
519
510
  (self.class.order_by ? "" : " (the struct declares no order_by:)")
520
511
  end
521
- test.call(c)
522
- end
512
+ c.public_send(op, 0)
513
+ end))
523
514
  end
524
515
 
525
- # One member of {#<=>}: NaN after every number, as a numeric array sorts.
516
+ # One member of {#<=>}: the order a sort puts the member's values in
517
+ # (NaN after every number).
526
518
  def self.compare_key (a, b)
527
- a_nan = a.is_a?(Float) && a.nan?
528
- b_nan = b.is_a?(Float) && b.nan?
529
- return (a_nan ? 0 : -1) if b_nan
530
- return 1 if a_nan
531
- a <=> b
519
+ CArray.send(:object_order_cmp, a, b)
532
520
  end
533
521
 
534
522
  # Byte-level identity. Two CAStruct instances are eql? iff they are
@@ -728,8 +716,8 @@ end
728
716
  # in Builder sets them up before any instance is created). Subclasses
729
717
  # created by Builder later don't need re-installation -- they inherit the
730
718
  # C methods from CAStruct.
731
- if CArray.respond_to?(:__install_castruct_methods__)
732
- CArray.__install_castruct_methods__(CAStruct)
719
+ if CArray.respond_to?(:install_struct_methods, true)
720
+ CArray.send(:install_struct_methods, CAStruct)
733
721
  end
734
722
 
735
723
  require 'carray/record_order'
@@ -224,11 +224,13 @@ class CAStruct::Builder # :nodoc:
224
224
  end
225
225
  vtype = {1 => :uint8, 2 => :uint16, 4 => :uint32, 8 => :uint64}[view_bytes]
226
226
  range = bit_in_byte..(bit_in_byte + bits - 1)
227
+ # The word is read little-endian on every host: bits are numbered
228
+ # byte by byte from the record's start, as the layout places them.
227
229
  reader = ->(data) {
228
- data.field(start_byte, vtype).bitfield(range)[0]
230
+ data.field(start_byte, vtype).endian(:little).bitfield(range)[0]
229
231
  }
230
232
  writer = ->(data, val) {
231
- data.field(start_byte, vtype).bitfield(range)[0] = val
233
+ data.field(start_byte, vtype).endian(:little).bitfield(range)[0] = val
232
234
  }
233
235
  [reader, writer].freeze
234
236
  end
@@ -241,7 +243,7 @@ class CAStruct::Builder # :nodoc:
241
243
  def self.build_nested_struct_dispatcher (offset, nested_class)
242
244
  reader = ->(data) {
243
245
  element = nested_class.allocate
244
- element.instance_variable_set(:@data, data.field(offset, nested_class).parent)
246
+ element.instance_variable_set(:@data, data.field(offset, nested_class).send(:face_storage))
245
247
  element.freeze if data.frozen?
246
248
  element
247
249
  }
@@ -288,6 +290,15 @@ class CAStruct::Builder # :nodoc:
288
290
  end
289
291
 
290
292
  # @!visibility private
293
+ # Shared with every Ractor when it can be. A member whose type is an
294
+ # array template holds a CArray, which cannot be shared, and its struct
295
+ # is then used in the main Ractor only.
296
+ def self.shareable (object)
297
+ Ractor.make_shareable(object)
298
+ rescue Ractor::Error
299
+ object.freeze
300
+ end
301
+
291
302
  ALIGN_TABLE = {
292
303
  CA_FIXLEN => CA_ALIGN_FIXLEN,
293
304
  CA_BOOLEAN => CA_ALIGN_INT8,
@@ -304,6 +315,7 @@ class CAStruct::Builder # :nodoc:
304
315
  CA_CMPLX64 => CA_ALIGN_FLOAT32,
305
316
  CA_CMPLX128 => CA_ALIGN_FLOAT64,
306
317
  }
318
+ Ractor.make_shareable(ALIGN_TABLE)
307
319
 
308
320
  # Raises ArgumentError when `opt` names a key outside `allowed`. The
309
321
  # options are read one key at a time, so a misspelt key would otherwise
@@ -463,9 +475,9 @@ class CAStruct::Builder # :nodoc:
463
475
  end
464
476
  table.freeze
465
477
  names.freeze
466
- klass.const_set(:MEMBER_TABLE, table) ### required element as data class
467
- klass.const_set(:MEMBERS, names)
468
- klass.const_set(:ORDER_BY, order_keys(table)) if @order_by
478
+ klass.const_set(:MEMBER_TABLE, self.class.shareable(table)) ### required element as data class
479
+ klass.const_set(:MEMBERS, self.class.shareable(names))
480
+ klass.const_set(:ORDER_BY, self.class.shareable(order_keys(table))) if @order_by
469
481
  # ---
470
482
  # Step 3 dispatch table: precompute a per-member [reader, writer]
471
483
  # Proc pair so `record[name]` is one Hash lookup + one Proc call,
@@ -473,7 +485,7 @@ class CAStruct::Builder # :nodoc:
473
485
  # checks and view-shape decisions happen here, once, at class
474
486
  # definition time.
475
487
  klass.const_set(:DISPATCH_TABLE,
476
- self.class.build_dispatch_table(table, @offset).freeze)
488
+ self.class.shareable(self.class.build_dispatch_table(table, @offset)))
477
489
  # Also precompute FAST_PRIMITIVES, the subset of members that the
478
490
  # C-side `[]` / `[]=` can handle without
479
491
  # allocating a CAField view. Entries are `[offset, type_code]`;
@@ -491,15 +503,16 @@ class CAStruct::Builder # :nodoc:
491
503
  klass.module_eval {
492
504
  taken = ->(m) { method_defined?(m) || private_method_defined?(m) }
493
505
  names.each do |name|
506
+ Ractor.make_shareable(name) # the accessors below capture it
494
507
  unless taken.(name)
495
- define_method(name) {
508
+ define_method(name, &Ractor.make_shareable(proc {
496
509
  return self[name]
497
- }
510
+ }))
498
511
  end
499
512
  unless taken.("#{name}=")
500
- define_method("#{name}=") { |val|
513
+ define_method("#{name}=", &Ractor.make_shareable(proc { |val|
501
514
  return self[name] = val
502
- }
515
+ }))
503
516
  end
504
517
  end
505
518
  }