carray 3.0.2 → 3.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +304 -0
  3. data/README.md +3 -4
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_dispatch.c +225 -0
  6. data/ext/ca_axis_group.c +177 -99
  7. data/ext/ca_binop_dispatch.c +54 -32
  8. data/ext/ca_binop_dispatch.h +1 -0
  9. data/ext/ca_categorical_iterator.c +66 -71
  10. data/ext/ca_compare.c +8 -2
  11. data/ext/ca_for_buffer.h +21 -68
  12. data/ext/ca_group_iter.c +27 -11
  13. data/ext/ca_kernel_iterator.c +663 -214
  14. data/ext/ca_kernel_iterator.h +231 -167
  15. data/ext/ca_monop_dispatch.c +4 -4
  16. data/ext/ca_monop_dispatch.h +4 -0
  17. data/ext/ca_obj_array.c +132 -22
  18. data/ext/ca_obj_bincmp.c +152 -121
  19. data/ext/ca_obj_binop.c +107 -210
  20. data/ext/ca_obj_bitarray.c +37 -24
  21. data/ext/ca_obj_bitfield.c +95 -62
  22. data/ext/ca_obj_block.c +1 -0
  23. data/ext/ca_obj_byte_swap.c +40 -40
  24. data/ext/ca_obj_const_string.c +96 -25
  25. data/ext/ca_obj_face.c +406 -64
  26. data/ext/ca_obj_face.h +92 -47
  27. data/ext/ca_obj_fake.c +21 -50
  28. data/ext/ca_obj_farray.c +1 -0
  29. data/ext/ca_obj_field.c +16 -3
  30. data/ext/ca_obj_fixlen_string.c +21 -21
  31. data/ext/ca_obj_grid.c +181 -124
  32. data/ext/ca_obj_meld.c +189 -107
  33. data/ext/ca_obj_moncmp.c +61 -61
  34. data/ext/ca_obj_monop.c +125 -107
  35. data/ext/ca_obj_object.c +543 -233
  36. data/ext/ca_obj_record.c +51 -16
  37. data/ext/ca_obj_reduce.c +5 -8
  38. data/ext/ca_obj_refer.c +117 -33
  39. data/ext/ca_obj_remap.c +57 -17
  40. data/ext/ca_obj_repeat.c +21 -5
  41. data/ext/ca_obj_roll.c +115 -11
  42. data/ext/ca_obj_select.c +47 -49
  43. data/ext/ca_obj_select_axis.c +58 -60
  44. data/ext/ca_obj_shift.c +22 -102
  45. data/ext/ca_obj_stack.c +223 -118
  46. data/ext/ca_obj_stride.c +195 -34
  47. data/ext/ca_obj_string.c +17 -18
  48. data/ext/ca_obj_tile.c +32 -16
  49. data/ext/ca_obj_time.c +23 -25
  50. data/ext/ca_obj_timedelta.c +21 -23
  51. data/ext/ca_obj_transpose.c +12 -3
  52. data/ext/ca_obj_triop.c +110 -117
  53. data/ext/ca_obj_window.c +149 -27
  54. data/ext/ca_op_ipower.c +24 -11
  55. data/ext/ca_op_powi.h +57 -30
  56. data/ext/ca_pow5_table.h +656 -0
  57. data/ext/ca_sort_kernels.h +70 -0
  58. data/ext/ca_sweep_engine.c +102 -79
  59. data/ext/ca_sweep_engine.h +7 -20
  60. data/ext/ca_triop_dispatch.c +4 -1
  61. data/ext/ca_triop_dispatch.h +5 -2
  62. data/ext/caframe_csv_reader.c +463 -0
  63. data/ext/carray.h +70 -27
  64. data/ext/carray_access.c +455 -211
  65. data/ext/carray_address_basis.c +177 -36
  66. data/ext/carray_attribute.c +45 -3
  67. data/ext/carray_bincount.c +32 -2
  68. data/ext/carray_broadcast.c +46 -14
  69. data/ext/carray_call_cfunc.c +0 -28
  70. data/ext/carray_cast.c +1444 -209
  71. data/ext/carray_cast_func.rb +34 -21
  72. data/ext/carray_conversion.c +197 -73
  73. data/ext/carray_copy.c +32 -11
  74. data/ext/carray_core.c +514 -122
  75. data/ext/carray_count.c +140 -87
  76. data/ext/carray_element.c +18 -27
  77. data/ext/carray_factorize.c +569 -277
  78. data/ext/carray_generate.c +76 -40
  79. data/ext/carray_histogram.c +152 -94
  80. data/ext/carray_hold.c +2 -1
  81. data/ext/carray_index_classifier.c +63 -96
  82. data/ext/carray_internal.h +385 -3
  83. data/ext/carray_lazy.c +834 -79
  84. data/ext/carray_loop.c +81 -65
  85. data/ext/carray_mask.c +682 -316
  86. data/ext/carray_math_kernel.h +1 -0
  87. data/ext/carray_median_percentile.c +221 -76
  88. data/ext/carray_memory_view.c +254 -248
  89. data/ext/carray_operator.c +1089 -359
  90. data/ext/carray_order.c +133 -69
  91. data/ext/carray_partition.c +102 -56
  92. data/ext/carray_random.c +273 -239
  93. data/ext/carray_scatter.c +308 -65
  94. data/ext/carray_segment.c +257 -0
  95. data/ext/carray_slab.c +170 -70
  96. data/ext/carray_sort.c +215 -218
  97. data/ext/carray_sort_kernel.c +8 -1
  98. data/ext/carray_struct.c +99 -21
  99. data/ext/carray_test.c +167 -64
  100. data/ext/carray_time_text.c +1182 -0
  101. data/ext/carray_utils.c +363 -98
  102. data/ext/extconf.rb +27 -2
  103. data/ext/mk_call_cfunc.rb +0 -2
  104. data/ext/mk_pow5_table.rb +35 -0
  105. data/ext/mkkernel.rb +1076 -846
  106. data/ext/ruby_carray.c +7 -1
  107. data/ext/version.h +4 -4
  108. data/ext/version.rb +4 -2
  109. data/lib/carray/attribute.rb +39 -0
  110. data/lib/carray/autoload_carray.rb +19 -0
  111. data/lib/carray/autoload_method_extension.rb +6 -0
  112. data/lib/carray/axis_group.rb +174 -26
  113. data/lib/carray/basics.rb +91 -25
  114. data/lib/carray/bincount_nd.rb +6 -0
  115. data/lib/carray/block_iterator.rb +118 -48
  116. data/lib/carray/boolean_reduce.rb +21 -29
  117. data/lib/carray/categorical.rb +94 -26
  118. data/lib/carray/categorical_iterator.rb +333 -826
  119. data/lib/carray/complex.rb +36 -29
  120. data/lib/carray/conditional.rb +94 -7
  121. data/lib/carray/const_string.rb +46 -0
  122. data/lib/carray/frame/concat.rb +157 -6
  123. data/lib/carray/frame/convert.rb +13 -7
  124. data/lib/carray/frame/csv_parser.rb +589 -82
  125. data/lib/carray/frame/describe.rb +79 -0
  126. data/lib/carray/frame/frame.rb +124 -25
  127. data/lib/carray/frame/group.rb +189 -63
  128. data/lib/carray/frame/io.rb +412 -72
  129. data/lib/carray/frame/join.rb +134 -46
  130. data/lib/carray/frame/records.rb +37 -16
  131. data/lib/carray/frame/reshape.rb +448 -0
  132. data/lib/carray/frame/sort.rb +26 -2
  133. data/lib/carray/frame/verbs.rb +715 -48
  134. data/lib/carray/frame.rb +2 -0
  135. data/lib/carray/fuse_source.rb +111 -2
  136. data/lib/carray/fusion.rb +229 -38
  137. data/lib/carray/inspect.rb +29 -11
  138. data/lib/carray/iterator.rb +14 -2
  139. data/lib/carray/lazy.rb +54 -30
  140. data/lib/carray/mask_gap_fill.rb +66 -52
  141. data/lib/carray/math.rb +32 -19
  142. data/lib/carray/meld_reduce.rb +40 -7
  143. data/lib/carray/methods/align_addr.rb +8 -4
  144. data/lib/carray/methods/bincount.rb +2 -2
  145. data/lib/carray/methods/bit_string.rb +14 -3
  146. data/lib/carray/methods/composition.rb +16 -10
  147. data/lib/carray/methods/factorize.rb +2 -2
  148. data/lib/carray/methods/gather_nd.rb +13 -2
  149. data/lib/carray/methods/insert_block.rb +9 -14
  150. data/lib/carray/methods/is_in.rb +117 -4
  151. data/lib/carray/methods/join.rb +1 -1
  152. data/lib/carray/methods/locate_addr.rb +177 -36
  153. data/lib/carray/methods/mode.rb +18 -1
  154. data/lib/carray/methods/repeat.rb +1 -20
  155. data/lib/carray/methods/resize.rb +21 -23
  156. data/lib/carray/methods/snap.rb +34 -12
  157. data/lib/carray/methods/unique.rb +11 -2
  158. data/lib/carray/record_order.rb +208 -0
  159. data/lib/carray/runtime.rb +14 -11
  160. data/lib/carray/segment_iterator.rb +990 -0
  161. data/lib/carray/serialize.rb +197 -27
  162. data/lib/carray/slab_iterator.rb +51 -27
  163. data/lib/carray/stack.rb +10 -4
  164. data/lib/carray/string_operation_extension.rb +4 -3
  165. data/lib/carray/struct.rb +142 -51
  166. data/lib/carray/struct_builder.rb +110 -14
  167. data/lib/carray/table.rb +2 -2
  168. data/lib/carray/time.rb +517 -103
  169. data/lib/carray/window_iterator.rb +127 -72
  170. data/lib/carray.rb +2 -2
  171. data/yard-stubs/ca_obj_array.rb +4 -4
  172. data/yard-stubs/ca_obj_bitarray.rb +7 -0
  173. data/yard-stubs/ca_obj_bitfield.rb +23 -12
  174. data/yard-stubs/ca_obj_fake.rb +1 -1
  175. data/yard-stubs/ca_obj_field.rb +1 -1
  176. data/yard-stubs/ca_obj_refer.rb +3 -3
  177. data/yard-stubs/ca_obj_roll.rb +0 -1
  178. data/yard-stubs/ca_obj_stride.rb +10 -7
  179. data/yard-stubs/ca_obj_transpose.rb +1 -1
  180. data/yard-stubs/ca_obj_window.rb +1 -2
  181. data/yard-stubs/carray_access.rb +5 -4
  182. data/yard-stubs/carray_broadcast.rb +1 -1
  183. data/yard-stubs/carray_cast.rb +78 -43
  184. data/yard-stubs/carray_conversion.rb +13 -1
  185. data/yard-stubs/carray_copy.rb +8 -3
  186. data/yard-stubs/carray_core.rb +3 -3
  187. data/yard-stubs/carray_count.rb +9 -0
  188. data/yard-stubs/carray_lazy.rb +5 -1
  189. data/yard-stubs/carray_loop.rb +4 -1
  190. data/yard-stubs/carray_mask.rb +34 -10
  191. data/yard-stubs/carray_math.rb +25 -13
  192. data/yard-stubs/carray_median_percentile.rb +10 -6
  193. data/yard-stubs/carray_order.rb +34 -13
  194. data/yard-stubs/carray_scatter.rb +10 -3
  195. data/yard-stubs/carray_segment.rb +72 -0
  196. data/yard-stubs/carray_slab.rb +2 -1
  197. data/yard-stubs/carray_sort.rb +43 -3
  198. data/yard-stubs/carray_test.rb +10 -0
  199. data/yard-stubs/carray_utils.rb +0 -26
  200. metadata +11 -19
  201. data/ext/ca_for_each_element.h +0 -239
  202. data/ext/carray_build_flags.h +0 -3
  203. data/ext/carray_config.h +0 -28
  204. data/ext/carray_kernels_bincmp.c +0 -4446
  205. data/ext/carray_kernels_binop.c +0 -11001
  206. data/ext/carray_kernels_init.c +0 -1131
  207. data/ext/carray_kernels_map.c +0 -3467
  208. data/ext/carray_kernels_moncmp.c +0 -2097
  209. data/ext/carray_kernels_monop.c +0 -18313
  210. data/ext/carray_kernels_reduce_aggregate.c +0 -26005
  211. data/ext/carray_kernels_reduce_boolean.c +0 -330
  212. data/ext/carray_kernels_reduce_cumulative.c +0 -14862
  213. data/ext/carray_kernels_reduce_extreme.c +0 -17494
  214. data/ext/carray_kernels_reduce_variance.c +0 -3910
  215. data/ext/carray_kernels_scan.c +0 -3693
  216. data/ext/carray_kernels_search.c +0 -32218
  217. data/ext/carray_kernels_sort.c +0 -10626
  218. data/ext/carray_kernels_triop.c +0 -1392
data/ext/extconf.rb CHANGED
@@ -107,6 +107,17 @@ if enable_config("dev-build", false)
107
107
  $stderr.puts "carray: CARRAY_DEV_BUILD enabled (smoke surface available)"
108
108
  end
109
109
 
110
+ # --- signed overflow wraps (`-fwrapv`)
111
+ #
112
+ # An integer kernel that overflows answers modulo 2^N. C leaves signed
113
+ # overflow undefined, and a compiler that sees several operations at once
114
+ # rewrites them as if it could not happen -- `(x + 1) > x` becomes true --
115
+ # which carray-jit's fused expressions, built with these flags, would show.
116
+ if try_compile("int main(void) { return 0; }", "-fwrapv")
117
+ $CFLAGS += " -fwrapv"
118
+ $carray_build_flags << "-fwrapv"
119
+ end
120
+
110
121
  # --- probe for `-fopenmp-simd` (SL.1.0, PROPOSAL_REDUCTION_SIMD_LICENSE)
111
122
  #
112
123
  # `-fopenmp-simd` enables `#pragma omp simd` (and its `reduction(+:acc)`
@@ -257,6 +268,8 @@ $INSTALLFILES << ['carray_config.h', '$(archdir)']
257
268
  # (b) Not in the closure, but written for ext authors to include
258
269
  # explicitly. ca_obj_face.h is the Face authoring surface: an
259
270
  # external gem defines its own Face by including it directly.
271
+ # ca_for_buffer.h is whole-buffer delivery to an external library;
272
+ # it pulls ca_sweep_engine.h.
260
273
  #
261
274
  # A header that is neither is internal, however useful it looks in-tree.
262
275
  # ca_op_powi.h (op_powi_<type> helpers for the generated kernels) and
@@ -272,11 +285,14 @@ $INSTALLFILES << ['carray_config.h', '$(archdir)']
272
285
  ca_axis_descriptor.h
273
286
  ca_monop_dispatch.h
274
287
  ca_binop_dispatch.h
288
+ ca_triop_dispatch.h
275
289
  ca_bincmp_dispatch.h
276
290
  ca_moncmp_dispatch.h
277
291
  ca_kernel_iterator.h
278
292
  ca_iter_substrate.h
279
293
  ca_obj_face.h
294
+ ca_for_buffer.h
295
+ ca_sweep_engine.h
280
296
  ].each do |h|
281
297
  $INSTALLFILES << [h, '$(archdir)']
282
298
  end
@@ -324,11 +340,14 @@ end
324
340
  # compiling static sources, which is one pass with no race.
325
341
  #
326
342
  # carray_kernels.stamp is the mtime witness:
327
- # - here: regenerate and touch it when mkkernel.rb is newer
343
+ # - here: regenerate and touch it when mkkernel.rb or ca_op_powi.h (whose
344
+ # helper text the generator copies) is newer
328
345
  # - in the Rakefile (rake build_ext): the same stamp triggers need_extconf
329
346
  CARRAY_KERNELS_STAMP = "carray_kernels.stamp"
330
347
  if ( not File.exist?(CARRAY_KERNELS_STAMP) ) or
331
- File.stat("mkkernel.rb").mtime > File.stat(CARRAY_KERNELS_STAMP).mtime
348
+ %w[mkkernel.rb ca_op_powi.h].any? { |f|
349
+ File.stat(f).mtime > File.stat(CARRAY_KERNELS_STAMP).mtime
350
+ }
332
351
  # passing "." makes the generator write the split files into ext/
333
352
  unless system("ruby mkkernel.rb .")
334
353
  raise "mkkernel.rb failed to generate split kernel files"
@@ -346,6 +365,12 @@ if Dir.glob("carray_kernels_*.c").empty?
346
365
  raise "no carray_kernels_*.c files found after mkkernel.rb; check the generator"
347
366
  end
348
367
 
368
+ # The 5^q table the decimal-to-double reader (carray_cast.c) multiplies by.
369
+ if ( not File.exist?("ca_pow5_table.h") ) or
370
+ File.stat("mk_pow5_table.rb").mtime > File.stat("ca_pow5_table.h").mtime
371
+ system("ruby mk_pow5_table.rb")
372
+ end
373
+
349
374
  if ( not File.exist?("carray_call_cfunc.c") ) or
350
375
  ( not File.exist?("carray_call_cfunc.h") ) or
351
376
  File.stat("mk_call_cfunc.rb").mtime > File.stat("carray_call_cfunc.c").mtime
data/ext/mk_call_cfunc.rb CHANGED
@@ -209,7 +209,6 @@ def emit_raw_common(n, r:)
209
209
  state.stride = stride;
210
210
  state.owned_buf = owned_buf;
211
211
  state.attached = attached;
212
- state.no_mask = 0;
213
212
  state.src_label = "#{name}";
214
213
 
215
214
  ca_sweep_acquire(&state);
@@ -412,7 +411,6 @@ def emit_slab_body(n, name, walker, userdata)
412
411
  state.stride = stride;
413
412
  state.owned_buf = owned_buf;
414
413
  state.attached = attached;
415
- state.no_mask = 0;
416
414
  state.src_label = "#{name}";
417
415
 
418
416
  ca_sweep_acquire_chunked(&state);
@@ -0,0 +1,35 @@
1
+ # Generates ca_pow5_table.h: for each decimal exponent q in [-342, 308], a
2
+ # 128-bit approximation of 5^q scaled by a power of two into [2^127, 2^128),
3
+ # as the Eisel-Lemire decimal-to-binary64 conversion takes it.
4
+ #
5
+ # q >= 0: floor(5^q * 2^k) (exact while 5^q < 2^128, q <= 55)
6
+ # -27 <= q < 0: ceil(2^k / 5^-q)
7
+ # q < -27: floor(2^k / 5^-q)
8
+ # with k chosen so the value has exactly 128 bits.
9
+
10
+ Q_MIN, Q_MAX = -342, 308
11
+ rows = (Q_MIN..Q_MAX).map do |q|
12
+ if q >= 0
13
+ v = 5**q
14
+ v <<= 128 - v.bit_length if v.bit_length < 128
15
+ v >>= v.bit_length - 128 if v.bit_length > 128
16
+ else
17
+ p5 = 5**-q
18
+ k = 127 + p5.bit_length
19
+ v, r = (1 << k).divmod(p5)
20
+ v += 1 if q >= -27 && r != 0
21
+ v >>= v.bit_length - 128 if v.bit_length > 128
22
+ v <<= 128 - v.bit_length if v.bit_length < 128
23
+ end
24
+ raise "bad width at #{q}" unless v.bit_length == 128
25
+ [v >> 64, v & ((1 << 64) - 1)]
26
+ end
27
+
28
+ File.open(File.join(__dir__, "ca_pow5_table.h"), "w") do |f|
29
+ f.puts "/* Generated by mk_pow5_table.rb -- do not edit. */"
30
+ f.puts "#define CA_POW5_Q_MIN (#{Q_MIN})"
31
+ f.puts "#define CA_POW5_Q_MAX #{Q_MAX}"
32
+ f.puts "static const uint64_t ca_pow5_table[#{rows.size * 2}] = {"
33
+ rows.each { |hi, lo| f.puts format(" 0x%016xULL, 0x%016xULL,", hi, lo) }
34
+ f.puts "};"
35
+ end