carray 3.0.3 → 3.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +469 -0
  3. data/README.md +3 -2
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_array_pool.c +25 -1
  6. data/ext/ca_axis_descriptor.h +27 -24
  7. data/ext/ca_axis_dispatch.c +14 -26
  8. data/ext/ca_axis_group.c +27 -27
  9. data/ext/ca_binop_dispatch.c +1 -29
  10. data/ext/ca_binop_dispatch.h +0 -1
  11. data/ext/ca_categorical_iterator.c +231 -153
  12. data/ext/ca_compare.c +9 -20
  13. data/ext/ca_composite_dispatch.c +9 -9
  14. data/ext/ca_group_iter.c +16 -17
  15. data/ext/ca_kernel_iterator.c +1240 -2000
  16. data/ext/ca_kernel_iterator.h +502 -519
  17. data/ext/ca_monop_dispatch.c +82 -59
  18. data/ext/ca_monop_dispatch.h +1 -0
  19. data/ext/ca_obj_array.c +152 -45
  20. data/ext/ca_obj_bincmp.c +25 -21
  21. data/ext/ca_obj_binop.c +16 -26
  22. data/ext/ca_obj_bitarray.c +8 -3
  23. data/ext/ca_obj_bitfield.c +63 -49
  24. data/ext/ca_obj_block.c +18 -9
  25. data/ext/ca_obj_byte_swap.c +7 -7
  26. data/ext/ca_obj_const_string.c +188 -22
  27. data/ext/ca_obj_face.c +280 -58
  28. data/ext/ca_obj_face.h +19 -0
  29. data/ext/ca_obj_fake.c +15 -29
  30. data/ext/ca_obj_farray.c +6 -3
  31. data/ext/ca_obj_field.c +30 -8
  32. data/ext/ca_obj_fixlen_string.c +2 -3
  33. data/ext/ca_obj_grid.c +455 -134
  34. data/ext/ca_obj_meld.c +172 -34
  35. data/ext/ca_obj_moncmp.c +23 -8
  36. data/ext/ca_obj_monop.c +113 -67
  37. data/ext/ca_obj_object.c +210 -52
  38. data/ext/ca_obj_record.c +15 -7
  39. data/ext/ca_obj_reduce.c +17 -8
  40. data/ext/ca_obj_refer.c +42 -35
  41. data/ext/ca_obj_remap.c +30 -9
  42. data/ext/ca_obj_repeat.c +9 -10
  43. data/ext/ca_obj_roll.c +31 -10
  44. data/ext/ca_obj_select.c +48 -14
  45. data/ext/ca_obj_select_axis.c +15 -47
  46. data/ext/ca_obj_shift.c +41 -7
  47. data/ext/ca_obj_stack.c +180 -34
  48. data/ext/ca_obj_stride.c +190 -134
  49. data/ext/ca_obj_string.c +80 -4
  50. data/ext/ca_obj_tile.c +32 -11
  51. data/ext/ca_obj_time.c +22 -6
  52. data/ext/ca_obj_timedelta.c +22 -6
  53. data/ext/ca_obj_transpose.c +13 -7
  54. data/ext/ca_obj_triop.c +26 -32
  55. data/ext/ca_obj_window.c +267 -74
  56. data/ext/ca_op_ipower.c +4 -4
  57. data/ext/ca_sort_kernels.h +6 -3
  58. data/ext/ca_sweep_engine.c +2 -2
  59. data/ext/caframe_csv_reader.c +80 -1
  60. data/ext/carray.h +214 -75
  61. data/ext/carray_access.c +200 -99
  62. data/ext/carray_address_basis.c +343 -87
  63. data/ext/carray_attribute.c +113 -6
  64. data/ext/carray_bincount.c +10 -8
  65. data/ext/carray_broadcast.c +13 -8
  66. data/ext/carray_build_flags.h +3 -0
  67. data/ext/carray_call_cfunc.c +108 -0
  68. data/ext/carray_cast.c +375 -96
  69. data/ext/carray_cast_func.rb +90 -20
  70. data/ext/carray_class.c +4 -3
  71. data/ext/carray_config.h +28 -0
  72. data/ext/carray_conversion.c +18 -9
  73. data/ext/carray_copy.c +51 -14
  74. data/ext/carray_core.c +150 -55
  75. data/ext/carray_count.c +56 -20
  76. data/ext/carray_element.c +81 -33
  77. data/ext/carray_factorize.c +138 -84
  78. data/ext/carray_generate.c +199 -52
  79. data/ext/carray_histogram.c +63 -18
  80. data/ext/carray_hold.c +12 -15
  81. data/ext/carray_index_classifier.c +388 -176
  82. data/ext/carray_internal.h +348 -29
  83. data/ext/carray_kernels_bincmp.c +4453 -0
  84. data/ext/carray_kernels_binop.c +11368 -0
  85. data/ext/carray_kernels_init.c +1734 -0
  86. data/ext/carray_kernels_moncmp.c +2104 -0
  87. data/ext/carray_kernels_monop.c +18369 -0
  88. data/ext/carray_kernels_reduce_aggregate.c +23818 -0
  89. data/ext/carray_kernels_reduce_boolean.c +406 -0
  90. data/ext/carray_kernels_reduce_cumulative.c +10086 -0
  91. data/ext/carray_kernels_reduce_extreme.c +17634 -0
  92. data/ext/carray_kernels_reduce_variance.c +3849 -0
  93. data/ext/carray_kernels_scan.c +4302 -0
  94. data/ext/carray_kernels_search.c +33829 -0
  95. data/ext/carray_kernels_sort.c +10897 -0
  96. data/ext/carray_kernels_triop.c +1864 -0
  97. data/ext/carray_key_windows.c +220 -0
  98. data/ext/carray_lazy.c +170 -111
  99. data/ext/carray_mask.c +337 -94
  100. data/ext/carray_median_percentile.c +82 -28
  101. data/ext/carray_memory_view.c +669 -561
  102. data/ext/carray_operator.c +129 -52
  103. data/ext/carray_order.c +604 -77
  104. data/ext/carray_partition.c +28 -27
  105. data/ext/carray_random.c +85 -31
  106. data/ext/carray_round.c +273 -0
  107. data/ext/carray_scatter.c +87 -45
  108. data/ext/carray_segment.c +3 -3
  109. data/ext/carray_slab.c +123 -37
  110. data/ext/carray_sort.c +259 -51
  111. data/ext/carray_sort_kernel.c +5 -7
  112. data/ext/carray_struct.c +54 -61
  113. data/ext/carray_test.c +26 -5
  114. data/ext/carray_time_text.c +4 -2
  115. data/ext/carray_undef.c +4 -0
  116. data/ext/carray_utils.c +309 -66
  117. data/ext/mk_call_cfunc.rb +19 -2
  118. data/ext/mkkernel.rb +403 -350
  119. data/ext/ruby_carray.c +10 -4
  120. data/ext/version.h +4 -4
  121. data/lib/carray/attribute.rb +139 -22
  122. data/lib/carray/autoload_carray.rb +8 -31
  123. data/lib/carray/autoload_method_extension.rb +14 -5
  124. data/lib/carray/axis_group.rb +255 -106
  125. data/lib/carray/basics.rb +105 -17
  126. data/lib/carray/bincount_nd.rb +12 -2
  127. data/lib/carray/block_iterator.rb +126 -23
  128. data/lib/carray/boolean_reduce.rb +17 -17
  129. data/lib/carray/categorical.rb +195 -59
  130. data/lib/carray/categorical_iterator.rb +132 -29
  131. data/lib/carray/complex.rb +5 -5
  132. data/lib/carray/conditional.rb +67 -18
  133. data/lib/carray/const_string.rb +24 -5
  134. data/lib/carray/construct.rb +15 -11
  135. data/lib/carray/data_type_extension.rb +9 -1
  136. data/lib/carray/distance_limit.rb +77 -0
  137. data/lib/carray/fixlen_string.rb +48 -3
  138. data/lib/carray/frame/column_verbs.rb +71 -0
  139. data/lib/carray/frame/concat.rb +17 -3
  140. data/lib/carray/frame/convert.rb +69 -0
  141. data/lib/carray/frame/csv_parser.rb +111 -17
  142. data/lib/carray/frame/describe.rb +3 -2
  143. data/lib/carray/frame/frame.rb +252 -28
  144. data/lib/carray/frame/group.rb +603 -101
  145. data/lib/carray/frame/io.rb +117 -28
  146. data/lib/carray/frame/join.rb +312 -51
  147. data/lib/carray/frame/records.rb +83 -7
  148. data/lib/carray/frame/reshape.rb +42 -35
  149. data/lib/carray/frame/sort.rb +7 -19
  150. data/lib/carray/frame/verbs.rb +188 -48
  151. data/lib/carray/frame/windows.rb +117 -0
  152. data/lib/carray/frame.rb +2 -0
  153. data/lib/carray/fuse_source.rb +42 -6
  154. data/lib/carray/fusion.rb +40 -16
  155. data/lib/carray/histogram.rb +6 -1
  156. data/lib/carray/inspect.rb +31 -8
  157. data/lib/carray/iterator.rb +85 -7
  158. data/lib/carray/key_windows.rb +620 -0
  159. data/lib/carray/lazy.rb +217 -183
  160. data/lib/carray/mask_gap_fill.rb +63 -57
  161. data/lib/carray/meld_reduce.rb +12 -14
  162. data/lib/carray/methods/bin.rb +26 -18
  163. data/lib/carray/methods/bincount.rb +16 -0
  164. data/lib/carray/methods/bit_string.rb +1 -1
  165. data/lib/carray/methods/choose.rb +16 -3
  166. data/lib/carray/methods/composition.rb +3 -1
  167. data/lib/carray/methods/gather_nd.rb +14 -5
  168. data/lib/carray/methods/insert_block.rb +2 -2
  169. data/lib/carray/methods/is_between.rb +59 -0
  170. data/lib/carray/methods/is_in.rb +61 -37
  171. data/lib/carray/methods/join.rb +1 -6
  172. data/lib/carray/methods/lag.rb +49 -0
  173. data/lib/carray/methods/locate_addr.rb +47 -51
  174. data/lib/carray/methods/mode.rb +1 -1
  175. data/lib/carray/methods/nunique.rb +11 -6
  176. data/lib/carray/methods/resize.rb +3 -3
  177. data/lib/carray/methods/value_counts.rb +4 -11
  178. data/lib/carray/record_order.rb +3 -3
  179. data/lib/carray/rng.rb +3 -0
  180. data/lib/carray/runtime.rb +53 -8
  181. data/lib/carray/segment_iterator.rb +323 -88
  182. data/lib/carray/serialize.rb +45 -14
  183. data/lib/carray/slab_iterator.rb +50 -0
  184. data/lib/carray/stack.rb +15 -12
  185. data/lib/carray/string.rb +113 -5
  186. data/lib/carray/string_operation_extension.rb +21 -6
  187. data/lib/carray/struct.rb +9 -21
  188. data/lib/carray/struct_builder.rb +24 -11
  189. data/lib/carray/time.rb +819 -158
  190. data/lib/carray/window_iterator.rb +122 -16
  191. data/lib/carray.rb +7 -0
  192. data/yard-stubs/ca_obj_bitfield.rb +1 -1
  193. data/yard-stubs/ca_obj_field.rb +4 -0
  194. data/yard-stubs/ca_obj_grid.rb +21 -10
  195. data/yard-stubs/ca_obj_refer.rb +2 -1
  196. data/yard-stubs/ca_obj_roll.rb +2 -2
  197. data/yard-stubs/ca_obj_shift.rb +5 -1
  198. data/yard-stubs/ca_obj_tile.rb +1 -1
  199. data/yard-stubs/ca_obj_transpose.rb +5 -4
  200. data/yard-stubs/ca_obj_window.rb +8 -3
  201. data/yard-stubs/carray_access.rb +6 -1
  202. data/yard-stubs/carray_attribute.rb +3 -6
  203. data/yard-stubs/carray_cast.rb +12 -0
  204. data/yard-stubs/carray_conversion.rb +7 -2
  205. data/yard-stubs/carray_copy.rb +4 -2
  206. data/yard-stubs/carray_count.rb +4 -4
  207. data/yard-stubs/carray_element.rb +17 -10
  208. data/yard-stubs/carray_generate.rb +21 -1
  209. data/yard-stubs/carray_mask.rb +32 -3
  210. data/yard-stubs/carray_math.rb +41 -5
  211. data/yard-stubs/carray_median_percentile.rb +5 -2
  212. data/yard-stubs/carray_memory_view.rb +40 -23
  213. data/yard-stubs/carray_order.rb +129 -22
  214. data/yard-stubs/carray_random.rb +13 -4
  215. data/yard-stubs/carray_scatter.rb +17 -10
  216. data/yard-stubs/carray_slab.rb +9 -2
  217. data/yard-stubs/carray_sort.rb +6 -5
  218. data/yard-stubs/carray_test.rb +5 -3
  219. metadata +25 -3
  220. data/lib/carray/arrow_tensor.rb +0 -401
  221. data/lib/carray/table.rb +0 -108
data/ext/carray_sort.c CHANGED
@@ -103,16 +103,207 @@ sort_addr_merge (struct sort_addr_key *a, struct sort_addr_key *aux,
103
103
  ca_size_t n, int (*cmp)(const void *, const void *))
104
104
  CA_SORT_MERGE_BY_CMP(struct sort_addr_key)
105
105
 
106
+ /* -------------------------------------------------------------------
107
+
108
+ Multi-key sort by refinement (MSD), for keys of the numeric data types.
109
+
110
+ The first key is sorted whole by the typed pair kernel
111
+ (ca_sort_quick_pair_*); then only the runs of rows that tie on it are
112
+ sorted by the second key, and so on, until no run is left or the keys
113
+ run out. The answer is the one sort_addr_cmp gives:
114
+
115
+ - a pair sorts by (value, row), so rows that tie on every key stay in
116
+ row order (the stable tie-break of sort_addr_cmp);
117
+ - each run is split into numbers, NaN and masked cells, in the order
118
+ sort_addr_cmp puts them (numbers, NaN, masked for masked_last;
119
+ masked, numbers, NaN otherwise). The NaN and the masked cells of a
120
+ run tie among themselves (CA_FLOAT_ORDER_CMP / sort_addr_cmp), so
121
+ each is a run for the next key. The split is a stable one done in
122
+ the load, not ca_partition_nan_pair_*, which allocates on every call
123
+ and is called here once per run;
124
+ - -0.0 and 0.0 tie, by == as by CA_FLOAT_ORDER_CMP.
125
+
126
+ A run is (lo, n): the rows perm[lo..lo+n). Within a run the rows are
127
+ in row order, since they tie on every key before it, so the stable
128
+ split keeps them so.
129
+
130
+ ------------------------------------------------------------------- */
131
+
132
+ typedef struct {
133
+ ca_size_t lo;
134
+ ca_size_t n;
135
+ } msd_run_t;
136
+
137
+ typedef struct {
138
+ msd_run_t *runs;
139
+ ca_size_t count;
140
+ } msd_runs_t;
141
+
142
+ static void
143
+ msd_add_run (msd_runs_t *out, ca_size_t lo, ca_size_t n)
144
+ {
145
+ if ( out && n > 1 ) {
146
+ out->runs[out->count].lo = lo;
147
+ out->runs[out->count].n = n;
148
+ out->count++;
149
+ }
150
+ }
151
+
152
+ /* Sort each run of +in+ by +key+, writing the new order into perm, and
153
+ add the runs that still tie to +out+ (NULL for the last key). */
154
+ #define DEFINE_MSD_REFINE(TYPE, SUFFIX, IS_FLOAT) \
155
+ static void \
156
+ msd_refine_##SUFFIX (CArray *key, ca_size_t *perm, void *vbuf, \
157
+ msd_runs_t *in, msd_runs_t *out, int masked_last) \
158
+ { \
159
+ const TYPE *ptr = (const TYPE *) key->ptr; \
160
+ const boolean8_t *m = ( key->mask ) ? (boolean8_t *) key->mask->ptr : NULL; \
161
+ ca_pair_##SUFFIX *buf = (ca_pair_##SUFFIX *) vbuf; \
162
+ ca_size_t r; \
163
+ \
164
+ for (r=0; r<in->count; r++) { \
165
+ ca_size_t lo = in->runs[r].lo; \
166
+ ca_size_t n = in->runs[r].n; \
167
+ ca_size_t *p = perm + lo; \
168
+ ca_size_t n_num = 0, n_nan = 0, n_mask = 0; \
169
+ ca_size_t o_num, o_nan, o_mask; \
170
+ ca_size_t k, s; \
171
+ \
172
+ for (k=0; k<n; k++) { \
173
+ ca_size_t row = p[k]; \
174
+ if ( m && m[row] ) n_mask++; \
175
+ else if ( IS_FLOAT && isnan((double) ptr[row]) ) n_nan++; \
176
+ else n_num++; \
177
+ } \
178
+ o_mask = masked_last ? n_num + n_nan : 0; \
179
+ o_num = masked_last ? 0 : n_mask; \
180
+ o_nan = o_num + n_num; \
181
+ { \
182
+ ca_size_t i_num = o_num, i_nan = o_nan, i_mask = o_mask; \
183
+ for (k=0; k<n; k++) { \
184
+ ca_size_t row = p[k]; \
185
+ TYPE v = ptr[row]; \
186
+ ca_size_t at; \
187
+ if ( m && m[row] ) at = i_mask++; \
188
+ else if ( IS_FLOAT && isnan((double) v) ) at = i_nan++; \
189
+ else at = i_num++; \
190
+ buf[at].v = v; \
191
+ buf[at].i = row; \
192
+ } \
193
+ } \
194
+ ca_sort_quick_pair_##SUFFIX(buf + o_num, n_num); \
195
+ for (k=0; k<n; k++) { \
196
+ p[k] = buf[k].i; \
197
+ } \
198
+ if ( ! out ) continue; \
199
+ for (s=0, k=1; k<=n_num; k++) { \
200
+ if ( k == n_num || buf[o_num + k].v != buf[o_num + s].v ) { \
201
+ msd_add_run(out, lo + o_num + s, k - s); \
202
+ s = k; \
203
+ } \
204
+ } \
205
+ msd_add_run(out, lo + o_nan, n_nan); \
206
+ msd_add_run(out, lo + o_mask, n_mask); \
207
+ } \
208
+ }
209
+
210
+ DEFINE_MSD_REFINE(int8_t, i8, 0)
211
+ DEFINE_MSD_REFINE(uint8_t, u8, 0)
212
+ DEFINE_MSD_REFINE(int16_t, i16, 0)
213
+ DEFINE_MSD_REFINE(uint16_t, u16, 0)
214
+ DEFINE_MSD_REFINE(int32_t, i32, 0)
215
+ DEFINE_MSD_REFINE(uint32_t, u32, 0)
216
+ DEFINE_MSD_REFINE(int64_t, i64, 0)
217
+ DEFINE_MSD_REFINE(uint64_t, u64, 0)
218
+ DEFINE_MSD_REFINE(float32_t, f32, 1)
219
+ DEFINE_MSD_REFINE(double, f64, 1)
220
+
221
+ /* Whether every key has a typed pair kernel, so that MSD can sort them. */
222
+ static int
223
+ msd_is_sortable (struct cmp_base *base)
224
+ {
225
+ int j;
226
+ for (j=0; j<base->n; j++) {
227
+ switch ( base->ca[j]->data_type ) {
228
+ case CA_BOOLEAN: case CA_INT8: case CA_UINT8:
229
+ case CA_INT16: case CA_UINT16:
230
+ case CA_INT32: case CA_UINT32:
231
+ case CA_INT64: case CA_UINT64:
232
+ case CA_FLOAT32: case CA_FLOAT64:
233
+ break;
234
+ default:
235
+ return 0;
236
+ }
237
+ }
238
+ return 1;
239
+ }
240
+
241
+ typedef struct {
242
+ struct cmp_base *base;
243
+ ca_size_t *perm; /* elements rows, the answer */
244
+ void *buf; /* elements pairs of 16 bytes */
245
+ msd_run_t *runs[2]; /* elements / 2 + 1 runs each */
246
+ ca_size_t elements;
247
+ } msd_sort_t;
248
+
249
+ static void
250
+ msd_sort (msd_sort_t *s)
251
+ {
252
+ struct cmp_base *base = s->base;
253
+ msd_runs_t a, b, *in = &a, *out = &b;
254
+ ca_size_t i;
255
+ int j;
256
+
257
+ for (i=0; i<s->elements; i++) {
258
+ s->perm[i] = i;
259
+ }
260
+ a.runs = s->runs[0]; a.count = 0;
261
+ b.runs = s->runs[1]; b.count = 0;
262
+ msd_add_run(in, 0, s->elements);
263
+
264
+ for (j=0; j<base->n && in->count > 0; j++) {
265
+ CArray *key = base->ca[j];
266
+ msd_runs_t *next = ( j < base->n - 1 ) ? out : NULL;
267
+ msd_runs_t *t;
268
+ if ( next ) next->count = 0;
269
+ switch ( key->data_type ) {
270
+ case CA_BOOLEAN:
271
+ case CA_INT8: msd_refine_i8 (key, s->perm, s->buf, in, next, base->masked_last); break;
272
+ case CA_UINT8: msd_refine_u8 (key, s->perm, s->buf, in, next, base->masked_last); break;
273
+ case CA_INT16: msd_refine_i16(key, s->perm, s->buf, in, next, base->masked_last); break;
274
+ case CA_UINT16: msd_refine_u16(key, s->perm, s->buf, in, next, base->masked_last); break;
275
+ case CA_INT32: msd_refine_i32(key, s->perm, s->buf, in, next, base->masked_last); break;
276
+ case CA_UINT32: msd_refine_u32(key, s->perm, s->buf, in, next, base->masked_last); break;
277
+ case CA_INT64: msd_refine_i64(key, s->perm, s->buf, in, next, base->masked_last); break;
278
+ case CA_UINT64: msd_refine_u64(key, s->perm, s->buf, in, next, base->masked_last); break;
279
+ case CA_FLOAT32: msd_refine_f32(key, s->perm, s->buf, in, next, base->masked_last); break;
280
+ case CA_FLOAT64: msd_refine_f64(key, s->perm, s->buf, in, next, base->masked_last); break;
281
+ default:
282
+ rb_raise(rb_eRuntimeError, "%s: [BUG] data type %d has no pair kernel",
283
+ ca_calling_method_name(), (int) key->data_type);
284
+ }
285
+ if ( ! next ) break;
286
+ t = in; in = out; out = t;
287
+ }
288
+ }
289
+
290
+ /* ----------------------------------------------------------------- */
291
+
106
292
  typedef struct {
107
293
  struct sort_addr_key *data;
108
294
  struct sort_addr_key *aux; /* NULL: no object key, libc sorts */
109
295
  ca_size_t elements;
296
+ msd_sort_t *msd; /* non-NULL: every key is numeric, sort by MSD */
110
297
  } sort_addr_run_t;
111
298
 
112
299
  static VALUE
113
300
  sort_addr_run (VALUE arg)
114
301
  {
115
302
  sort_addr_run_t *r = (sort_addr_run_t *) arg;
303
+ if ( r->msd ) {
304
+ msd_sort(r->msd);
305
+ return Qnil;
306
+ }
116
307
  if ( r->aux ) {
117
308
  sort_addr_merge(r->data, r->aux, r->elements,
118
309
  (int (*)(const void*,const void*)) sort_addr_cmp);
@@ -145,8 +336,9 @@ rb_ca_s_sort_addr_kw (int argc, VALUE *argv, VALUE ropt, VALUE self)
145
336
  CArray *co;
146
337
  struct cmp_base base_s, *base = &base_s;
147
338
  struct sort_addr_key *data;
148
- volatile VALUE hlist = 0, hdata = 0, haux = 0, hmodes = 0;
339
+ volatile VALUE hlist = 0, hdata = 0, haux = 0, hmodes = 0, hruns0 = 0, hruns1 = 0;
149
340
  sort_addr_run_t run;
341
+ msd_sort_t msd;
150
342
  char *modes;
151
343
  int has_object = 0;
152
344
  ca_size_t elements;
@@ -172,31 +364,10 @@ rb_ca_s_sort_addr_kw (int argc, VALUE *argv, VALUE ropt, VALUE self)
172
364
  if ( elements != NUM2SIZE(rb_ca_elements(argv[j])) ) {
173
365
  rb_raise(rb_eArgError, "elements mismatch");
174
366
  }
175
- /* Face gate: an ORDERABLE Face sorts by its storage, so it descends to
176
- it (attach / comparator / template output then run on plain storage,
177
- which also avoids a face-lifted NULL-ptr index output). A fixlen
178
- storage that is not ORDERABLE (CAConstString's offsets, a record's
179
- bytes) has no order in its bytes; the Face's own order stands in for
180
- it, as a dense rank, so ties stay ties for the next key. A Face that
181
- has no order (CARecord without order_by:) raises there. Any other
182
- storage raises here. */
183
367
  TypedData_Get_Struct(argv[j], CArray, &carray_data_type, ca_j);
184
368
  if ( ca_is_face(ca_j) ) {
185
- int orderable = ca_test_flag(ca_j, CA_FLAG_FACE_ORDERABLE_STORAGE);
186
- VALUE stripped = rb_ca_strip_face_value(argv[j]);
187
- CArray *sc;
188
- TypedData_Get_Struct(stripped, CArray, &carray_data_type, sc);
189
- if ( sc->data_type == CA_FIXLEN && ! orderable ) {
190
- VALUE opts = rb_hash_new();
191
- rb_hash_aset(opts, ID2SYM(rb_intern("method")), ID2SYM(rb_intern("dense")));
192
- argv[j] = rb_funcallv_kw(argv[j], rb_intern("order"), 1, &opts,
193
- RB_PASS_KEYWORDS);
194
- continue;
195
- }
196
- if ( ! orderable ) {
197
- ca_face_refuse_order("sort_addr", argv[j], "order");
198
- }
199
- argv[j] = stripped;
369
+ argv[j] = ca_order_key(argv[j], "sort_addr");
370
+ continue;
200
371
  }
201
372
  {
202
373
  CArray *k;
@@ -224,26 +395,42 @@ rb_ca_s_sort_addr_kw (int argc, VALUE *argv, VALUE ropt, VALUE self)
224
395
  }
225
396
  modes[argc] = '\0';
226
397
 
227
- data = ALLOCV_N(struct sort_addr_key, hdata, elements);
228
- for (i=0; i<elements; i++) {
229
- data[i].i = i;
230
- data[i].base = base;
231
- }
232
-
233
- run.data = data;
234
- run.aux = has_object ? ALLOCV_N(struct sort_addr_key, haux, elements) : NULL;
235
- run.elements = elements;
236
- ca_attach_window(argc, base->ca, modes, sort_addr_run, (VALUE) &run);
237
-
238
398
  out = rb_ca_template_with_type(argv[0], INT2NUM(CA_SIZE), INT2NUM(0));
239
399
  TypedData_Get_Struct(out, CArray, &carray_data_type, co);
240
400
  q = (ca_size_t *) co->ptr;
241
-
242
- for (i=0; i<elements; i++) {
243
- *q = data[i].i;
244
- q++;
401
+
402
+ run.data = NULL;
403
+ run.aux = NULL;
404
+ run.elements = elements;
405
+ run.msd = NULL;
406
+
407
+ if ( msd_is_sortable(base) ) {
408
+ ca_size_t nruns = elements / 2 + 1;
409
+ msd.base = base;
410
+ msd.perm = q;
411
+ msd.buf = ALLOCV(hdata, (size_t) elements * sizeof(ca_pair_f64));
412
+ msd.runs[0] = ALLOCV_N(msd_run_t, hruns0, nruns);
413
+ msd.runs[1] = ( argc > 1 ) ? ALLOCV_N(msd_run_t, hruns1, nruns) : NULL;
414
+ msd.elements = elements;
415
+ run.msd = &msd;
416
+ ca_attach_window(argc, base->ca, modes, sort_addr_run, (VALUE) &run);
417
+ }
418
+ else {
419
+ data = ALLOCV_N(struct sort_addr_key, hdata, elements);
420
+ for (i=0; i<elements; i++) {
421
+ data[i].i = i;
422
+ data[i].base = base;
423
+ }
424
+ run.data = data;
425
+ run.aux = has_object ? ALLOCV_N(struct sort_addr_key, haux, elements) : NULL;
426
+ ca_attach_window(argc, base->ca, modes, sort_addr_run, (VALUE) &run);
427
+ for (i=0; i<elements; i++) {
428
+ q[i] = data[i].i;
429
+ }
245
430
  }
246
431
 
432
+ ALLOCV_END(hruns1);
433
+ ALLOCV_END(hruns0);
247
434
  ALLOCV_END(haux);
248
435
  ALLOCV_END(hdata);
249
436
  ALLOCV_END(hmodes);
@@ -442,14 +629,9 @@ rb_ca_axis2addr_c (VALUE self, VALUE vindices, VALUE vaxis)
442
629
  TypedData_Get_Struct(self, CArray, &carray_data_type, ca);
443
630
  TypedData_Get_Struct(vindices, CArray, &carray_data_type, idx_ca);
444
631
 
445
- /* axis normalization + range check. */
446
- int axis_raw = NIL_P(vaxis) ? 0 : (int) ca_axis_integer(vaxis, NULL);
447
- int axis = (axis_raw < 0) ? ((int) ca->ndim + axis_raw) : axis_raw;
448
- if ( axis < 0 || axis >= ca->ndim ) {
449
- rb_raise(rb_eArgError,
450
- "%s: axis %d out of range for ndim %d",
451
- ca_calling_method_name(), axis_raw, (int) ca->ndim);
452
- }
632
+ long axis_l = ca_normalize_axis_arg(NIL_P(vaxis) ? INT2FIX(0) : vaxis,
633
+ (int) ca->ndim, NULL);
634
+ int axis = (int) axis_l;
453
635
 
454
636
  /* Shape rule: indices.ndim == self.ndim, dims match except at axis. */
455
637
  if ( idx_ca->ndim != ca->ndim ) {
@@ -470,10 +652,11 @@ rb_ca_axis2addr_c (VALUE self, VALUE vindices, VALUE vaxis)
470
652
 
471
653
  /* indices data_type: integer kind accepted; cast to CA_SIZE if needed
472
654
  (zero-copy when already CA_SIZE via to_type identity). */
655
+ ca_check_index_array(vindices);
473
656
  if ( ! ca_is_integer_type(idx_ca) ) {
474
657
  rb_raise(rb_eArgError,
475
- "%s: indices data_type must be integer kind (got %d)",
476
- ca_calling_method_name(), (int) idx_ca->data_type);
658
+ "%s: indices data_type must be integer kind (got %s)",
659
+ ca_calling_method_name(), ca_type_name[idx_ca->data_type]);
477
660
  }
478
661
  volatile VALUE vidx_cast = vindices;
479
662
  CArray *idx_cast = idx_ca;
@@ -535,6 +718,31 @@ rb_ca_axis2addr (int argc, VALUE *argv, VALUE self)
535
718
  }
536
719
 
537
720
 
721
+ /* The array the order of v is read from, for a method that orders v's
722
+ cells (a sort, a selection, a ranking, an extremum by key). A plain
723
+ array is its own key. An ORDERABLE Face orders by its storage, so the
724
+ key is the storage. A Face that is not ORDERABLE but defines its own
725
+ #order (CAConstString, whose cells are offsets; CARecord, whose bytes
726
+ are no order) is keyed by that order, as a dense rank, so ties stay ties
727
+ and a masked cell stays masked. A Face that has no order (a CARecord
728
+ without order_by:) raises in that order; any other Face raises here. */
729
+ VALUE
730
+ ca_order_key (VALUE v, const char *name)
731
+ {
732
+ CArray *ca;
733
+ TypedData_Get_Struct(v, CArray, &carray_data_type, ca);
734
+ if ( ! ca_is_face(ca) ) {
735
+ return v;
736
+ }
737
+ if ( ca_face_orders_by_itself(v) ) {
738
+ VALUE opts = rb_hash_new();
739
+ rb_hash_aset(opts, ID2SYM(rb_intern("method")), ID2SYM(rb_intern("dense")));
740
+ return rb_funcallv_kw(v, rb_intern("order"), 1, &opts, RB_PASS_KEYWORDS);
741
+ }
742
+ ca_face_require_orderable(v, name, "order");
743
+ return rb_ca_strip_face_value(v);
744
+ }
745
+
538
746
  /* ===== sort / sort_copy (value & view surface, sibling of the
539
747
  sort_addr / axis2addr block above). =============== */
540
748
 
@@ -717,8 +925,7 @@ rb_ca_sort_copy (int argc, VALUE *argv, VALUE self)
717
925
  TypedData_Get_Struct(target, CArray, &carray_data_type, cat);
718
926
 
719
927
  /* Normalize axis (negative -> +ndim, range check). */
720
- int axis = rb_ca_normalize_axis_for_ndim(ca_axis_integer(vaxis_use, NULL),
721
- cat->ndim, NULL);
928
+ int axis = ca_normalize_axis_arg(vaxis_use, cat->ndim, NULL);
722
929
 
723
930
  /* Allocate output: same shape and data_type as target
724
931
  (bytes=0 = preserve native bytes). */
@@ -805,6 +1012,7 @@ rb_ca_sort_copy (int argc, VALUE *argv, VALUE self)
805
1012
  ALLOCV_END(aux_holder);
806
1013
 
807
1014
  RB_GC_GUARD(target);
1015
+ rb_ca_inherit_attr(vout, self);
808
1016
  return vout;
809
1017
  }
810
1018
 
@@ -453,16 +453,15 @@ DEFINE_PARTITION_NAN(double, f64);
453
453
  preserved), so argsort on NaN-containing input keeps NaN indices
454
454
  in ascending order.
455
455
 
456
- Implementation: 2-pass stable filter via xmalloc scratch (single
457
- xmalloc + xfree per fiber; negligible overhead vs the sort
458
- itself). */
456
+ Implementation: 2-pass stable filter through scratch, n pairs that
457
+ the caller provides. The caller takes it before it opens its walks:
458
+ an allocation here, inside the walk, would raise past their finish. */
459
459
  #define DEFINE_PARTITION_NAN_PAIR(TYPE, SUFFIX) \
460
460
  ca_size_t \
461
- ca_partition_nan_pair_##SUFFIX (ca_pair_##SUFFIX *a, ca_size_t n) \
461
+ ca_partition_nan_pair_##SUFFIX (ca_pair_##SUFFIX *a, ca_size_t n, \
462
+ ca_pair_##SUFFIX *scratch) \
462
463
  { \
463
464
  if ( n == 0 ) return 0; \
464
- ca_pair_##SUFFIX *scratch = \
465
- (ca_pair_##SUFFIX *) xmalloc((size_t) n * sizeof(ca_pair_##SUFFIX)); \
466
465
  ca_size_t fin = 0, nan_pos = 0; \
467
466
  for ( ca_size_t k = 0; k < n; k++ ) { \
468
467
  if ( !isnan(a[k].v) ) scratch[fin++] = a[k]; \
@@ -472,7 +471,6 @@ ca_partition_nan_pair_##SUFFIX (ca_pair_##SUFFIX *a, ca_size_t n) \
472
471
  if ( isnan(a[k].v) ) scratch[finite_count + nan_pos++] = a[k]; \
473
472
  } \
474
473
  memcpy(a, scratch, (size_t) n * sizeof(ca_pair_##SUFFIX)); \
475
- xfree(scratch); \
476
474
  return finite_count; \
477
475
  } \
478
476
  struct ca_partition_nan_pair_##SUFFIX##_eat_semicolon
data/ext/carray_struct.c CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  C-native member access for CAStruct subclasses. The hooked
4
4
  methods (`[]`, `[]=`, `initialize`) are installed on a subclass by
5
- `CArray.__install_castruct_methods__(klass)` — called from the
5
+ `CArray.install_struct_methods(klass)` — called from the
6
6
  bottom of lib/carray/struct.rb once CAStruct is defined, so this
7
7
  ext can load before that file.
8
8
 
@@ -22,6 +22,7 @@
22
22
  ---------------------------------------------------------------------------- */
23
23
 
24
24
  #include "carray.h"
25
+ #include "carray_internal.h" /* ca_obj2integer_fitting */
25
26
  #include <string.h>
26
27
 
27
28
  /* ---------------------------------------------------------------------
@@ -130,42 +131,42 @@ ca_struct_write_primitive_fast (char *p, int type_code, VALUE val)
130
131
  return;
131
132
  }
132
133
  case CA_INT8: {
133
- int8_t v = (int8_t) NUM2INT(val);
134
+ int8_t v = (int8_t) NUM2LL(ca_obj2integer_fitting(val, CA_INT8));
134
135
  memcpy(p, &v, 1);
135
136
  return;
136
137
  }
137
138
  case CA_UINT8: {
138
- uint8_t v = (uint8_t) NUM2UINT(val);
139
+ uint8_t v = (uint8_t) rb_num2ull(ca_obj2integer_fitting(val, CA_UINT8));
139
140
  memcpy(p, &v, 1);
140
141
  return;
141
142
  }
142
143
  case CA_INT16: {
143
- int16_t v = (int16_t) NUM2INT(val);
144
+ int16_t v = (int16_t) NUM2LL(ca_obj2integer_fitting(val, CA_INT16));
144
145
  memcpy(p, &v, 2);
145
146
  return;
146
147
  }
147
148
  case CA_UINT16: {
148
- uint16_t v = (uint16_t) NUM2UINT(val);
149
+ uint16_t v = (uint16_t) rb_num2ull(ca_obj2integer_fitting(val, CA_UINT16));
149
150
  memcpy(p, &v, 2);
150
151
  return;
151
152
  }
152
153
  case CA_INT32: {
153
- int32_t v = (int32_t) NUM2LONG(val);
154
+ int32_t v = (int32_t) NUM2LL(ca_obj2integer_fitting(val, CA_INT32));
154
155
  memcpy(p, &v, 4);
155
156
  return;
156
157
  }
157
158
  case CA_UINT32: {
158
- uint32_t v = (uint32_t) NUM2ULONG(val);
159
+ uint32_t v = (uint32_t) rb_num2ull(ca_obj2integer_fitting(val, CA_UINT32));
159
160
  memcpy(p, &v, 4);
160
161
  return;
161
162
  }
162
163
  case CA_INT64: {
163
- int64_t v = (int64_t) NUM2LL(val);
164
+ int64_t v = (int64_t) NUM2LL(ca_obj2integer_fitting(val, CA_INT64));
164
165
  memcpy(p, &v, 8);
165
166
  return;
166
167
  }
167
168
  case CA_UINT64: {
168
- uint64_t v = (uint64_t) NUM2ULL(val);
169
+ uint64_t v = (uint64_t) rb_num2ull(ca_obj2integer_fitting(val, CA_UINT64));
169
170
  memcpy(p, &v, 8);
170
171
  return;
171
172
  }
@@ -193,20 +194,40 @@ ca_struct_write_primitive_fast (char *p, int type_code, VALUE val)
193
194
  to keep neighbouring bits untouched.
194
195
  --------------------------------------------------------------------- */
195
196
 
197
+ /* The word's bytes are read and written least significant first on
198
+ every host: the layout numbers the bits byte by byte from the record's
199
+ start, so a field read through a wider word than its neighbour still
200
+ sees the same bits. */
201
+ static uint64_t
202
+ ca_struct_load_le (const char *p, int n)
203
+ {
204
+ uint64_t word = 0;
205
+ int i;
206
+ for (i = 0; i < n; i++) {
207
+ word |= (uint64_t) (uint8_t) p[i] << (8 * i);
208
+ }
209
+ return word;
210
+ }
211
+
212
+ static void
213
+ ca_struct_store_le (char *p, int n, uint64_t word)
214
+ {
215
+ int i;
216
+ for (i = 0; i < n; i++) {
217
+ p[i] = (char) (word >> (8 * i));
218
+ }
219
+ }
220
+
196
221
  static VALUE
197
222
  ca_struct_read_bitfield_fast (const char *p, int view_bytes,
198
223
  int bit_in_word, int bits)
199
224
  {
200
- uint64_t word = 0;
201
- switch (view_bytes) {
202
- case 1: { uint8_t v; memcpy(&v, p, 1); word = v; break; }
203
- case 2: { uint16_t v; memcpy(&v, p, 2); word = v; break; }
204
- case 4: { uint32_t v; memcpy(&v, p, 4); word = v; break; }
205
- case 8: { uint64_t v; memcpy(&v, p, 8); word = v; break; }
206
- default:
225
+ uint64_t word;
226
+ if ( view_bytes != 1 && view_bytes != 2 && view_bytes != 4 && view_bytes != 8 ) {
207
227
  rb_raise(rb_eRuntimeError,
208
228
  "[BUG] bitfield view_bytes %d not in {1,2,4,8}", view_bytes);
209
229
  }
230
+ word = ca_struct_load_le(p, view_bytes);
210
231
  uint64_t mask = (bits >= 64) ? ~(uint64_t)0
211
232
  : ((uint64_t)1 << bits) - 1;
212
233
  uint64_t val = (word >> bit_in_word) & mask;
@@ -223,7 +244,11 @@ static void
223
244
  ca_struct_write_bitfield_fast (char *p, int view_bytes,
224
245
  int bit_in_word, int bits, VALUE val)
225
246
  {
226
- uint64_t v;
247
+ uint64_t v, word;
248
+ if ( view_bytes != 1 && view_bytes != 2 && view_bytes != 4 && view_bytes != 8 ) {
249
+ rb_raise(rb_eRuntimeError,
250
+ "[BUG] bitfield view_bytes %d not in {1,2,4,8}", view_bytes);
251
+ }
227
252
  if ( bits == 1 ) {
228
253
  if ( val == Qtrue || val == INT2FIX(1) ) v = 1;
229
254
  else if ( val == Qfalse || val == INT2FIX(0) ) v = 0;
@@ -239,41 +264,9 @@ ca_struct_write_bitfield_fast (char *p, int view_bytes,
239
264
  uint64_t mask = (bits >= 64) ? ~(uint64_t)0
240
265
  : ((uint64_t)1 << bits) - 1;
241
266
  v &= mask;
242
- uint64_t shifted_mask = mask << bit_in_word;
243
- uint64_t shifted_val = v << bit_in_word;
244
- switch (view_bytes) {
245
- case 1: {
246
- uint8_t word;
247
- memcpy(&word, p, 1);
248
- word = (uint8_t)((word & ~(uint8_t)shifted_mask) | (uint8_t)shifted_val);
249
- memcpy(p, &word, 1);
250
- return;
251
- }
252
- case 2: {
253
- uint16_t word;
254
- memcpy(&word, p, 2);
255
- word = (uint16_t)((word & ~(uint16_t)shifted_mask) | (uint16_t)shifted_val);
256
- memcpy(p, &word, 2);
257
- return;
258
- }
259
- case 4: {
260
- uint32_t word;
261
- memcpy(&word, p, 4);
262
- word = (uint32_t)((word & ~(uint32_t)shifted_mask) | (uint32_t)shifted_val);
263
- memcpy(p, &word, 4);
264
- return;
265
- }
266
- case 8: {
267
- uint64_t word;
268
- memcpy(&word, p, 8);
269
- word = (word & ~shifted_mask) | shifted_val;
270
- memcpy(p, &word, 8);
271
- return;
272
- }
273
- default:
274
- rb_raise(rb_eRuntimeError,
275
- "[BUG] bitfield view_bytes %d not in {1,2,4,8}", view_bytes);
276
- }
267
+ word = ca_struct_load_le(p, view_bytes);
268
+ word = (word & ~(mask << bit_in_word)) | (v << bit_in_word);
269
+ ca_struct_store_le(p, view_bytes, word);
277
270
  }
278
271
 
279
272
  /* ---------------------------------------------------------------------
@@ -333,32 +326,32 @@ ca_struct_write_endian_swapped_fast (char *p, int type_code, VALUE val)
333
326
  {
334
327
  switch (type_code) {
335
328
  case CA_INT16: {
336
- uint16_t v = (uint16_t) NUM2INT(val);
329
+ uint16_t v = (uint16_t) NUM2LL(ca_obj2integer_fitting(val, CA_INT16));
337
330
  v = CA_BSWAP16(v); memcpy(p, &v, 2);
338
331
  return;
339
332
  }
340
333
  case CA_UINT16: {
341
- uint16_t v = (uint16_t) NUM2UINT(val);
334
+ uint16_t v = (uint16_t) rb_num2ull(ca_obj2integer_fitting(val, CA_UINT16));
342
335
  v = CA_BSWAP16(v); memcpy(p, &v, 2);
343
336
  return;
344
337
  }
345
338
  case CA_INT32: {
346
- uint32_t v = (uint32_t) NUM2LONG(val);
339
+ uint32_t v = (uint32_t) NUM2LL(ca_obj2integer_fitting(val, CA_INT32));
347
340
  v = CA_BSWAP32(v); memcpy(p, &v, 4);
348
341
  return;
349
342
  }
350
343
  case CA_UINT32: {
351
- uint32_t v = (uint32_t) NUM2ULONG(val);
344
+ uint32_t v = (uint32_t) rb_num2ull(ca_obj2integer_fitting(val, CA_UINT32));
352
345
  v = CA_BSWAP32(v); memcpy(p, &v, 4);
353
346
  return;
354
347
  }
355
348
  case CA_INT64: {
356
- uint64_t v = (uint64_t) NUM2LL(val);
349
+ uint64_t v = (uint64_t) NUM2LL(ca_obj2integer_fitting(val, CA_INT64));
357
350
  v = CA_BSWAP64(v); memcpy(p, &v, 8);
358
351
  return;
359
352
  }
360
353
  case CA_UINT64: {
361
- uint64_t v = NUM2ULL(val);
354
+ uint64_t v = (uint64_t) rb_num2ull(ca_obj2integer_fitting(val, CA_UINT64));
362
355
  v = CA_BSWAP64(v); memcpy(p, &v, 8);
363
356
  return;
364
357
  }
@@ -767,7 +760,7 @@ void
767
760
  Init_carray_struct (void)
768
761
  {
769
762
  /* Singleton on rb_cCArray so lib/carray/struct.rb can call it as
770
- `CArray.__install_castruct_methods__(klass)`. */
771
- rb_define_singleton_method(rb_cCArray, "__install_castruct_methods__",
772
- rb_carray_install_castruct_methods, 1);
763
+ `CArray.install_struct_methods(klass)`. */
764
+ rb_define_private_method(rb_singleton_class(rb_cCArray), "install_struct_methods",
765
+ rb_carray_install_castruct_methods, 1);
773
766
  }