carray 3.0.3 → 3.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +469 -0
  3. data/README.md +3 -2
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_array_pool.c +25 -1
  6. data/ext/ca_axis_descriptor.h +27 -24
  7. data/ext/ca_axis_dispatch.c +14 -26
  8. data/ext/ca_axis_group.c +27 -27
  9. data/ext/ca_binop_dispatch.c +1 -29
  10. data/ext/ca_binop_dispatch.h +0 -1
  11. data/ext/ca_categorical_iterator.c +231 -153
  12. data/ext/ca_compare.c +9 -20
  13. data/ext/ca_composite_dispatch.c +9 -9
  14. data/ext/ca_group_iter.c +16 -17
  15. data/ext/ca_kernel_iterator.c +1240 -2000
  16. data/ext/ca_kernel_iterator.h +502 -519
  17. data/ext/ca_monop_dispatch.c +82 -59
  18. data/ext/ca_monop_dispatch.h +1 -0
  19. data/ext/ca_obj_array.c +152 -45
  20. data/ext/ca_obj_bincmp.c +25 -21
  21. data/ext/ca_obj_binop.c +16 -26
  22. data/ext/ca_obj_bitarray.c +8 -3
  23. data/ext/ca_obj_bitfield.c +63 -49
  24. data/ext/ca_obj_block.c +18 -9
  25. data/ext/ca_obj_byte_swap.c +7 -7
  26. data/ext/ca_obj_const_string.c +188 -22
  27. data/ext/ca_obj_face.c +280 -58
  28. data/ext/ca_obj_face.h +19 -0
  29. data/ext/ca_obj_fake.c +15 -29
  30. data/ext/ca_obj_farray.c +6 -3
  31. data/ext/ca_obj_field.c +30 -8
  32. data/ext/ca_obj_fixlen_string.c +2 -3
  33. data/ext/ca_obj_grid.c +455 -134
  34. data/ext/ca_obj_meld.c +172 -34
  35. data/ext/ca_obj_moncmp.c +23 -8
  36. data/ext/ca_obj_monop.c +113 -67
  37. data/ext/ca_obj_object.c +210 -52
  38. data/ext/ca_obj_record.c +15 -7
  39. data/ext/ca_obj_reduce.c +17 -8
  40. data/ext/ca_obj_refer.c +42 -35
  41. data/ext/ca_obj_remap.c +30 -9
  42. data/ext/ca_obj_repeat.c +9 -10
  43. data/ext/ca_obj_roll.c +31 -10
  44. data/ext/ca_obj_select.c +48 -14
  45. data/ext/ca_obj_select_axis.c +15 -47
  46. data/ext/ca_obj_shift.c +41 -7
  47. data/ext/ca_obj_stack.c +180 -34
  48. data/ext/ca_obj_stride.c +190 -134
  49. data/ext/ca_obj_string.c +80 -4
  50. data/ext/ca_obj_tile.c +32 -11
  51. data/ext/ca_obj_time.c +22 -6
  52. data/ext/ca_obj_timedelta.c +22 -6
  53. data/ext/ca_obj_transpose.c +13 -7
  54. data/ext/ca_obj_triop.c +26 -32
  55. data/ext/ca_obj_window.c +267 -74
  56. data/ext/ca_op_ipower.c +4 -4
  57. data/ext/ca_sort_kernels.h +6 -3
  58. data/ext/ca_sweep_engine.c +2 -2
  59. data/ext/caframe_csv_reader.c +80 -1
  60. data/ext/carray.h +214 -75
  61. data/ext/carray_access.c +200 -99
  62. data/ext/carray_address_basis.c +343 -87
  63. data/ext/carray_attribute.c +113 -6
  64. data/ext/carray_bincount.c +10 -8
  65. data/ext/carray_broadcast.c +13 -8
  66. data/ext/carray_build_flags.h +3 -0
  67. data/ext/carray_call_cfunc.c +108 -0
  68. data/ext/carray_cast.c +375 -96
  69. data/ext/carray_cast_func.rb +90 -20
  70. data/ext/carray_class.c +4 -3
  71. data/ext/carray_config.h +28 -0
  72. data/ext/carray_conversion.c +18 -9
  73. data/ext/carray_copy.c +51 -14
  74. data/ext/carray_core.c +150 -55
  75. data/ext/carray_count.c +56 -20
  76. data/ext/carray_element.c +81 -33
  77. data/ext/carray_factorize.c +138 -84
  78. data/ext/carray_generate.c +199 -52
  79. data/ext/carray_histogram.c +63 -18
  80. data/ext/carray_hold.c +12 -15
  81. data/ext/carray_index_classifier.c +388 -176
  82. data/ext/carray_internal.h +348 -29
  83. data/ext/carray_kernels_bincmp.c +4453 -0
  84. data/ext/carray_kernels_binop.c +11368 -0
  85. data/ext/carray_kernels_init.c +1734 -0
  86. data/ext/carray_kernels_moncmp.c +2104 -0
  87. data/ext/carray_kernels_monop.c +18369 -0
  88. data/ext/carray_kernels_reduce_aggregate.c +23818 -0
  89. data/ext/carray_kernels_reduce_boolean.c +406 -0
  90. data/ext/carray_kernels_reduce_cumulative.c +10086 -0
  91. data/ext/carray_kernels_reduce_extreme.c +17634 -0
  92. data/ext/carray_kernels_reduce_variance.c +3849 -0
  93. data/ext/carray_kernels_scan.c +4302 -0
  94. data/ext/carray_kernels_search.c +33829 -0
  95. data/ext/carray_kernels_sort.c +10897 -0
  96. data/ext/carray_kernels_triop.c +1864 -0
  97. data/ext/carray_key_windows.c +220 -0
  98. data/ext/carray_lazy.c +170 -111
  99. data/ext/carray_mask.c +337 -94
  100. data/ext/carray_median_percentile.c +82 -28
  101. data/ext/carray_memory_view.c +669 -561
  102. data/ext/carray_operator.c +129 -52
  103. data/ext/carray_order.c +604 -77
  104. data/ext/carray_partition.c +28 -27
  105. data/ext/carray_random.c +85 -31
  106. data/ext/carray_round.c +273 -0
  107. data/ext/carray_scatter.c +87 -45
  108. data/ext/carray_segment.c +3 -3
  109. data/ext/carray_slab.c +123 -37
  110. data/ext/carray_sort.c +259 -51
  111. data/ext/carray_sort_kernel.c +5 -7
  112. data/ext/carray_struct.c +54 -61
  113. data/ext/carray_test.c +26 -5
  114. data/ext/carray_time_text.c +4 -2
  115. data/ext/carray_undef.c +4 -0
  116. data/ext/carray_utils.c +309 -66
  117. data/ext/mk_call_cfunc.rb +19 -2
  118. data/ext/mkkernel.rb +403 -350
  119. data/ext/ruby_carray.c +10 -4
  120. data/ext/version.h +4 -4
  121. data/lib/carray/attribute.rb +139 -22
  122. data/lib/carray/autoload_carray.rb +8 -31
  123. data/lib/carray/autoload_method_extension.rb +14 -5
  124. data/lib/carray/axis_group.rb +255 -106
  125. data/lib/carray/basics.rb +105 -17
  126. data/lib/carray/bincount_nd.rb +12 -2
  127. data/lib/carray/block_iterator.rb +126 -23
  128. data/lib/carray/boolean_reduce.rb +17 -17
  129. data/lib/carray/categorical.rb +195 -59
  130. data/lib/carray/categorical_iterator.rb +132 -29
  131. data/lib/carray/complex.rb +5 -5
  132. data/lib/carray/conditional.rb +67 -18
  133. data/lib/carray/const_string.rb +24 -5
  134. data/lib/carray/construct.rb +15 -11
  135. data/lib/carray/data_type_extension.rb +9 -1
  136. data/lib/carray/distance_limit.rb +77 -0
  137. data/lib/carray/fixlen_string.rb +48 -3
  138. data/lib/carray/frame/column_verbs.rb +71 -0
  139. data/lib/carray/frame/concat.rb +17 -3
  140. data/lib/carray/frame/convert.rb +69 -0
  141. data/lib/carray/frame/csv_parser.rb +111 -17
  142. data/lib/carray/frame/describe.rb +3 -2
  143. data/lib/carray/frame/frame.rb +252 -28
  144. data/lib/carray/frame/group.rb +603 -101
  145. data/lib/carray/frame/io.rb +117 -28
  146. data/lib/carray/frame/join.rb +312 -51
  147. data/lib/carray/frame/records.rb +83 -7
  148. data/lib/carray/frame/reshape.rb +42 -35
  149. data/lib/carray/frame/sort.rb +7 -19
  150. data/lib/carray/frame/verbs.rb +188 -48
  151. data/lib/carray/frame/windows.rb +117 -0
  152. data/lib/carray/frame.rb +2 -0
  153. data/lib/carray/fuse_source.rb +42 -6
  154. data/lib/carray/fusion.rb +40 -16
  155. data/lib/carray/histogram.rb +6 -1
  156. data/lib/carray/inspect.rb +31 -8
  157. data/lib/carray/iterator.rb +85 -7
  158. data/lib/carray/key_windows.rb +620 -0
  159. data/lib/carray/lazy.rb +217 -183
  160. data/lib/carray/mask_gap_fill.rb +63 -57
  161. data/lib/carray/meld_reduce.rb +12 -14
  162. data/lib/carray/methods/bin.rb +26 -18
  163. data/lib/carray/methods/bincount.rb +16 -0
  164. data/lib/carray/methods/bit_string.rb +1 -1
  165. data/lib/carray/methods/choose.rb +16 -3
  166. data/lib/carray/methods/composition.rb +3 -1
  167. data/lib/carray/methods/gather_nd.rb +14 -5
  168. data/lib/carray/methods/insert_block.rb +2 -2
  169. data/lib/carray/methods/is_between.rb +59 -0
  170. data/lib/carray/methods/is_in.rb +61 -37
  171. data/lib/carray/methods/join.rb +1 -6
  172. data/lib/carray/methods/lag.rb +49 -0
  173. data/lib/carray/methods/locate_addr.rb +47 -51
  174. data/lib/carray/methods/mode.rb +1 -1
  175. data/lib/carray/methods/nunique.rb +11 -6
  176. data/lib/carray/methods/resize.rb +3 -3
  177. data/lib/carray/methods/value_counts.rb +4 -11
  178. data/lib/carray/record_order.rb +3 -3
  179. data/lib/carray/rng.rb +3 -0
  180. data/lib/carray/runtime.rb +53 -8
  181. data/lib/carray/segment_iterator.rb +323 -88
  182. data/lib/carray/serialize.rb +45 -14
  183. data/lib/carray/slab_iterator.rb +50 -0
  184. data/lib/carray/stack.rb +15 -12
  185. data/lib/carray/string.rb +113 -5
  186. data/lib/carray/string_operation_extension.rb +21 -6
  187. data/lib/carray/struct.rb +9 -21
  188. data/lib/carray/struct_builder.rb +24 -11
  189. data/lib/carray/time.rb +819 -158
  190. data/lib/carray/window_iterator.rb +122 -16
  191. data/lib/carray.rb +7 -0
  192. data/yard-stubs/ca_obj_bitfield.rb +1 -1
  193. data/yard-stubs/ca_obj_field.rb +4 -0
  194. data/yard-stubs/ca_obj_grid.rb +21 -10
  195. data/yard-stubs/ca_obj_refer.rb +2 -1
  196. data/yard-stubs/ca_obj_roll.rb +2 -2
  197. data/yard-stubs/ca_obj_shift.rb +5 -1
  198. data/yard-stubs/ca_obj_tile.rb +1 -1
  199. data/yard-stubs/ca_obj_transpose.rb +5 -4
  200. data/yard-stubs/ca_obj_window.rb +8 -3
  201. data/yard-stubs/carray_access.rb +6 -1
  202. data/yard-stubs/carray_attribute.rb +3 -6
  203. data/yard-stubs/carray_cast.rb +12 -0
  204. data/yard-stubs/carray_conversion.rb +7 -2
  205. data/yard-stubs/carray_copy.rb +4 -2
  206. data/yard-stubs/carray_count.rb +4 -4
  207. data/yard-stubs/carray_element.rb +17 -10
  208. data/yard-stubs/carray_generate.rb +21 -1
  209. data/yard-stubs/carray_mask.rb +32 -3
  210. data/yard-stubs/carray_math.rb +41 -5
  211. data/yard-stubs/carray_median_percentile.rb +5 -2
  212. data/yard-stubs/carray_memory_view.rb +40 -23
  213. data/yard-stubs/carray_order.rb +129 -22
  214. data/yard-stubs/carray_random.rb +13 -4
  215. data/yard-stubs/carray_scatter.rb +17 -10
  216. data/yard-stubs/carray_slab.rb +9 -2
  217. data/yard-stubs/carray_sort.rb +6 -5
  218. data/yard-stubs/carray_test.rb +5 -3
  219. metadata +25 -3
  220. data/lib/carray/arrow_tensor.rb +0 -401
  221. data/lib/carray/table.rb +0 -108
data/ext/ca_obj_farray.c CHANGED
@@ -7,6 +7,7 @@
7
7
  ---------------------------------------------------------------------------- */
8
8
 
9
9
  #include "carray.h"
10
+ #include "carray_internal.h" /* ca_view_inherit_mask */
10
11
 
11
12
  extern ca_operation_function_t ca_stride_func;
12
13
 
@@ -46,9 +47,9 @@ ca_farray_setup (CAStride *ca, CArray *parent)
46
47
  strides[i] = parent_byte_stride[ndim - 1 - i];
47
48
  }
48
49
 
49
- ca_stride_setup(ca, CA_OBJ_FARRAY, parent,
50
- parent->data_type, parent->bytes,
51
- ndim, newdim, strides, 0);
50
+ ca_stride_setup_view(ca, CA_OBJ_FARRAY, parent,
51
+ parent->data_type, parent->bytes,
52
+ ndim, newdim, strides, 0);
52
53
 
53
54
  if (ca_is_scalar(parent)) {
54
55
  ca_set_flag(ca, CA_FLAG_SCALAR);
@@ -59,8 +60,10 @@ ca_farray_setup (CAStride *ca, CArray *parent)
59
60
  CAStride *
60
61
  ca_farray_new (CArray *parent)
61
62
  {
63
+ /* check: the parent's shape, reversed, is already valid */
62
64
  CAStride *ca = (CAStride *) ca_array_alloc(CA_OBJ_FARRAY, parent->ndim);
63
65
  ca_farray_setup(ca, parent);
66
+ ca_fill_or_free((CArray *) ca, ca_view_inherit_mask, (VALUE) ca);
64
67
  return ca;
65
68
  }
66
69
 
data/ext/ca_obj_field.c CHANGED
@@ -32,19 +32,30 @@ ca_field_setup (CAStride *ca, CArray *parent,
32
32
  ca_size_t strides[CA_RANK_MAX];
33
33
  int8_t i;
34
34
 
35
- CA_CHECK_DATA_TYPE(data_type);
36
- CA_CHECK_BYTES(data_type, bytes);
35
+ CA_CHECK_DATA_TYPE_GIVEN(data_type);
36
+ CA_CHECK_BYTES_GIVEN(data_type, bytes);
37
37
 
38
38
  if ( offset < 0 ) {
39
39
  rb_raise(rb_eArgError, "negative offset");
40
40
  }
41
41
  if ( data_type == CA_OBJECT ) {
42
42
  rb_raise(rb_eCADataTypeError,
43
- "CA_OBJECT can not to be a data_type for CAField");
43
+ "%s: a field can't be read as object", ca_calling_method_name());
44
44
  }
45
45
  if ( ca_is_object_type(parent) ) {
46
46
  rb_raise(rb_eCADataTypeError,
47
- "a field can not be taken from an object array");
47
+ "%s: a field can't be taken from an object array",
48
+ ca_calling_method_name());
49
+ }
50
+ /* A boolean cell holds 0 or 1, and a field of a numeric array is
51
+ written as that number. A record's boolean member is a field of a
52
+ fixlen array, whose member stores keep it 0 or 1. */
53
+ if ( data_type == CA_BOOLEAN && parent->data_type != CA_BOOLEAN
54
+ && parent->data_type != CA_FIXLEN ) {
55
+ rb_raise(rb_eCADataTypeError,
56
+ "%s: a field of a %s array can't be read as boolean (its bytes "
57
+ "need not be 0 or 1); compare it instead, e.g. ne(0)",
58
+ ca_calling_method_name(), ca_type_name[parent->data_type]);
48
59
  }
49
60
  if ( parent->bytes < offset + bytes ) {
50
61
  rb_raise(rb_eArgError, "offset or bytes out of range");
@@ -67,9 +78,9 @@ ca_field_setup (CAStride *ca, CArray *parent,
67
78
  return 0;
68
79
  }
69
80
 
70
- ca_stride_setup(ca, CA_OBJ_FIELD, parent,
71
- data_type, bytes,
72
- parent->ndim, parent->dim, strides, offset);
81
+ ca_stride_setup_view(ca, CA_OBJ_FIELD, parent,
82
+ data_type, bytes,
83
+ parent->ndim, parent->dim, strides, offset);
73
84
 
74
85
  return 0;
75
86
  }
@@ -81,6 +92,7 @@ ca_field_new (CArray *parent, ca_size_t offset, int8_t data_type, ca_size_t byte
81
92
  ca_field_setup(NULL, parent, offset, data_type, bytes);
82
93
  ca = (CAStride *) ca_array_alloc(CA_OBJ_FIELD, parent->ndim);
83
94
  ca_field_setup(ca, parent, offset, data_type, bytes);
95
+ ca_fill_or_free((CArray *) ca, ca_view_inherit_mask, (VALUE) ca);
84
96
  return ca;
85
97
  }
86
98
 
@@ -94,6 +106,14 @@ rb_ca_field_new (VALUE cary, ca_size_t offset, int8_t data_type, ca_size_t bytes
94
106
  CAStride *ca;
95
107
  rb_check_carray_object(cary);
96
108
  TypedData_Get_Struct(cary, CArray, &carray_data_type, parent);
109
+ /* A record's boolean member is a byte of a fixlen array that other
110
+ fields (a uint8 one, a String store of the record) can write as any
111
+ byte. It is read through a cast to boolean, which refuses a byte
112
+ that is not 0 or 1, as fake(:boolean) does. */
113
+ if ( data_type == CA_BOOLEAN && parent->data_type == CA_FIXLEN ) {
114
+ obj = rb_ca_field_new(cary, offset, CA_UINT8, 1);
115
+ return rb_ca_fake_new(obj, CA_BOOLEAN, 1);
116
+ }
97
117
  ca = ca_field_new(parent, offset, data_type, bytes);
98
118
  obj = ca_wrap_struct(ca);
99
119
  rb_ca_set_parent(obj, cary);
@@ -128,7 +148,7 @@ rb_ca_field (int argc, VALUE *argv, VALUE self)
128
148
  rb_scan_args(argc, argv, "21", (VALUE *) &voffset, (VALUE *) &rtype, (VALUE *) &ropt);
129
149
  rb_scan_options(ropt, "bytes", &rbytes);
130
150
 
131
- offset = NUM2SIZE(voffset);
151
+ offset = ca_integer_arg(voffset, "offset", NULL);
132
152
 
133
153
  if ( rb_obj_is_carray(rtype) ) {
134
154
  CArray *ct;
@@ -138,6 +158,7 @@ rb_ca_field (int argc, VALUE *argv, VALUE self)
138
158
  TypedData_Get_Struct(rtype, CArray, &carray_data_type, ct);
139
159
  data_type = CA_FIXLEN;
140
160
  bytes = ct->bytes * ct->elements;
161
+ ca_check_rank_fits((long) ca->ndim + ct->ndim, "field");
141
162
  obj = rb_ca_field_new(self, offset, data_type, bytes);
142
163
  ndim = ca->ndim + ct->ndim;
143
164
  for (i=0; i<ca->ndim; i++) {
@@ -181,6 +202,7 @@ rb_ca_field_initialize_copy (VALUE self, VALUE other)
181
202
  /* Reconstruct from the source's base_offset / data_type / bytes;
182
203
  ca_field_setup recomputes strides from parent's shape. */
183
204
  ca_field_setup(ca, cs->parent, cs->base_offset, cs->data_type, cs->bytes);
205
+ ca_view_inherit_mask((VALUE) ca);
184
206
 
185
207
  return self;
186
208
  }
@@ -137,9 +137,6 @@ ca_fixlen_string_setup (CAFixlenString *ca, CArray *parent)
137
137
  ca->attach = 0;
138
138
  ca->nosync = 0;
139
139
 
140
- if ( ca_has_mask(parent) ) {
141
- ca_create_mask((CArray *) ca);
142
- }
143
140
 
144
141
  if ( ca_is_scalar(parent) ) {
145
142
  ca_set_flag(ca, CA_FLAG_SCALAR);
@@ -159,6 +156,7 @@ ca_fixlen_string_new (CArray *parent)
159
156
  ca_fixlen_string_setup(NULL, parent);
160
157
  ca = (CAFixlenString *) ca_array_alloc(CA_OBJ_FIXLEN_STRING, parent->ndim);
161
158
  ca_fixlen_string_setup(ca, parent);
159
+ ca_fill_or_free((CArray *) ca, ca_view_inherit_mask, (VALUE) ca);
162
160
  return ca;
163
161
  }
164
162
 
@@ -282,6 +280,7 @@ rb_ca_fixlen_string_initialize_copy (VALUE self, VALUE other)
282
280
  ca_array_pool_alloc(ca, CA_OBJ_FIXLEN_STRING, cs->parent->ndim);
283
281
  }
284
282
  ca_fixlen_string_setup(ca, cs->parent);
283
+ ca_view_inherit_mask((VALUE) ca);
285
284
  return self;
286
285
  }
287
286