red_amber 0.1.3 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +31 -7
- data/CHANGELOG.md +214 -10
- data/Gemfile +4 -0
- data/README.md +117 -342
- data/benchmark/csv_load_penguins.yml +15 -0
- data/benchmark/drop_nil.yml +11 -0
- data/doc/DataFrame.md +854 -0
- data/doc/Vector.md +449 -0
- data/doc/image/arrow_table_new.png +0 -0
- data/doc/image/dataframe/assign.png +0 -0
- data/doc/image/dataframe/drop.png +0 -0
- data/doc/image/dataframe/pick.png +0 -0
- data/doc/image/dataframe/remove.png +0 -0
- data/doc/image/dataframe/rename.png +0 -0
- data/doc/image/dataframe/slice.png +0 -0
- data/doc/image/dataframe_model.png +0 -0
- data/doc/image/example_in_red_arrow.png +0 -0
- data/doc/image/tdr.png +0 -0
- data/doc/image/tdr_and_table.png +0 -0
- data/doc/image/tidy_data_in_TDR.png +0 -0
- data/doc/image/vector/binary_element_wise.png +0 -0
- data/doc/image/vector/unary_aggregation.png +0 -0
- data/doc/image/vector/unary_aggregation_w_option.png +0 -0
- data/doc/image/vector/unary_element_wise.png +0 -0
- data/doc/tdr.md +56 -0
- data/doc/tdr_ja.md +56 -0
- data/lib/red-amber.rb +27 -0
- data/lib/red_amber/data_frame.rb +91 -37
- data/lib/red_amber/{data_frame_output.rb → data_frame_displayable.rb} +49 -41
- data/lib/red_amber/data_frame_indexable.rb +38 -0
- data/lib/red_amber/data_frame_observation_operation.rb +11 -0
- data/lib/red_amber/data_frame_selectable.rb +155 -48
- data/lib/red_amber/data_frame_variable_operation.rb +137 -0
- data/lib/red_amber/helper.rb +61 -0
- data/lib/red_amber/vector.rb +69 -16
- data/lib/red_amber/vector_functions.rb +80 -45
- data/lib/red_amber/vector_selectable.rb +124 -0
- data/lib/red_amber/vector_updatable.rb +104 -0
- data/lib/red_amber/version.rb +1 -1
- data/lib/red_amber.rb +1 -16
- data/red_amber.gemspec +3 -6
- metadata +38 -9
data/doc/Vector.md
ADDED
@@ -0,0 +1,449 @@
|
|
1
|
+
# Vector
|
2
|
+
|
3
|
+
Class `RedAmber::Vector` represents a series of data in the DataFrame.
|
4
|
+
|
5
|
+
## Constructor
|
6
|
+
|
7
|
+
### Create from a column in a DataFrame
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
df = RedAmber::DataFrame.new(x: [1, 2, 3])
|
11
|
+
df[:x]
|
12
|
+
# =>
|
13
|
+
#<RedAmber::Vector(:uint8, size=3):0x000000000000f4ec>
|
14
|
+
[1, 2, 3]
|
15
|
+
```
|
16
|
+
|
17
|
+
### New from an Array
|
18
|
+
|
19
|
+
```ruby
|
20
|
+
vector = RedAmber::Vector.new([1, 2, 3])
|
21
|
+
# or
|
22
|
+
vector = RedAmber::Vector.new(1, 2, 3)
|
23
|
+
# or
|
24
|
+
vector = RedAmber::Vector.new(1..3)
|
25
|
+
# or
|
26
|
+
vector = RedAmber::Vector.new(Arrow::Array([1, 2, 3])
|
27
|
+
|
28
|
+
# =>
|
29
|
+
#<RedAmber::Vector(:uint8, size=3):0x000000000000f514>
|
30
|
+
[1, 2, 3]
|
31
|
+
```
|
32
|
+
|
33
|
+
## Properties
|
34
|
+
|
35
|
+
### `to_s`
|
36
|
+
|
37
|
+
### `values`, `to_a`, `entries`
|
38
|
+
|
39
|
+
### `indices`, `indexes`, `indeces`
|
40
|
+
|
41
|
+
Return indices in an Array
|
42
|
+
|
43
|
+
### `to_ary`
|
44
|
+
Vector has `#to_ary`. It implicitly converts a Vector to an Array when required.
|
45
|
+
|
46
|
+
```ruby
|
47
|
+
[1, 2] + Vector.new([3, 4])
|
48
|
+
|
49
|
+
# =>
|
50
|
+
[1, 2, 3, 4]
|
51
|
+
```
|
52
|
+
|
53
|
+
### `size`, `length`, `n_rows`, `nrow`
|
54
|
+
|
55
|
+
### `empty?`
|
56
|
+
|
57
|
+
### `type`
|
58
|
+
|
59
|
+
### `boolean?`, `numeric?`, `string?`, `temporal?`
|
60
|
+
|
61
|
+
### `type_class`
|
62
|
+
|
63
|
+
### [ ] `each` (not impremented yet)
|
64
|
+
|
65
|
+
### [ ] `chunked?` (not impremented yet)
|
66
|
+
|
67
|
+
### [ ] `n_chunks` (not impremented yet)
|
68
|
+
|
69
|
+
### [ ] `each_chunk` (not impremented yet)
|
70
|
+
|
71
|
+
### `n_nils`, `n_nans`
|
72
|
+
|
73
|
+
- `n_nulls` is an alias of `n_nils`
|
74
|
+
|
75
|
+
### `has_nil?`
|
76
|
+
|
77
|
+
Returns `true` if self has any `nil`. Otherwise returns `false`.
|
78
|
+
|
79
|
+
### `inspect(limit: 80)`
|
80
|
+
|
81
|
+
- `limit` sets size limit to display long array.
|
82
|
+
|
83
|
+
```ruby
|
84
|
+
vector = RedAmber::Vector.new((1..50).to_a)
|
85
|
+
# =>
|
86
|
+
#<RedAmber::Vector(:uint8, size=50):0x000000000000f528>
|
87
|
+
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, ... ]
|
88
|
+
```
|
89
|
+
|
90
|
+
## Selecting Values
|
91
|
+
|
92
|
+
### `take(indices)`, `[](indices)`
|
93
|
+
|
94
|
+
- Acceptable class for indices:
|
95
|
+
- Integer, Float
|
96
|
+
- Vector of integer or float
|
97
|
+
- Arrow::Arry of integer or float
|
98
|
+
- Negative index is also OK like the Ruby's primitive Array.
|
99
|
+
|
100
|
+
```ruby
|
101
|
+
array = RedAmber::Vector.new(%w[A B C D E])
|
102
|
+
indices = RedAmber::Vector.new([0.1, -0.5, -5.1])
|
103
|
+
array.take(indices)
|
104
|
+
# or
|
105
|
+
array[indices]
|
106
|
+
|
107
|
+
# =>
|
108
|
+
#<RedAmber::Vector(:string, size=3):0x000000000000f820>
|
109
|
+
["A", "E", "A"]
|
110
|
+
```
|
111
|
+
|
112
|
+
### `filter(booleans)`, `[](booleans)`
|
113
|
+
|
114
|
+
- Acceptable class for booleans:
|
115
|
+
- An array of true, false, or nil
|
116
|
+
- Boolean Vector
|
117
|
+
- Arrow::BooleanArray
|
118
|
+
|
119
|
+
```ruby
|
120
|
+
array = RedAmber::Vector.new(%w[A B C D E])
|
121
|
+
booleans = [true, false, nil, false, true]
|
122
|
+
array.filter(booleans)
|
123
|
+
# or
|
124
|
+
array[booleans]
|
125
|
+
|
126
|
+
# =>
|
127
|
+
#<RedAmber::Vector(:string, size=2):0x000000000000f21c>
|
128
|
+
["A", "E"]
|
129
|
+
```
|
130
|
+
|
131
|
+
## Functions
|
132
|
+
|
133
|
+
### Unary aggregations: `vector.func => scalar`
|
134
|
+
|
135
|
+

|
136
|
+
|
137
|
+
| Method |Boolean|Numeric|String|Options|Remarks|
|
138
|
+
| ----------- | --- | --- | --- | --- | --- |
|
139
|
+
| ✓ `all?` | ✓ | | | ✓ ScalarAggregate| alias `all` |
|
140
|
+
| ✓ `any?` | ✓ | | | ✓ ScalarAggregate| alias `any` |
|
141
|
+
| ✓ `approximate_median`| |✓| | ✓ ScalarAggregate| alias `median`|
|
142
|
+
| ✓ `count` | ✓ | ✓ | ✓ | ✓ Count | |
|
143
|
+
| ✓ `count_distinct`| ✓ | ✓ | ✓ | ✓ Count |alias `count_uniq`|
|
144
|
+
|[ ]`index` | [ ] | [ ] | [ ] |[ ] Index | |
|
145
|
+
| ✓ `max` | ✓ | ✓ | ✓ | ✓ ScalarAggregate| |
|
146
|
+
| ✓ `mean` | ✓ | ✓ | | ✓ ScalarAggregate| |
|
147
|
+
| ✓ `min` | ✓ | ✓ | ✓ | ✓ ScalarAggregate| |
|
148
|
+
| ✓ `min_max` | ✓ | ✓ | ✓ | ✓ ScalarAggregate| |
|
149
|
+
|[ ]`mode` | | [ ] | |[ ] Mode | |
|
150
|
+
| ✓ `product` | ✓ | ✓ | | ✓ ScalarAggregate| |
|
151
|
+
|[ ]`quantile`| | [ ] | |[ ] Quantile| |
|
152
|
+
| ✓ `sd ` | | ✓ | | |ddof: 1 at `stddev`|
|
153
|
+
| ✓ `stddev` | | ✓ | | ✓ Variance|ddof: 0 by default|
|
154
|
+
| ✓ `sum` | ✓ | ✓ | | ✓ ScalarAggregate| |
|
155
|
+
|[ ]`tdigest` | | [ ] | |[ ] TDigest | |
|
156
|
+
| ✓ `var `| | ✓ | | |ddof: 1 at `variance`<br>alias `unbiased_variance`|
|
157
|
+
| ✓ `variance`| | ✓ | | ✓ Variance|ddof: 0 by default|
|
158
|
+
|
159
|
+
|
160
|
+
Options can be used as follows.
|
161
|
+
See the [document of C++ function](https://arrow.apache.org/docs/cpp/compute.html) for detail.
|
162
|
+
|
163
|
+
```ruby
|
164
|
+
double = RedAmber::Vector.new([1, 0/0.0, -1/0.0, 1/0.0, nil, ""])
|
165
|
+
#=>
|
166
|
+
#<RedAmber::Vector(:double, size=6):0x000000000000f910>
|
167
|
+
[1.0, NaN, -Infinity, Infinity, nil, 0.0]
|
168
|
+
|
169
|
+
double.count #=> 5
|
170
|
+
double.count(opts: {mode: :only_valid}) #=> 5, default
|
171
|
+
double.count(opts: {mode: :only_null}) #=> 1
|
172
|
+
double.count(opts: {mode: :all}) #=> 6
|
173
|
+
|
174
|
+
boolean = RedAmber::Vector.new([true, true, nil])
|
175
|
+
#=>
|
176
|
+
#<RedAmber::Vector(:boolean, size=3):0x000000000000f924>
|
177
|
+
[true, true, nil]
|
178
|
+
|
179
|
+
boolean.all #=> true
|
180
|
+
boolean.all(opts: {skip_nulls: true}) #=> true
|
181
|
+
boolean.all(opts: {skip_nulls: false}) #=> false
|
182
|
+
```
|
183
|
+
|
184
|
+
### Unary element-wise: `vector.func => vector`
|
185
|
+
|
186
|
+

|
187
|
+
|
188
|
+
| Method |Boolean|Numeric|String|Options|Remarks|
|
189
|
+
| ------------ | --- | --- | --- | --- | ----- |
|
190
|
+
| ✓ `-@` | | ✓ | | |as `-vector`|
|
191
|
+
| ✓ `negate` | | ✓ | | |`-@` |
|
192
|
+
| ✓ `abs` | | ✓ | | | |
|
193
|
+
|[ ]`acos` | | [ ] | | | |
|
194
|
+
|[ ]`asin` | | [ ] | | | |
|
195
|
+
| ✓ `atan` | | ✓ | | | |
|
196
|
+
| ✓ `bit_wise_not`| | (✓) | | |integer only|
|
197
|
+
| ✓ `ceil` | | ✓ | | | |
|
198
|
+
| ✓ `cos` | | ✓ | | | |
|
199
|
+
| ✓`fill_nil_backward`| ✓ | ✓ | ✓ | | |
|
200
|
+
| ✓`fill_nil_forward` | ✓ | ✓ | ✓ | | |
|
201
|
+
| ✓ `floor` | | ✓ | | | |
|
202
|
+
| ✓ `invert` | ✓ | | | |`!`, alias `not`|
|
203
|
+
|[ ]`ln` | | [ ] | | | |
|
204
|
+
|[ ]`log10` | | [ ] | | | |
|
205
|
+
|[ ]`log1p` | | [ ] | | | |
|
206
|
+
|[ ]`log2` | | [ ] | | | |
|
207
|
+
| ✓ `round` | | ✓ | | ✓ Round (:mode, :n_digits)| |
|
208
|
+
| ✓ `round_to_multiple`| | ✓ | | ✓ RoundToMultiple :mode, :multiple| multiple must be an Arrow::Scalar|
|
209
|
+
| ✓ `sign` | | ✓ | | | |
|
210
|
+
| ✓ `sin` | | ✓ | | | |
|
211
|
+
| ✓`sort_indexes`| ✓ | ✓ | ✓ |:order|alias `sort_indices`|
|
212
|
+
| ✓ `tan` | | ✓ | | | |
|
213
|
+
| ✓ `trunc` | | ✓ | | | |
|
214
|
+
|
215
|
+
### Binary element-wise: `vector.func(vector) => vector`
|
216
|
+
|
217
|
+

|
218
|
+
|
219
|
+
| Method |Boolean|Numeric|String|Options|Remarks|
|
220
|
+
| ----------------- | --- | --- | --- | --- | ----- |
|
221
|
+
| ✓ `add` | | ✓ | | | `+` |
|
222
|
+
| ✓ `atan2` | | ✓ | | | |
|
223
|
+
| ✓ `and_kleene` | ✓ | | | | `&` |
|
224
|
+
| ✓ `and_org ` | ✓ | | | |`and` in Red Arrow|
|
225
|
+
| ✓ `and_not` | ✓ | | | | |
|
226
|
+
| ✓ `and_not_kleene`| ✓ | | | | |
|
227
|
+
| ✓ `bit_wise_and` | | (✓) | | |integer only|
|
228
|
+
| ✓ `bit_wise_or` | | (✓) | | |integer only|
|
229
|
+
| ✓ `bit_wise_xor` | | (✓) | | |integer only|
|
230
|
+
| ✓ `divide` | | ✓ | | | `/` |
|
231
|
+
| ✓ `equal` | ✓ | ✓ | ✓ | |`==`, alias `eq`|
|
232
|
+
| ✓ `greater` | ✓ | ✓ | ✓ | |`>`, alias `gt`|
|
233
|
+
| ✓ `greater_equal` | ✓ | ✓ | ✓ | |`>=`, alias `ge`|
|
234
|
+
| ✓ `is_finite` | | ✓ | | | |
|
235
|
+
| ✓ `is_inf` | | ✓ | | | |
|
236
|
+
| ✓ `is_na` | ✓ | ✓ | ✓ | | |
|
237
|
+
| ✓ `is_nan` | | ✓ | | | |
|
238
|
+
|[ ]`is_nil` | ✓ | ✓ | ✓ |[ ] Null|alias `is_null`|
|
239
|
+
| ✓ `is_valid` | ✓ | ✓ | ✓ | | |
|
240
|
+
| ✓ `less` | ✓ | ✓ | ✓ | |`<`, alias `lt`|
|
241
|
+
| ✓ `less_equal` | ✓ | ✓ | ✓ | |`<=`, alias `le`|
|
242
|
+
|[ ]`logb` | | [ ] | | | |
|
243
|
+
|[ ]`mod` | | [ ] | | | `%` |
|
244
|
+
| ✓ `multiply` | | ✓ | | | `*` |
|
245
|
+
| ✓ `not_equal` | ✓ | ✓ | ✓ | |`!=`, alias `ne`|
|
246
|
+
| ✓ `or_kleene` | ✓ | | | | `\|` |
|
247
|
+
| ✓ `or_org` | ✓ | | | |`or` in Red Arrow|
|
248
|
+
| ✓ `power` | | ✓ | | | `**` |
|
249
|
+
| ✓ `subtract` | | ✓ | | | `-` |
|
250
|
+
| ✓ `shift_left` | | (✓) | | |`<<`, integer only|
|
251
|
+
| ✓ `shift_right` | | (✓) | | |`>>`, integer only|
|
252
|
+
| ✓ `xor` | ✓ | | | | `^` |
|
253
|
+
|
254
|
+
### `uniq`
|
255
|
+
|
256
|
+
Returns a new array with distinct elements.
|
257
|
+
|
258
|
+
(Not impremented functions)
|
259
|
+
|
260
|
+
### `tally` and `value_counts`
|
261
|
+
|
262
|
+
Compute counts of unique elements and return a Hash.
|
263
|
+
|
264
|
+
It returns almost same result as Ruby's tally. These methods consider NaNs are same.
|
265
|
+
|
266
|
+
```ruby
|
267
|
+
array = [0.0/0, Float::NAN]
|
268
|
+
array.tally #=> {NaN=>1, NaN=>1}
|
269
|
+
|
270
|
+
vector = RedAmber::Vector.new(array)
|
271
|
+
vector.tally #=> {NaN=>2}
|
272
|
+
vector.value_counts #=> {NaN=>2}
|
273
|
+
```
|
274
|
+
### `index(element)`
|
275
|
+
|
276
|
+
Returns index of specified element.
|
277
|
+
|
278
|
+
### `sort_indexes`, `sort_indices`, `array_sort_indices`
|
279
|
+
|
280
|
+
### [ ] `sort`, `sort_by`
|
281
|
+
### [ ] argmin, argmax
|
282
|
+
### [ ] (array functions)
|
283
|
+
### [ ] (strings functions)
|
284
|
+
### [ ] (temporal functions)
|
285
|
+
### [ ] (conditional functions)
|
286
|
+
### [ ] (index functions)
|
287
|
+
### [ ] (other functions)
|
288
|
+
|
289
|
+
## Coerce (not impremented)
|
290
|
+
|
291
|
+
## Update vector's value
|
292
|
+
### `replace(specifier, replacer)` => vector
|
293
|
+
|
294
|
+
- Accepts Scalar, Range of Integer, Vector, Array, Arrow::Array as a specifier
|
295
|
+
- Accepts Scalar, Vector, Array and Arrow::Array as a replacer.
|
296
|
+
- Boolean specifiers specify the position of replacer in true.
|
297
|
+
- Index specifiers specify the position of replacer in indices.
|
298
|
+
- replacer specifies the values to be replaced.
|
299
|
+
- The number of true in booleans must be equal to the length of replacer
|
300
|
+
|
301
|
+
```ruby
|
302
|
+
vector = RedAmber::Vector.new([1, 2, 3])
|
303
|
+
booleans = [true, false, true]
|
304
|
+
replacer = [4, 5]
|
305
|
+
vector.replace(booleans, replacer)
|
306
|
+
# =>
|
307
|
+
#<RedAmber::Vector(:uint8, size=3):0x000000000001ee10>
|
308
|
+
[4, 2, 5]
|
309
|
+
```
|
310
|
+
|
311
|
+
- Scalar value in replacer can be broadcasted.
|
312
|
+
|
313
|
+
```ruby
|
314
|
+
replacer = 0
|
315
|
+
vector.replace(booleans, replacer)
|
316
|
+
# =>
|
317
|
+
#<RedAmber::Vector(:uint8, size=3):0x000000000001ee10>
|
318
|
+
[0, 2, 0]
|
319
|
+
```
|
320
|
+
|
321
|
+
- Returned data type is automatically up-casted by replacer.
|
322
|
+
|
323
|
+
```ruby
|
324
|
+
replacer = 1.0
|
325
|
+
vector.replace(booleans, replacer)
|
326
|
+
# =>
|
327
|
+
#<RedAmber::Vector(:double, size=3):0x0000000000025d78>
|
328
|
+
[1.0, 2.0, 1.0]
|
329
|
+
```
|
330
|
+
|
331
|
+
- Position of nil in booleans is replaced with nil.
|
332
|
+
|
333
|
+
```ruby
|
334
|
+
booleans = [true, false, nil]
|
335
|
+
replacer = -1
|
336
|
+
vec.replace(booleans, replacer)
|
337
|
+
=>
|
338
|
+
#<RedAmber::Vector(:int8, size=3):0x00000000000304d0>
|
339
|
+
[-1, 2, nil]
|
340
|
+
```
|
341
|
+
|
342
|
+
- replacer can have nil in it.
|
343
|
+
|
344
|
+
```ruby
|
345
|
+
booleans = [true, false, true]
|
346
|
+
replacer = [nil]
|
347
|
+
vec.replace(booleans, replacer)
|
348
|
+
=>
|
349
|
+
#<RedAmber::Vector(:int8, size=3):0x00000000000304d0>
|
350
|
+
[nil, 2, nil]
|
351
|
+
```
|
352
|
+
|
353
|
+
- If no replacer specified, it is same as to specify nil.
|
354
|
+
|
355
|
+
```ruby
|
356
|
+
booleans = [true, false, true]
|
357
|
+
vec.replace(booleans)
|
358
|
+
=>
|
359
|
+
#<RedAmber::Vector(:int8, size=3):0x00000000000304d0>
|
360
|
+
[nil, 2, nil]
|
361
|
+
```
|
362
|
+
|
363
|
+
- An example to replace 'NA' to nil.
|
364
|
+
|
365
|
+
```ruby
|
366
|
+
vector = RedAmber::Vector.new(['A', 'B', 'NA'])
|
367
|
+
vector.replace(vector == 'NA', nil)
|
368
|
+
# =>
|
369
|
+
#<RedAmber::Vector(:string, size=3):0x000000000000f8ac>
|
370
|
+
["A", "B", nil]
|
371
|
+
```
|
372
|
+
|
373
|
+
- Specifier in indices.
|
374
|
+
|
375
|
+
Specified indices are used 'as sorted'. Position in indices and replacer may not have correspondence.
|
376
|
+
|
377
|
+
```ruby
|
378
|
+
vector = RedAmber::Vector.new([1, 2, 3])
|
379
|
+
indices = [2, 1]
|
380
|
+
replacer = [4, 5]
|
381
|
+
vector.replace(indices, replacer)
|
382
|
+
# =>
|
383
|
+
#<RedAmber::Vector(:uint8, size=3):0x000000000000f244>
|
384
|
+
[1, 4, 5] # not [1, 5, 4]
|
385
|
+
```
|
386
|
+
|
387
|
+
|
388
|
+
### `fill_nil_forward`, `fill_nil_backward` => vector
|
389
|
+
|
390
|
+
Propagate the last valid observation forward (or backward).
|
391
|
+
Or preserve nil if all previous values are nil or at the end.
|
392
|
+
|
393
|
+
```ruby
|
394
|
+
integer = RedAmber::Vector.new([0, 1, nil, 3, nil])
|
395
|
+
integer.fill_nil_forward
|
396
|
+
# =>
|
397
|
+
#<RedAmber::Vector(:uint8, size=5):0x000000000000f960>
|
398
|
+
[0, 1, 1, 3, 3]
|
399
|
+
|
400
|
+
integer.fill_nil_backward
|
401
|
+
# =>
|
402
|
+
#<RedAmber::Vector(:uint8, size=5):0x000000000000f974>
|
403
|
+
[0, 1, 3, 3, nil]
|
404
|
+
```
|
405
|
+
|
406
|
+
### `boolean_vector.if_else(true_choice, false_choice)` => vector
|
407
|
+
|
408
|
+
Choose values based on self. Self must be a boolean Vector.
|
409
|
+
|
410
|
+
`true_choice`, `false_choice` must be of the same type scalar / array / Vector.
|
411
|
+
`nil` values in `cond` will be promoted to the output.
|
412
|
+
|
413
|
+
This example will normalize negative indices to positive ones.
|
414
|
+
|
415
|
+
```ruby
|
416
|
+
indices = RedAmber::Vector.new([1, -1, 3, -4])
|
417
|
+
array_size = 10
|
418
|
+
normalized_indices = (indices < 0).if_else(indices + array_size, indices)
|
419
|
+
|
420
|
+
# =>
|
421
|
+
#<RedAmber::Vector(:int16, size=4):0x000000000000f85c>
|
422
|
+
[1, 9, 3, 6]
|
423
|
+
```
|
424
|
+
|
425
|
+
### `is_in(values)` => boolean vector
|
426
|
+
|
427
|
+
For each element in self, return true if it is found in given `values`, false otherwise.
|
428
|
+
By default, nulls are matched against the value set. (This will be changed in SetLookupOptions: not impremented.)
|
429
|
+
|
430
|
+
```ruby
|
431
|
+
vector = RedAmber::Vector.new %W[A B C D]
|
432
|
+
values = ['A', 'C', 'X']
|
433
|
+
vector.is_in(values)
|
434
|
+
|
435
|
+
# =>
|
436
|
+
#<RedAmber::Vector(:boolean, size=4):0x000000000000f2a8>
|
437
|
+
[true, false, true, false]
|
438
|
+
```
|
439
|
+
|
440
|
+
`values` are casted to the same Class of Vector.
|
441
|
+
|
442
|
+
```ruby
|
443
|
+
vector = RedAmber::Vector.new([1, 2, 255])
|
444
|
+
vector.is_in(1, -1)
|
445
|
+
|
446
|
+
# =>
|
447
|
+
#<RedAmber::Vector(:boolean, size=3):0x000000000000f320>
|
448
|
+
[true, false, true]
|
449
|
+
```
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/doc/image/tdr.png
ADDED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/doc/tdr.md
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
# TDR (Transposed DataFrame Representation)
|
2
|
+
|
3
|
+
([Japanese version](tdr_ja.md) of this document is available)
|
4
|
+
|
5
|
+
TDR is a presentation style of 2D data. It shows columnar vector values in *row Vector* and observations in *column* just like a **transposed** table.
|
6
|
+
|
7
|
+

|
8
|
+
|
9
|
+
Row-oriented data table (1) and columnar data table (2) have different data allocation in memory within a context of Arrow Columnar Format. But they have the same data placement (in rows and columns) in our brain.
|
10
|
+
|
11
|
+
TDR (3) is a logical concept of data placement to transpose rows and columns in a columnar table (2).
|
12
|
+
|
13
|
+

|
14
|
+
|
15
|
+
TDR is not an implementation in software but a logical image in our mind.
|
16
|
+
|
17
|
+
TDR is consistent with the 'transposed' tidy data concept. The only thing we should do is not to use the positional words 'row' and 'column'.
|
18
|
+
|
19
|
+

|
20
|
+
|
21
|
+
TDR is one of a simple way to create DataFrame object in many libraries. For example, we can initalize Arrow::Table in Red Arrow like the right below and get table as left.
|
22
|
+
|
23
|
+

|
24
|
+
|
25
|
+
We are using TDR style code naturally. For other example:
|
26
|
+
- Ruby: Daru::DataFrame, Rover::DataFrame accept same arguments.
|
27
|
+
- Python: similar style in Pandas for pd.DataFrame(data_in_dict)
|
28
|
+
- R: similar style in tidyr for tibble(x = 1:3, y = c("A", "B", "C"))
|
29
|
+
|
30
|
+
There are other ways to initialize data frame, but they are not intuitive.
|
31
|
+
|
32
|
+
## Table and TDR API
|
33
|
+
|
34
|
+
The API based on TDR is draft and RedAmber is a small experiment to test the TDR concept. The following is a comparison of Table and TDR (draft).
|
35
|
+
|
36
|
+
| |Basic Table|Transposed DataFrame|Comment for TDR|
|
37
|
+
|-----------|---------|------------|---|
|
38
|
+
|name in TDR|`Table`|`TDR`|**T**ransposed **D**ataFrame **R**epresentation|
|
39
|
+
|variable |located in a column|a key and a `Vector` in lateral|select by keys|
|
40
|
+
|observation|located in a row|sliced in vertical|select by indices|
|
41
|
+
|number of variables|n_columns etc. |`n_keys` |`n_cols` is available as an alias|
|
42
|
+
|number of observations|n_rows etc. |`size` |`n_rows` is available as an alias|
|
43
|
+
|shape |[n_rows, n_columns] |`shape`=`[size, n_keys]` |same order as Table|
|
44
|
+
|Select variables|select, filter, [ ], etc.|`pick` or `[keys]` |accepts arguments or a block|
|
45
|
+
|Reject variables|drop, etc.|`drop` |accepts arguments or a block|
|
46
|
+
|Select observations|slice, [ ], iloc, etc.|`slice` or `[indices]` |accepts arguments or a block|
|
47
|
+
|Reject observations|drop, etc.|`remove` |accepts arguments or a block|
|
48
|
+
|Add variables|mutate, assign, etc.|`assign` |accepts arguments or a block|
|
49
|
+
|update variables|transmute, [ ]=, etc.|`assign` |accepts arguments or a block|
|
50
|
+
|inner join| inner_join(a,b)<br>merge(a, b, how='inner')|`a.inner_join(b)` |with a option on:|
|
51
|
+
|left join| left_join(a,b)<br>merge(a, b, how='left')|`a.join(b)` |naturally join from bottom<br>with a option on:|
|
52
|
+
|right join| right_join(a,b))<br>merge(a, b, how='right')|`b.join(a)` |naturally join from bottom<br>with a option on:|
|
53
|
+
|
54
|
+
## Q and A for TDR
|
55
|
+
|
56
|
+
(Not prepared yet)
|
data/doc/tdr_ja.md
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
# TDR (Transposed DataFrame Representation)
|
2
|
+
|
3
|
+
([英語版](tdr.md) もあります)
|
4
|
+
|
5
|
+
TDR は、2次元のデータの表現方法につけた名前です。TDR では下の図のように同じ型のデータに key というラベルをつけて横に並べ、それらを縦に積み重ねてデータを表現します。
|
6
|
+
|
7
|
+

|
8
|
+
|
9
|
+
Arrow Columnar Format では、csv のような従来の行指向データ(1)に対して、列方向に連続したデータ(2)を取り扱います。この行、列という言葉は私たちの脳内イメージを規定していて、データフレームの構造といえば(1)または(2)のような形を思い浮かべることでしょう。しかし、本質は連続したデータの配置にあるので、我々の頭の中では(3)のように行と列を入れ替えて考えてもいいはずです。
|
10
|
+
|
11
|
+

|
12
|
+
|
13
|
+
大事なことは、TDR は頭の中の論理的なイメージであって、実装上のアーキテクチャではないということです。
|
14
|
+
|
15
|
+
TDR は、整然データ(tidy data)の考え方とも矛盾しません。TDR における整然データは行と列を入れ替えた形で全く同じデータを表しています。一つだけ気をつけることは、混乱を避けるため、位置や方向に関するワードである行(row)や列(column)を避けるべきであるということです。
|
16
|
+
|
17
|
+

|
18
|
+
|
19
|
+
TDR は、現時点でも2次元データを楽に初期化できる記法で、ごく自然に使われています。例えば、Red Arrow ではArrow::Table を初期化する際に下の図の右のように書けます。
|
20
|
+
|
21
|
+

|
22
|
+
|
23
|
+
これはごく自然な書き方ですが、この形は TDR の形と一致しています。その他の例として:
|
24
|
+
- Ruby: Daru::DataFrame, Rover::DataFrame でも上と同じように書けます。
|
25
|
+
- Python: Pandas で pd.DataFrame(data_in_dict) のように dict を使う場合が同じです。
|
26
|
+
- R: tidyr で tibble(x = 1:3, y = c("A", "B", "C")) のように書けます。
|
27
|
+
|
28
|
+
それぞれのライブラリーで、データフレームを初期化するやり方はこれだけではありませんが、他の方法は少し回りくどいような印象があります。
|
29
|
+
|
30
|
+
TDR で考えた方がちょっぴりうまくいくというのは単なる仮説ですが、その理由は「この惑星では横書きでコードを書く」からではないかと私は考えています。
|
31
|
+
|
32
|
+
## Table and TDR API
|
33
|
+
|
34
|
+
TDR に基づいた API はまだ暫定板の段階であり、RedAmber は TDR の実験の場であると考えています。下記の表に TDR と行x列形式の Table のAPIの比較を示します(暫定版)。
|
35
|
+
|
36
|
+
| |従来の Table|Transposed DataFrame|TDRに対するコメント|
|
37
|
+
|-----------|---------|------------|---|
|
38
|
+
|TDRでの呼称|`Table`|`TDR`|**T**ransposed **D**ataFrame **R**epresentationの略|
|
39
|
+
|変数 |列に配置|`variables`<br>key と `Vector` として横方向に配置|key で選択|
|
40
|
+
|観測 |行に配置|`observations`<br>縦方向に切った一つ一つはslice|index や `slice` メソッドで選択|
|
41
|
+
|変数(列)の数|ncol, n_columns など |`n_keys` |`n_cols` をエイリアスとして設定|
|
42
|
+
|観測(行)の数|nrow, n_rows など |`size` |`n_rows` をエイリアスとして設定|
|
43
|
+
|形状 |[nrow, ncol] |`shape`=`[size, n_keys]` |行, 列の順番は同じ|
|
44
|
+
|変数(列)の選択|select, filter, [ ], など|`pick` or `[keys]` |引数またはブロックで指定|
|
45
|
+
|変数(列)の削除|drop, など|`drop` |引数またはブロックで指定|
|
46
|
+
|観測(行)の選択|slice, [ ], iloc, など|`slice` or `[indices]` |引数またはブロックで指定|
|
47
|
+
|観測(行)の削除|drop, など|`remove` |引数またはブロックで指定|
|
48
|
+
|変数(列)の追加|mutate, assign, など|`assign` |引数またはブロックで指定|
|
49
|
+
|変数(列)の更新|transmute, [ ]=, など|`assign` |引数またはブロックで指定|
|
50
|
+
|内部結合| inner_join(a,b)<br>merge(a, b, how='inner')|`a.inner_join(b)` |オプション on:|
|
51
|
+
|左結合| left_join(a,b)<br>merge(a, b, how='left')|`a.join(b)` |自然に下にくっつける<br>オプション on:|
|
52
|
+
|右結合| right_join(a,b))<br>merge(a, b, how='right')|`b.join(a)` |自然に下にくっつける<br>オプション on:|
|
53
|
+
|
54
|
+
## Q and A for TDR
|
55
|
+
|
56
|
+
(作成中)
|
data/lib/red-amber.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'arrow'
|
4
|
+
require 'rover-df'
|
5
|
+
|
6
|
+
require_relative 'red_amber/helper'
|
7
|
+
require_relative 'red_amber/data_frame_displayable'
|
8
|
+
require_relative 'red_amber/data_frame_indexable'
|
9
|
+
require_relative 'red_amber/data_frame_selectable'
|
10
|
+
require_relative 'red_amber/data_frame_observation_operation'
|
11
|
+
require_relative 'red_amber/data_frame_variable_operation'
|
12
|
+
require_relative 'red_amber/data_frame'
|
13
|
+
require_relative 'red_amber/vector_functions'
|
14
|
+
require_relative 'red_amber/vector_updatable'
|
15
|
+
require_relative 'red_amber/vector_selectable'
|
16
|
+
require_relative 'red_amber/vector'
|
17
|
+
require_relative 'red_amber/version'
|
18
|
+
|
19
|
+
module RedAmber
|
20
|
+
class Error < StandardError; end
|
21
|
+
|
22
|
+
class DataFrameArgumentError < ArgumentError; end
|
23
|
+
class DataFrameTypeError < TypeError; end
|
24
|
+
|
25
|
+
class VectorArgumentError < ArgumentError; end
|
26
|
+
class VectorTypeError < TypeError; end
|
27
|
+
end
|