red_amber 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +56 -22
- data/.yardopts +2 -0
- data/CHANGELOG.md +178 -0
- data/Gemfile +1 -1
- data/LICENSE +1 -1
- data/README.md +29 -30
- data/benchmark/basic.yml +7 -7
- data/benchmark/combine.yml +3 -3
- data/benchmark/dataframe.yml +15 -9
- data/benchmark/group.yml +6 -6
- data/benchmark/reshape.yml +6 -6
- data/benchmark/vector.yml +6 -3
- data/doc/DataFrame.md +32 -12
- data/doc/DataFrame_Comparison.md +65 -0
- data/doc/SubFrames.md +11 -0
- data/doc/Vector.md +207 -1
- data/doc/yard-templates/default/fulldoc/html/css/common.css +6 -0
- data/lib/red_amber/data_frame.rb +454 -85
- data/lib/red_amber/data_frame_combinable.rb +609 -115
- data/lib/red_amber/data_frame_displayable.rb +313 -34
- data/lib/red_amber/data_frame_indexable.rb +122 -19
- data/lib/red_amber/data_frame_loadsave.rb +78 -10
- data/lib/red_amber/data_frame_reshaping.rb +184 -14
- data/lib/red_amber/data_frame_selectable.rb +623 -70
- data/lib/red_amber/data_frame_variable_operation.rb +452 -35
- data/lib/red_amber/group.rb +186 -22
- data/lib/red_amber/helper.rb +74 -14
- data/lib/red_amber/refinements.rb +26 -6
- data/lib/red_amber/subframes.rb +1101 -0
- data/lib/red_amber/vector.rb +362 -11
- data/lib/red_amber/vector_aggregation.rb +312 -0
- data/lib/red_amber/vector_binary_element_wise.rb +506 -0
- data/lib/red_amber/vector_selectable.rb +265 -23
- data/lib/red_amber/vector_unary_element_wise.rb +529 -0
- data/lib/red_amber/vector_updatable.rb +278 -34
- data/lib/red_amber/version.rb +2 -1
- data/lib/red_amber.rb +13 -1
- data/red_amber.gemspec +2 -2
- metadata +13 -8
- data/doc/image/dataframe/reshaping_DataFrames.png +0 -0
- data/lib/red_amber/vector_functions.rb +0 -242
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 264e7637475fd01946900335751a1592a3859e9bfa772ecc0800ab05c4d852f0
|
|
4
|
+
data.tar.gz: a57400445419698a66d6b5c94e15fa8c040f2f3930f9fbf75603ffb6e18bd9cf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0fdbcdb732e36bb866a8251800ab3fa1a714fa075234bf8cd516f2542ab6704ebfa429a7177da2bd8cd6fa6eb1158efb0d68f46f43d1dc088a9a0f0debdc5c54
|
|
7
|
+
data.tar.gz: f9c1dffaa157ecf34b0b4fec6c1d7972b4773bbf7a11101a345172d621753cd9fc3818753b329dd2906a506af294d6a96c0180a0fb4dc84c2b54bceef6b520f5
|
data/.rubocop.yml
CHANGED
|
@@ -31,16 +31,28 @@ Style/TrailingCommaInHashLiteral:
|
|
|
31
31
|
|
|
32
32
|
# To let you know the possibility of refactoring ===
|
|
33
33
|
|
|
34
|
-
#
|
|
35
|
-
|
|
34
|
+
# Max: 120
|
|
35
|
+
# This cop supports safe autocorrection (--autocorrect).
|
|
36
|
+
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, IgnoredPatterns.
|
|
37
|
+
# URISchemes: http, https
|
|
38
|
+
Layout/LineLength:
|
|
39
|
+
Max: 90
|
|
36
40
|
Exclude:
|
|
37
41
|
- 'test/**/*'
|
|
38
42
|
|
|
43
|
+
# EnforcedStyle: aligned
|
|
44
|
+
Layout/MultilineMethodCallIndentation:
|
|
45
|
+
EnforcedStyle: indented_relative_to_receiver
|
|
46
|
+
|
|
39
47
|
# Disabled to define Vector operators
|
|
40
48
|
# Offense count: 38
|
|
41
49
|
Lint/BinaryOperatorWithIdenticalOperands:
|
|
42
50
|
Exclude:
|
|
43
|
-
- 'test/
|
|
51
|
+
- 'test/test_vector_binary_element_wise.rb'
|
|
52
|
+
|
|
53
|
+
Lint/Debugger:
|
|
54
|
+
Exclude:
|
|
55
|
+
- 'bin/example'
|
|
44
56
|
|
|
45
57
|
# Need for test with empty block
|
|
46
58
|
# Offense count: 1
|
|
@@ -55,15 +67,6 @@ Lint/UselessAssignment:
|
|
|
55
67
|
Exclude:
|
|
56
68
|
- 'test/**/*'
|
|
57
69
|
|
|
58
|
-
# Max: 120
|
|
59
|
-
# This cop supports safe autocorrection (--autocorrect).
|
|
60
|
-
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, IgnoredPatterns.
|
|
61
|
-
# URISchemes: http, https
|
|
62
|
-
Layout/LineLength:
|
|
63
|
-
Max: 90
|
|
64
|
-
Exclude:
|
|
65
|
-
- 'test/**/*'
|
|
66
|
-
|
|
67
70
|
# <= 17 satisfactory
|
|
68
71
|
# 18..30 unsatisfactory
|
|
69
72
|
# > 30 dangerous
|
|
@@ -83,6 +86,7 @@ Metrics/AbcSize:
|
|
|
83
86
|
'drop', # 31.42
|
|
84
87
|
'[]', # 33.76
|
|
85
88
|
'split', # 37.35
|
|
89
|
+
'aggregate', # 38.13
|
|
86
90
|
]
|
|
87
91
|
|
|
88
92
|
# Max: 25
|
|
@@ -104,7 +108,9 @@ Metrics/ClassLength:
|
|
|
104
108
|
- 'test/**/*'
|
|
105
109
|
- 'lib/red_amber/data_frame.rb' # 162
|
|
106
110
|
- 'lib/red_amber/group.rb' # 105
|
|
111
|
+
- 'lib/red_amber/subframes.rb' # 110
|
|
107
112
|
- 'lib/red_amber/vector.rb' # 152
|
|
113
|
+
- 'lib/red_amber/vector_binary_element_wise.rb' # 109
|
|
108
114
|
|
|
109
115
|
# Only for monitoring. I will measure by PerceivedComplexity.
|
|
110
116
|
# Max: 7
|
|
@@ -113,14 +119,17 @@ Metrics/ClassLength:
|
|
|
113
119
|
Metrics/CyclomaticComplexity:
|
|
114
120
|
Max: 12
|
|
115
121
|
AllowedMethods: [
|
|
116
|
-
'
|
|
122
|
+
'split', # 33
|
|
117
123
|
'format_table', # 21
|
|
124
|
+
'normalize_element', # 17
|
|
118
125
|
'slice_by', # 16
|
|
126
|
+
'assign_update', # 14
|
|
127
|
+
'join', # 14
|
|
128
|
+
'parse_range', # 14
|
|
119
129
|
'remove', # 14
|
|
120
|
-
'normalize_element', # 17
|
|
121
130
|
'[]', # 13
|
|
122
|
-
'
|
|
123
|
-
'
|
|
131
|
+
'drop', # 13
|
|
132
|
+
'aggregate', # 13
|
|
124
133
|
]
|
|
125
134
|
|
|
126
135
|
# Max: 10
|
|
@@ -134,6 +143,8 @@ Metrics/MethodLength:
|
|
|
134
143
|
'format_table', # 53
|
|
135
144
|
'slice_by', # 38
|
|
136
145
|
'assign_update', # 35
|
|
146
|
+
'drop', # 32
|
|
147
|
+
'aggregate', # 31
|
|
137
148
|
]
|
|
138
149
|
|
|
139
150
|
# Max: 100
|
|
@@ -163,17 +174,24 @@ Metrics/ParameterLists:
|
|
|
163
174
|
Metrics/PerceivedComplexity:
|
|
164
175
|
Max: 10
|
|
165
176
|
AllowedMethods: [
|
|
166
|
-
'join', # 14
|
|
167
|
-
'dataframe_info', # 13
|
|
168
177
|
'format_table', # 22
|
|
169
178
|
'slice_by', # 20
|
|
170
|
-
'remove', # 14
|
|
171
|
-
'drop', # 12
|
|
172
|
-
'filters', # 11
|
|
173
179
|
'normalize_element', # 17
|
|
174
|
-
'
|
|
180
|
+
'assign_update', # 15
|
|
175
181
|
'parse_range', # 15
|
|
182
|
+
'join', # 14
|
|
183
|
+
'remove', # 14
|
|
176
184
|
'split', # 14
|
|
185
|
+
'dataframe_info', # 13
|
|
186
|
+
'replace', # 13
|
|
187
|
+
'drop', # 12
|
|
188
|
+
'initialize', # 12
|
|
189
|
+
'aggregate', # 12
|
|
190
|
+
'[]', # 11
|
|
191
|
+
'filters', # 11
|
|
192
|
+
'html_table', # 11
|
|
193
|
+
'slice', # 11
|
|
194
|
+
'pick', # 11
|
|
177
195
|
]
|
|
178
196
|
|
|
179
197
|
# Offense count: 1
|
|
@@ -197,6 +215,18 @@ Naming/PredicateName:
|
|
|
197
215
|
- 'lib/red_amber/vector_functions.rb'
|
|
198
216
|
- 'lib/red_amber/vector_selectable.rb'
|
|
199
217
|
|
|
218
|
+
# avoid unused variable asignment
|
|
219
|
+
Rubycw/Rubycw:
|
|
220
|
+
Exclude:
|
|
221
|
+
- 'test/**/*'
|
|
222
|
+
- 'bin/example'
|
|
223
|
+
|
|
224
|
+
# Offense count: 16
|
|
225
|
+
# This cop supports safe autocorrection (--autocorrect).
|
|
226
|
+
Style/OperatorMethodCall:
|
|
227
|
+
Exclude:
|
|
228
|
+
- 'test/test_vector_binary_element_wise.rb'
|
|
229
|
+
|
|
200
230
|
# Necessary to test when range.end == -1
|
|
201
231
|
# Offense count: 2
|
|
202
232
|
# This cop supports unsafe autocorrection (--autocorrect-all).
|
|
@@ -204,6 +234,10 @@ Style/SlicingWithRange:
|
|
|
204
234
|
Exclude:
|
|
205
235
|
- 'test/test_data_frame_selectable.rb'
|
|
206
236
|
|
|
237
|
+
Style/MixinUsage:
|
|
238
|
+
Exclude:
|
|
239
|
+
- 'bin/example'
|
|
240
|
+
|
|
207
241
|
# Necessary to Vector < 0 element-wise comparison
|
|
208
242
|
# Offense count: 5
|
|
209
243
|
# This cop supports unsafe autocorrection (--autocorrect-all).
|
data/.yardopts
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,181 @@
|
|
|
1
|
+
## [0.4.1] - 2023-03-11
|
|
2
|
+
|
|
3
|
+
- Breaking change
|
|
4
|
+
- Remove Vector.aggregate? method (#200)
|
|
5
|
+
|
|
6
|
+
- Bug fixes
|
|
7
|
+
- Return self in DataFrame#drop when dropper is empty (reverts 746ac263) (#193)
|
|
8
|
+
- Return self in DataFrame#rename when renaming to same name (#193)
|
|
9
|
+
- Return self in DataFrame#pick when pick itself (#199)
|
|
10
|
+
- Fix column width for non-ascii elemnts in DataFrame#to_s (#193)
|
|
11
|
+
- This change uses String#width.
|
|
12
|
+
- Fix DataFrame#to_iruby when data is date32 type (#193)
|
|
13
|
+
- Fix DataFrame#shorthand to show temporal type data simply (#193)
|
|
14
|
+
- Fix Vector#rank when data is ChunkedArray (#198)
|
|
15
|
+
- Fix Vector element-wise functions with nil as scalar (#198)
|
|
16
|
+
- Support :force_order for all methods of join family (#199)
|
|
17
|
+
- Supports :force_order option to force sorting after join for all #join familiy.
|
|
18
|
+
- This will valuable in some cases such as large dataframes.
|
|
19
|
+
- Ensure baseframe's schema for SubFrames (#200)
|
|
20
|
+
|
|
21
|
+
- New features and improvements
|
|
22
|
+
- Add Vector#first, #last method (#198)
|
|
23
|
+
- This method will be used in SubFrames feature.
|
|
24
|
+
- Add Vector#modulo method (#198)
|
|
25
|
+
- The divmod function in Arrow C++ is still in draft state.
|
|
26
|
+
This method was created by combining existing functions
|
|
27
|
+
- Add Vector#quotient method (#198)
|
|
28
|
+
- Add aliases #div, #mod, #mul, #pow, #quo and #sub for Vector (#198)
|
|
29
|
+
- Add Vector#*_checked functions (#198)
|
|
30
|
+
- This functions will check numeric range overflow.
|
|
31
|
+
- Add 'tdra' and 'plain' in display mode (#193)
|
|
32
|
+
- The plain mode and default inspect will show up to 128 rows and 128 columns.
|
|
33
|
+
- Add String#width method in refinements (#193)
|
|
34
|
+
- This will be used to update DataFrame#to_s.
|
|
35
|
+
- Introduce pre-loaded REPL environment (#199)
|
|
36
|
+
- This commit will add bin/example and it will start irb environment
|
|
37
|
+
with enabled commonly used datasets such as penguins, diamonds, etc.
|
|
38
|
+
- Upgrade SubFrames#aggregate to accept block (#200)
|
|
39
|
+
|
|
40
|
+
- Refactoring
|
|
41
|
+
- Use symbolized keys in refinements of Table#keys, #key? (#193)
|
|
42
|
+
- This can be treat Tables and DataFrames as same manner.
|
|
43
|
+
- Use key_name.succ in suffix of DataFrame#join (#193)
|
|
44
|
+
- This will make simple to get name candidate.
|
|
45
|
+
- Use ||= to memorize instance variables (#193)
|
|
46
|
+
- Refine vector projection to use #variables (#193)
|
|
47
|
+
- #variables is fastest when picking Vectors.
|
|
48
|
+
- Refine Vector#is_in to avoid #pack (#198)
|
|
49
|
+
- Refine Vector#index (#198)
|
|
50
|
+
|
|
51
|
+
- Improve in tests/CI
|
|
52
|
+
- Tests
|
|
53
|
+
- Update benchmarks to test from older version (#193)
|
|
54
|
+
- Refine test of Vector function with scalar (#198)
|
|
55
|
+
- Refine test subframes and test_vector_selectable (#200)
|
|
56
|
+
|
|
57
|
+
- Cops
|
|
58
|
+
- CI
|
|
59
|
+
|
|
60
|
+
- Documentation
|
|
61
|
+
- Update documents(small fix) (#201)
|
|
62
|
+
|
|
63
|
+
- GitHub site
|
|
64
|
+
|
|
65
|
+
- Thanks
|
|
66
|
+
|
|
67
|
+
## [0.4.0] - 2023-02-25
|
|
68
|
+
|
|
69
|
+
- Breaking change
|
|
70
|
+
- Upgrade dependency to Arrow 11.0.0 (#188)
|
|
71
|
+
|
|
72
|
+
- Bug fixes
|
|
73
|
+
- Add :force_order option for DataFrame#join (#174)
|
|
74
|
+
- Return error for empty DataFrame in DataFrame#filter (#172)
|
|
75
|
+
- Accept ChunkedArray in DataFrame#filter (#172)
|
|
76
|
+
- Fix Vector#replace to accept Arrow::Array as a replacer (#179)
|
|
77
|
+
- Fix Vector#round_to_multiple to accept Float or Integer (#180)
|
|
78
|
+
- Change Vector atan2 to a class method (#180)
|
|
79
|
+
- Fix Vector#shift when boolean Vector (#184)
|
|
80
|
+
- Fix processing empty SubFrames (#183)
|
|
81
|
+
- Do not check object id in DataFrame#rename, #drop for self (#188)
|
|
82
|
+
|
|
83
|
+
- New features and improvements
|
|
84
|
+
- Accept a block in DataFrame#filter (#172)
|
|
85
|
+
- Add Vector.aggregate? method (#175)
|
|
86
|
+
- Introduce Vector#propagate method (#175)
|
|
87
|
+
- Add Vector#rank methods (#176)
|
|
88
|
+
- Add Vector#sample method (#176)
|
|
89
|
+
- Add Vector#sort method (#176)
|
|
90
|
+
- Promote DataFrame#shape_str to public (#184)
|
|
91
|
+
- Introduce Vector#concatenate (#184)
|
|
92
|
+
- Add #numeric? in refinements of Array (#184)
|
|
93
|
+
- Add Vector#cumulative_sum_checked and #cumsum (#184)
|
|
94
|
+
- Add Vector#resolve method (#184)
|
|
95
|
+
- Add DataFrame#tdra method (#184)
|
|
96
|
+
- Add #expand as an alias for Vector#propagate (#184)
|
|
97
|
+
- Add #glimpse as an alias for DataFrame#tdr (#184)
|
|
98
|
+
- New class SubFrames (#183)
|
|
99
|
+
- Introduce class SubFrames
|
|
100
|
+
- Memorize dataframes in SubFrames
|
|
101
|
+
- Add @frames to memorize sub DataFrames
|
|
102
|
+
- Accept filters in SubFrames.new
|
|
103
|
+
- Accept block in SubFrames.new
|
|
104
|
+
- Add SubFrames.by_filter
|
|
105
|
+
- Introduce methods creating SubFrames from DataFrame
|
|
106
|
+
- Introduce SubFrames#each method
|
|
107
|
+
- Add SubFrames#to_s method
|
|
108
|
+
- Add SubFrames#concatenate method
|
|
109
|
+
- Add SubFrames#offset_indices method
|
|
110
|
+
- SubFrames#aggregate method
|
|
111
|
+
- Redefine SubFrames#map to return SubFrames
|
|
112
|
+
- Define SubFrame#map dynamically
|
|
113
|
+
- Add SubFrames#assign method
|
|
114
|
+
- Redefine SubFrames#select to return SubFrames
|
|
115
|
+
- Add SubFrames#reject method
|
|
116
|
+
- Add SubFrames#filter_map method
|
|
117
|
+
- Refine DataFrame#indices memorizing @indices
|
|
118
|
+
- Rename SubFrames#universal_frame as #baseframe
|
|
119
|
+
- Set Group iteration feature to @api private
|
|
120
|
+
|
|
121
|
+
- Refactoring
|
|
122
|
+
- Generate Vector functions in class method (#177)
|
|
123
|
+
- Set Constant visibility to private (#179)
|
|
124
|
+
- Separate test_vector_function (#179)
|
|
125
|
+
- Relocate methods in DataFrameIndexable (#179)
|
|
126
|
+
- Rename Array refinements to the same name as Vector (#184)
|
|
127
|
+
|
|
128
|
+
- Improve in tests/CI
|
|
129
|
+
- Tests
|
|
130
|
+
- Update benchmarks to set 0.3.0 as a reference (#167)
|
|
131
|
+
- Move test of Vector#logb to proper location (#180)
|
|
132
|
+
|
|
133
|
+
- Cops
|
|
134
|
+
- Update .rubocop.yml to align with latest cops (#174)
|
|
135
|
+
- Unify style of MethodCallIndentation as relative to reciever (#184)
|
|
136
|
+
|
|
137
|
+
- CI
|
|
138
|
+
- Fix setting up Arrow by homebrew in CI (#167)
|
|
139
|
+
- Fix CI error on homebrew deleting python link (#167)
|
|
140
|
+
- Set cache-version to get new C extensions in CI (#173)
|
|
141
|
+
- Thanks to @kou for suggestion.
|
|
142
|
+
|
|
143
|
+
- Documentation
|
|
144
|
+
- Update DataFrame.md about loading csv without headers (#165)
|
|
145
|
+
- Thanks to kojix2
|
|
146
|
+
- Update YARD in DataFrame combinable (#168)
|
|
147
|
+
- Update comment for Ruby 2.7 support in README.md
|
|
148
|
+
- Update license year
|
|
149
|
+
- Update README (#172)
|
|
150
|
+
- Update Vector.md and yardoc in #propagate (#175)
|
|
151
|
+
- Use customized style sheet for YARD (#179)
|
|
152
|
+
- Add examples for the doc of #pick and #drop (#179)
|
|
153
|
+
- Add examples to YARD in DataFrame reshaping methods (#179)
|
|
154
|
+
- Update documents in DataFrameDisplayable (#179)
|
|
155
|
+
- Update documents in DataFrameVariableOperation (#179)
|
|
156
|
+
- Update document for dynamically generated methods (#179)
|
|
157
|
+
- Unify style in document (#179)
|
|
158
|
+
- Update documents in DataFrameSelectable (#179)
|
|
159
|
+
- Update documents of basic Vector methods (#179)
|
|
160
|
+
- Update document in VectorUpdatable (#179)
|
|
161
|
+
- Update document of Group (#179)
|
|
162
|
+
- Update document of DataFrameLoadSave (#180)
|
|
163
|
+
- Add examples for document of ArrowFunction (#180)
|
|
164
|
+
- Update document of Vector_unary_aggregation (#180)
|
|
165
|
+
- Update document of Vector_unary_element_wise (#180)
|
|
166
|
+
- Update document of Vector_biary_element_wise (#180)
|
|
167
|
+
- Add documentation to give comparison of dataframes(#169)
|
|
168
|
+
- Thanks to Benson Muite
|
|
169
|
+
- Update documents for consistency of method indentation (#189)
|
|
170
|
+
- Update CHANGELOG (#189)
|
|
171
|
+
- Update README for 0.4.0 (#189)
|
|
172
|
+
|
|
173
|
+
- GitHub site
|
|
174
|
+
|
|
175
|
+
- Thanks
|
|
176
|
+
- kojix2
|
|
177
|
+
- Benson Muite
|
|
178
|
+
|
|
1
179
|
## [0.3.0] - 2022-12-18
|
|
2
180
|
|
|
3
181
|
- Breaking change
|
data/Gemfile
CHANGED
data/LICENSE
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
The MIT License (MIT)
|
|
2
2
|
|
|
3
|
-
Copyright (c) 2022 Hirokazu SUZUKI (heronshoes)
|
|
3
|
+
Copyright (c) 2022-2023 Hirokazu SUZUKI (heronshoes)
|
|
4
4
|
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
data/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# RedAmber
|
|
2
2
|
|
|
3
|
-
[](https://rubygems.org/gems/red_amber)
|
|
4
4
|
[](https://github.com/heronshoes/red_amber/actions/workflows/ci.yml)
|
|
5
5
|
[](https://codeclimate.com/github/heronshoes/red_amber/maintainability)
|
|
6
6
|
[](https://codeclimate.com/github/heronshoes/red_amber/test_coverage)
|
|
@@ -10,34 +10,32 @@
|
|
|
10
10
|
A simple dataframe library for Ruby.
|
|
11
11
|
|
|
12
12
|
- Powered by [Red Arrow](https://github.com/apache/arrow/tree/master/ruby/red-arrow)
|
|
13
|
-
[](https://gitter.im/red-data-tools/en)
|
|
13
|
+
[](https://gitter.im/red-data-tools/en) [](https://rubygems.org/gems/red-arrow)
|
|
14
14
|
- Inspired by the dataframe library [Rover-df](https://github.com/ankane/rover)
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
## Requirements
|
|
19
|
-
|
|
19
|
+
### Ruby
|
|
20
20
|
Supported Ruby version is >= 3.0 (since RedAmber 0.3.0).
|
|
21
|
+
- I decided to remove support for Ruby 2.7 without waiting for its EOL. See [Release note for v0.3.0](https://github.com/heronshoes/red_amber/discussions/162) for details.
|
|
21
22
|
|
|
22
|
-
|
|
23
|
-
|
|
23
|
+
### Libraries
|
|
24
24
|
```ruby
|
|
25
|
-
#
|
|
26
|
-
gem 'red-
|
|
27
|
-
|
|
28
|
-
gem 'red-parquet', '~> 10.0.0' # Optional, if you use IO from/to parquet
|
|
25
|
+
gem 'red-arrow', '~> 11.0.0' # Requires Apache Arrow (see installation below)
|
|
26
|
+
gem 'red-parquet', '~> 11.0.0' # Optional, if you use IO from/to parquet
|
|
29
27
|
gem 'rover-df', '~> 0.3.0' # Optional, if you use IO from/to Rover::DataFrame
|
|
30
28
|
```
|
|
31
29
|
|
|
32
30
|
## Installation
|
|
33
31
|
|
|
34
|
-
Install requirements before you install
|
|
32
|
+
Install requirements before you install RedAmber.
|
|
35
33
|
|
|
36
|
-
- Apache Arrow (~>
|
|
37
|
-
- Apache Arrow GLib (~>
|
|
38
|
-
- Apache Parquet GLib (~>
|
|
34
|
+
- Apache Arrow (~> 11.0.0)
|
|
35
|
+
- Apache Arrow GLib (~> 11.0.0)
|
|
36
|
+
- Apache Parquet GLib (~> 11.0.0) # If you use IO from/to parquet
|
|
39
37
|
|
|
40
|
-
|
|
38
|
+
See [Apache Arrow install document](https://arrow.apache.org/install/).
|
|
41
39
|
|
|
42
40
|
- Minimum installation example for the latest Ubuntu:
|
|
43
41
|
|
|
@@ -58,43 +56,44 @@ Install requirements before you install Red Amber.
|
|
|
58
56
|
sudo dnf -y install gcc-c++ libarrow-devel libarrow-glib-devel ruby-devel
|
|
59
57
|
```
|
|
60
58
|
|
|
61
|
-
- On macOS,
|
|
59
|
+
- On macOS, using Homebrew:
|
|
62
60
|
|
|
63
61
|
```
|
|
64
62
|
brew install apache-arrow
|
|
65
|
-
```
|
|
66
|
-
|
|
67
|
-
and GLib (C) package with:
|
|
68
|
-
|
|
69
|
-
```
|
|
70
63
|
brew install apache-arrow-glib
|
|
71
64
|
```
|
|
72
65
|
|
|
73
66
|
If you prepared Apache Arrow, add these lines to your Gemfile:
|
|
74
67
|
|
|
75
68
|
```ruby
|
|
76
|
-
gem 'red-arrow', '~>
|
|
69
|
+
gem 'red-arrow', '~> 11.0.0'
|
|
77
70
|
gem 'red_amber'
|
|
78
|
-
gem 'red-parquet', '~>
|
|
71
|
+
gem 'red-parquet', '~> 11.0.0' # Optional, if you use IO from/to parquet
|
|
79
72
|
gem 'rover-df', '~> 0.3.0' # Optional, if you use IO from/to Rover::DataFrame
|
|
80
73
|
gem 'red-datasets-arrow' # Optional, recommended if you use Red Datasets
|
|
81
74
|
gem 'red-arrow-numo-narray' # Optional, recommended if you use inputs from Numo::NArray
|
|
82
75
|
```
|
|
83
76
|
|
|
84
|
-
And then execute `bundle install` or install
|
|
77
|
+
And then execute `bundle install` or install them yourself such as `gem install red_amber`.
|
|
85
78
|
|
|
86
79
|
## Docker image and Jupyter Notebook
|
|
87
80
|
|
|
88
|
-
[RubyData Docker Stacks](https://github.com/RubyData/docker-stacks) is available as a ready-to-run Docker image containing Jupyter and useful data tools as well as RedAmber (Thanks to
|
|
81
|
+
[RubyData Docker Stacks](https://github.com/RubyData/docker-stacks) is available as a ready-to-run Docker image containing Jupyter and useful data tools as well as RedAmber (Thanks to Kenta Murata).
|
|
89
82
|
|
|
90
83
|
Also you can try the contents of this README interactively by [Binder](https://mybinder.org/v2/gh/heronshoes/docker-stacks/RedAmber-binder?filepath=red-amber.ipynb).
|
|
91
84
|
[](https://mybinder.org/v2/gh/heronshoes/docker-stacks/RedAmber-binder?filepath=red-amber.ipynb)
|
|
92
85
|
|
|
86
|
+
## Comparison of DataFrames
|
|
87
|
+
|
|
88
|
+
Comparison of basic features of RedAmber with Python
|
|
89
|
+
[pandas](https://pandas.pydata.org/),
|
|
90
|
+
R [Tidyverse](https://www.tidyverse.org/) and
|
|
91
|
+
Julia [Dataframes](https://dataframes.juliadata.org/stable/) is in [DataFrame_Comparison.md](doc/DataFrame_Comparison.md) (Thanks to Benson Muite).
|
|
93
92
|
|
|
94
93
|
## Data frame in `RedAmber`
|
|
95
94
|
|
|
96
95
|
Class `RedAmber::DataFrame` represents a set of data in 2D-shape.
|
|
97
|
-
|
|
96
|
+
Its entity is a Red Arrow's Table object.
|
|
98
97
|
|
|
99
98
|
|
|
100
99
|
|
|
@@ -116,7 +115,7 @@ then
|
|
|
116
115
|
require 'datasets-arrow' # to load sample data
|
|
117
116
|
|
|
118
117
|
dataset = Datasets::Diamonds.new
|
|
119
|
-
diamonds = DataFrame.new(dataset) #
|
|
118
|
+
diamonds = DataFrame.new(dataset) # before v0.2.3, should be `dataset.to_arrow`
|
|
120
119
|
|
|
121
120
|
# =>
|
|
122
121
|
#<RedAmber::DataFrame : 53940 x 10 Vectors, 0x000000000000f668>
|
|
@@ -137,7 +136,7 @@ For example, we can compute mean prices per cut for the data larger than 1 carat
|
|
|
137
136
|
|
|
138
137
|
```ruby
|
|
139
138
|
df = diamonds
|
|
140
|
-
.slice { carat > 1 }
|
|
139
|
+
.slice { carat > 1 } # or use #filter instead of #slice
|
|
141
140
|
.group(:cut)
|
|
142
141
|
.mean(:price) # `pick` prior to `group` is not required if `:price` is specified here.
|
|
143
142
|
.sort('-mean(price)')
|
|
@@ -175,7 +174,7 @@ df.rename('mean(price)': :mean_price_USD)
|
|
|
175
174
|
|
|
176
175
|
### Example: starwars dataset
|
|
177
176
|
|
|
178
|
-
Next example is `starwars` dataset reading from the downloaded CSV file. Followed by minimum data
|
|
177
|
+
Next example is `starwars` dataset reading from the downloaded CSV file. Followed by minimum data cleaning.
|
|
179
178
|
|
|
180
179
|
```ruby
|
|
181
180
|
uri = URI('https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/starwars.csv')
|
|
@@ -186,7 +185,7 @@ starwars
|
|
|
186
185
|
.drop(0) # delete unnecessary index column
|
|
187
186
|
.remove { species == "NA" } # delete unnecessary rows
|
|
188
187
|
.group(:species) { [count(:species), mean(:height, :mass)] }
|
|
189
|
-
.slice { count > 1 }
|
|
188
|
+
.slice { count > 1 } # or use #filter instead of slice
|
|
190
189
|
|
|
191
190
|
# =>
|
|
192
191
|
#<RedAmber::DataFrame : 8 x 4 Vectors, 0x000000000000f848>
|
|
@@ -213,7 +212,7 @@ See [Vector.md](doc/Vector.md) for details.
|
|
|
213
212
|
|
|
214
213
|
## Jupyter notebook
|
|
215
214
|
|
|
216
|
-
[
|
|
215
|
+
[Examples of Red Amber](https://github.com/heronshoes/docker-stacks/blob/RedAmber-binder/binder/examples_of_red_amber.ipynb)
|
|
217
216
|
([raw file](https://raw.githubusercontent.com/heronshoes/docker-stacks/RedAmber-binder/binder/examples_of_red_amber.ipynb)) shows more examples in jupyter notebook.
|
|
218
217
|
|
|
219
218
|
You can try this notebook on [Binder](https://mybinder.org/v2/gh/heronshoes/docker-stacks/RedAmber-binder?filepath=examples_of_red_amber.ipynb).
|
data/benchmark/basic.yml
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
loop_count: 3
|
|
2
2
|
|
|
3
3
|
contexts:
|
|
4
|
-
- name:
|
|
5
|
-
prelude: |
|
|
6
|
-
$LOAD_PATH.unshift(File.expand_path('lib'))
|
|
7
|
-
- name: 0.2.3
|
|
4
|
+
- name: 0.1.5
|
|
8
5
|
gems:
|
|
9
|
-
red_amber: 0.
|
|
6
|
+
red_amber: 0.1.5
|
|
10
7
|
- name: 0.2.0
|
|
11
8
|
gems:
|
|
12
9
|
red_amber: 0.2.0
|
|
13
|
-
- name: 0.
|
|
10
|
+
- name: 0.3.0
|
|
14
11
|
gems:
|
|
15
|
-
red_amber: 0.
|
|
12
|
+
red_amber: 0.3.0
|
|
13
|
+
- name: HEAD
|
|
14
|
+
prelude: |
|
|
15
|
+
$LOAD_PATH.unshift(File.expand_path('lib'))
|
|
16
16
|
|
|
17
17
|
prelude: |
|
|
18
18
|
require 'red_amber'
|
data/benchmark/combine.yml
CHANGED
data/benchmark/dataframe.yml
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
loop_count: 3
|
|
2
2
|
|
|
3
3
|
contexts:
|
|
4
|
-
- name: HEAD
|
|
5
|
-
prelude: |
|
|
6
|
-
$LOAD_PATH.unshift(File.expand_path('lib'))
|
|
7
|
-
- name: 0.2.3
|
|
8
|
-
gems:
|
|
9
|
-
red_amber: 0.2.3
|
|
10
4
|
- name: 0.2.0
|
|
11
5
|
gems:
|
|
12
6
|
red_amber: 0.2.0
|
|
7
|
+
- name: 0.3.0
|
|
8
|
+
gems:
|
|
9
|
+
red_amber: 0.3.0
|
|
10
|
+
- name: HEAD
|
|
11
|
+
prelude: |
|
|
12
|
+
$LOAD_PATH.unshift(File.expand_path('lib'))
|
|
13
13
|
|
|
14
14
|
prelude: |
|
|
15
15
|
require 'red_amber'
|
|
@@ -19,8 +19,14 @@ prelude: |
|
|
|
19
19
|
|
|
20
20
|
starwars = RedAmber::DataFrame.new(Datasets::Rdataset.new('dplyr', 'starwars').to_arrow)
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
import_cars = RedAmber::DataFrame.load(Arrow::Buffer.new(<<~TSV), format: :tsv)
|
|
23
|
+
Year Audi BMW BMW_MINI Mercedes-Benz VW
|
|
24
|
+
2017 28336 52527 25427 68221 49040
|
|
25
|
+
2018 26473 50982 25984 67554 51961
|
|
26
|
+
2019 24222 46814 23813 66553 46794
|
|
27
|
+
2020 22304 35712 20196 57041 36576
|
|
28
|
+
2021 22535 35905 18211 51722 35215
|
|
29
|
+
TSV
|
|
24
30
|
|
|
25
31
|
ds = Datasets::Rdataset.new('openintro', 'simpsons_paradox_covid')
|
|
26
32
|
simpsons_paradox_covid = RedAmber::DataFrame.new(ds.to_arrow)
|
|
@@ -43,7 +49,7 @@ benchmark:
|
|
|
43
49
|
.group(:species) { [count(:species), mean(:height, :mass)] }
|
|
44
50
|
.slice { v(:count) > 1 }
|
|
45
51
|
|
|
46
|
-
'D03:
|
|
52
|
+
'D03: Import cars test': |
|
|
47
53
|
import_cars
|
|
48
54
|
.to_long(:Year, name: :Manufacturer, value: :Num_of_imported)
|
|
49
55
|
.to_wide(name: :Manufacturer, value: :Num_of_imported)
|
data/benchmark/group.yml
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
loop_count: 3
|
|
2
2
|
|
|
3
3
|
contexts:
|
|
4
|
-
- name: HEAD
|
|
5
|
-
prelude: |
|
|
6
|
-
$LOAD_PATH.unshift(File.expand_path('lib'))
|
|
7
|
-
- name: 0.2.3
|
|
8
|
-
gems:
|
|
9
|
-
red_amber: 0.2.3
|
|
10
4
|
- name: 0.2.2
|
|
11
5
|
gems:
|
|
12
6
|
red_amber: 0.2.2
|
|
7
|
+
- name: 0.3.0
|
|
8
|
+
gems:
|
|
9
|
+
red_amber: 0.3.0
|
|
10
|
+
- name: HEAD
|
|
11
|
+
prelude: |
|
|
12
|
+
$LOAD_PATH.unshift(File.expand_path('lib'))
|
|
13
13
|
|
|
14
14
|
prelude: |
|
|
15
15
|
require 'red_amber'
|
data/benchmark/reshape.yml
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
loop_count: 3
|
|
2
2
|
|
|
3
3
|
contexts:
|
|
4
|
-
- name: HEAD
|
|
5
|
-
prelude: |
|
|
6
|
-
$LOAD_PATH.unshift(File.expand_path('lib'))
|
|
7
|
-
- name: 0.2.3
|
|
8
|
-
gems:
|
|
9
|
-
red_amber: 0.2.3
|
|
10
4
|
- name: 0.2.2
|
|
11
5
|
gems:
|
|
12
6
|
red_amber: 0.2.2
|
|
7
|
+
- name: 0.3.0
|
|
8
|
+
gems:
|
|
9
|
+
red_amber: 0.3.0
|
|
10
|
+
- name: HEAD
|
|
11
|
+
prelude: |
|
|
12
|
+
$LOAD_PATH.unshift(File.expand_path('lib'))
|
|
13
13
|
|
|
14
14
|
prelude: |
|
|
15
15
|
require 'red_amber'
|
data/benchmark/vector.yml
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
loop_count: 10
|
|
2
2
|
|
|
3
3
|
contexts:
|
|
4
|
-
- name: HEAD
|
|
5
|
-
prelude: |
|
|
6
|
-
$LOAD_PATH.unshift(File.expand_path('lib'))
|
|
7
4
|
- name: 0.2.0
|
|
8
5
|
gems:
|
|
9
6
|
red_amber: 0.2.0
|
|
7
|
+
- name: 0.3.0
|
|
8
|
+
gems:
|
|
9
|
+
red_amber: 0.3.0
|
|
10
|
+
- name: HEAD
|
|
11
|
+
prelude: |
|
|
12
|
+
$LOAD_PATH.unshift(File.expand_path('lib'))
|
|
10
13
|
|
|
11
14
|
prelude: |
|
|
12
15
|
require 'red_amber'
|