red-arrow 0.4.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -2
- data/dependency-check/Rakefile +1 -1
- data/doc/text/news.md +37 -1
- data/image/red-arrow.png +0 -0
- data/lib/arrow/array-builder.rb +38 -34
- data/lib/arrow/array.rb +11 -5
- data/lib/arrow/chunked-array.rb +18 -2
- data/lib/arrow/column.rb +4 -0
- data/lib/arrow/csv-reader.rb +162 -0
- data/lib/arrow/loader.rb +14 -12
- data/lib/arrow/slicer.rb +391 -0
- data/lib/arrow/table-formatter.rb +88 -0
- data/lib/arrow/table.rb +309 -3
- data/lib/arrow/timestamp-array.rb +69 -0
- data/lib/arrow/version.rb +1 -1
- data/red-arrow.gemspec +1 -0
- data/test/fixture/with-header.csv +4 -0
- data/test/fixture/without-header.csv +3 -0
- data/test/helper.rb +3 -0
- data/test/helper/fixture.rb +25 -0
- data/test/test-array.rb +6 -0
- data/test/test-column.rb +2 -2
- data/test/test-csv-reader.rb +90 -0
- data/test/test-slicer.rb +401 -0
- data/test/test-table.rb +321 -13
- data/test/test-timestamp-array.rb +23 -0
- metadata +24 -7
data/test/test-table.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
|
1
|
+
# Copyright 2017-2018 Kouhei Sutou <kou@clear-code.com>
|
2
2
|
#
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
4
|
# you may not use this file except in compliance with the License.
|
@@ -13,20 +13,328 @@
|
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
15
|
class TableTest < Test::Unit::TestCase
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
16
|
+
def setup
|
17
|
+
@count_field = Arrow::Field.new("count", :uint32)
|
18
|
+
@visible_field = Arrow::Field.new("visible", :boolean)
|
19
|
+
schema = Arrow::Schema.new([@count_field, @visible_field])
|
20
|
+
count_arrays = [
|
21
|
+
Arrow::UInt32Array.new([1, 2]),
|
22
|
+
Arrow::UInt32Array.new([4, 8, 16]),
|
23
|
+
Arrow::UInt32Array.new([32, 64]),
|
24
|
+
Arrow::UInt32Array.new([128]),
|
20
25
|
]
|
21
|
-
|
22
|
-
|
23
|
-
Arrow::
|
24
|
-
|
25
|
-
Arrow::
|
26
|
-
|
26
|
+
visible_arrays = [
|
27
|
+
Arrow::BooleanArray.new([true, false, nil]),
|
28
|
+
Arrow::BooleanArray.new([true]),
|
29
|
+
Arrow::BooleanArray.new([true, false]),
|
30
|
+
Arrow::BooleanArray.new([nil]),
|
31
|
+
Arrow::BooleanArray.new([nil]),
|
27
32
|
]
|
28
|
-
|
33
|
+
@count_array = Arrow::ChunkedArray.new(count_arrays)
|
34
|
+
@visible_array = Arrow::ChunkedArray.new(visible_arrays)
|
35
|
+
@count_column = Arrow::Column.new(@count_field, @count_array)
|
36
|
+
@visible_column = Arrow::Column.new(@visible_field, @visible_array)
|
37
|
+
@table = Arrow::Table.new(schema, [@count_column, @visible_column])
|
38
|
+
end
|
39
|
+
|
40
|
+
test("#columns") do
|
29
41
|
assert_equal(["count", "visible"],
|
30
|
-
table.columns.collect(&:name))
|
42
|
+
@table.columns.collect(&:name))
|
43
|
+
end
|
44
|
+
|
45
|
+
sub_test_case("#slice") do
|
46
|
+
test("Arrow::BooleanArray") do
|
47
|
+
target_rows_raw = [nil, true, true, false, true, false, true, true]
|
48
|
+
target_rows = Arrow::BooleanArray.new(target_rows_raw)
|
49
|
+
assert_equal(<<-TABLE, @table.slice(target_rows).to_s)
|
50
|
+
count visible
|
51
|
+
0 2 false
|
52
|
+
1 4
|
53
|
+
2 16 true
|
54
|
+
3 64
|
55
|
+
4 128
|
56
|
+
TABLE
|
57
|
+
end
|
58
|
+
|
59
|
+
test("Integer") do
|
60
|
+
assert_equal(<<-TABLE, @table.slice(2).to_s)
|
61
|
+
count visible
|
62
|
+
0 4
|
63
|
+
TABLE
|
64
|
+
end
|
65
|
+
|
66
|
+
test("Range: include end") do
|
67
|
+
assert_equal(<<-TABLE, @table.slice(2..4).to_s)
|
68
|
+
count visible
|
69
|
+
0 4
|
70
|
+
1 8 true
|
71
|
+
2 16 true
|
72
|
+
TABLE
|
73
|
+
end
|
74
|
+
|
75
|
+
test("Range: exclude end") do
|
76
|
+
assert_equal(<<-TABLE, @table.slice(2...4).to_s)
|
77
|
+
count visible
|
78
|
+
0 4
|
79
|
+
1 8 true
|
80
|
+
TABLE
|
81
|
+
end
|
82
|
+
|
83
|
+
test("[from, to]") do
|
84
|
+
assert_equal(<<-TABLE, @table.slice([0, 2]).to_s)
|
85
|
+
count visible
|
86
|
+
0 1 true
|
87
|
+
1 2 false
|
88
|
+
TABLE
|
89
|
+
end
|
90
|
+
|
91
|
+
test("Integer, Range, ...") do
|
92
|
+
assert_equal(<<-TABLE, @table.slice(0, 4...7).to_s)
|
93
|
+
count visible
|
94
|
+
0 1 true
|
95
|
+
1 16 true
|
96
|
+
2 32 false
|
97
|
+
3 64
|
98
|
+
TABLE
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
sub_test_case("#[]") do
|
103
|
+
test("[String]") do
|
104
|
+
assert_equal(@count_column, @table["count"])
|
105
|
+
end
|
106
|
+
|
107
|
+
test("[Symbol]") do
|
108
|
+
assert_equal(@visible_column, @table[:visible])
|
109
|
+
end
|
110
|
+
|
111
|
+
test("[String, Symbol]") do
|
112
|
+
assert_equal(Arrow::Table.new(@table.schema,
|
113
|
+
[@visible_column, @count_column]).to_s,
|
114
|
+
@table["visible", :count].to_s)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
sub_test_case("#merge") do
|
119
|
+
sub_test_case("Hash") do
|
120
|
+
test("add") do
|
121
|
+
name_array = Arrow::StringArray.new(["a", "b", "c", "d", "e", "f", "g", "h"])
|
122
|
+
assert_equal(<<-TABLE, @table.merge(:name => name_array).to_s)
|
123
|
+
count visible name
|
124
|
+
0 1 true a
|
125
|
+
1 2 false b
|
126
|
+
2 4 c
|
127
|
+
3 8 true d
|
128
|
+
4 16 true e
|
129
|
+
5 32 false f
|
130
|
+
6 64 g
|
131
|
+
7 128 h
|
132
|
+
TABLE
|
133
|
+
end
|
134
|
+
|
135
|
+
test("remove") do
|
136
|
+
assert_equal(<<-TABLE, @table.merge(:visible => nil).to_s)
|
137
|
+
count
|
138
|
+
0 1
|
139
|
+
1 2
|
140
|
+
2 4
|
141
|
+
3 8
|
142
|
+
4 16
|
143
|
+
5 32
|
144
|
+
6 64
|
145
|
+
7 128
|
146
|
+
TABLE
|
147
|
+
end
|
148
|
+
|
149
|
+
test("replace") do
|
150
|
+
visible_array = Arrow::Int32Array.new([1] * @visible_array.length)
|
151
|
+
assert_equal(<<-TABLE, @table.merge(:visible => visible_array).to_s)
|
152
|
+
count visible
|
153
|
+
0 1 1
|
154
|
+
1 2 1
|
155
|
+
2 4 1
|
156
|
+
3 8 1
|
157
|
+
4 16 1
|
158
|
+
5 32 1
|
159
|
+
6 64 1
|
160
|
+
7 128 1
|
161
|
+
TABLE
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
sub_test_case("Arrow::Table") do
|
166
|
+
test("add") do
|
167
|
+
name_array = Arrow::StringArray.new(["a", "b", "c", "d", "e", "f", "g", "h"])
|
168
|
+
table = Arrow::Table.new("name" => name_array)
|
169
|
+
assert_equal(<<-TABLE, @table.merge(table).to_s)
|
170
|
+
count visible name
|
171
|
+
0 1 true a
|
172
|
+
1 2 false b
|
173
|
+
2 4 c
|
174
|
+
3 8 true d
|
175
|
+
4 16 true e
|
176
|
+
5 32 false f
|
177
|
+
6 64 g
|
178
|
+
7 128 h
|
179
|
+
TABLE
|
180
|
+
end
|
181
|
+
|
182
|
+
test("replace") do
|
183
|
+
visible_array = Arrow::Int32Array.new([1] * @visible_array.length)
|
184
|
+
table = Arrow::Table.new("visible" => visible_array)
|
185
|
+
assert_equal(<<-TABLE, @table.merge(table).to_s)
|
186
|
+
count visible
|
187
|
+
0 1 1
|
188
|
+
1 2 1
|
189
|
+
2 4 1
|
190
|
+
3 8 1
|
191
|
+
4 16 1
|
192
|
+
5 32 1
|
193
|
+
6 64 1
|
194
|
+
7 128 1
|
195
|
+
TABLE
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
test("column name getter") do
|
201
|
+
assert_equal(@visible_column, @table.visible)
|
202
|
+
end
|
203
|
+
|
204
|
+
sub_test_case("#remove_column") do
|
205
|
+
test("String") do
|
206
|
+
assert_equal(<<-TABLE, @table.remove_column("visible").to_s)
|
207
|
+
count
|
208
|
+
0 1
|
209
|
+
1 2
|
210
|
+
2 4
|
211
|
+
3 8
|
212
|
+
4 16
|
213
|
+
5 32
|
214
|
+
6 64
|
215
|
+
7 128
|
216
|
+
TABLE
|
217
|
+
end
|
218
|
+
|
219
|
+
test("Symbol") do
|
220
|
+
assert_equal(<<-TABLE, @table.remove_column(:visible).to_s)
|
221
|
+
count
|
222
|
+
0 1
|
223
|
+
1 2
|
224
|
+
2 4
|
225
|
+
3 8
|
226
|
+
4 16
|
227
|
+
5 32
|
228
|
+
6 64
|
229
|
+
7 128
|
230
|
+
TABLE
|
231
|
+
end
|
232
|
+
|
233
|
+
test("unknown column name") do
|
234
|
+
assert_raise(KeyError) do
|
235
|
+
@table.remove_column(:nonexistent)
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
test("Integer") do
|
240
|
+
assert_equal(<<-TABLE, @table.remove_column(1).to_s)
|
241
|
+
count
|
242
|
+
0 1
|
243
|
+
1 2
|
244
|
+
2 4
|
245
|
+
3 8
|
246
|
+
4 16
|
247
|
+
5 32
|
248
|
+
6 64
|
249
|
+
7 128
|
250
|
+
TABLE
|
251
|
+
end
|
252
|
+
|
253
|
+
test("negative integer") do
|
254
|
+
assert_equal(<<-TABLE, @table.remove_column(-1).to_s)
|
255
|
+
count
|
256
|
+
0 1
|
257
|
+
1 2
|
258
|
+
2 4
|
259
|
+
3 8
|
260
|
+
4 16
|
261
|
+
5 32
|
262
|
+
6 64
|
263
|
+
7 128
|
264
|
+
TABLE
|
265
|
+
end
|
266
|
+
|
267
|
+
test("too small index") do
|
268
|
+
assert_raise(IndexError) do
|
269
|
+
@table.remove_column(-3)
|
270
|
+
end
|
271
|
+
end
|
272
|
+
|
273
|
+
test("too large index") do
|
274
|
+
assert_raise(IndexError) do
|
275
|
+
@table.remove_column(2)
|
276
|
+
end
|
277
|
+
end
|
278
|
+
end
|
279
|
+
|
280
|
+
sub_test_case("#select_columns") do
|
281
|
+
def setup
|
282
|
+
raw_table = {
|
283
|
+
:a => Arrow::UInt32Array.new([1]),
|
284
|
+
:b => Arrow::UInt32Array.new([1]),
|
285
|
+
:c => Arrow::UInt32Array.new([1]),
|
286
|
+
:d => Arrow::UInt32Array.new([1]),
|
287
|
+
:e => Arrow::UInt32Array.new([1]),
|
288
|
+
}
|
289
|
+
@table = Arrow::Table.new(raw_table)
|
290
|
+
end
|
291
|
+
|
292
|
+
test("names") do
|
293
|
+
assert_equal(<<-TABLE, @table.select_columns(:c, :a).to_s)
|
294
|
+
c a
|
295
|
+
0 1 1
|
296
|
+
TABLE
|
297
|
+
end
|
298
|
+
|
299
|
+
test("range") do
|
300
|
+
assert_equal(<<-TABLE, @table.select_columns(2...4).to_s)
|
301
|
+
c d
|
302
|
+
0 1 1
|
303
|
+
TABLE
|
304
|
+
end
|
305
|
+
|
306
|
+
test("indexes") do
|
307
|
+
assert_equal(<<-TABLE, @table.select_columns(0, -1, 2).to_s)
|
308
|
+
a e c
|
309
|
+
0 1 1 1
|
310
|
+
TABLE
|
311
|
+
end
|
312
|
+
|
313
|
+
test("mixed") do
|
314
|
+
assert_equal(<<-TABLE, @table.select_columns(:a, -1, 2..3).to_s)
|
315
|
+
a e c d
|
316
|
+
0 1 1 1 1
|
317
|
+
TABLE
|
318
|
+
end
|
319
|
+
|
320
|
+
test("block") do
|
321
|
+
selected_table = @table.select_columns.with_index do |column, i|
|
322
|
+
column.name == "a" or i.odd?
|
323
|
+
end
|
324
|
+
assert_equal(<<-TABLE, selected_table.to_s)
|
325
|
+
a b d
|
326
|
+
0 1 1 1
|
327
|
+
TABLE
|
328
|
+
end
|
329
|
+
|
330
|
+
test("names, indexes and block") do
|
331
|
+
selected_table = @table.select_columns(:a, -1) do |column|
|
332
|
+
column.name == "a"
|
333
|
+
end
|
334
|
+
assert_equal(<<-TABLE, selected_table.to_s)
|
335
|
+
a
|
336
|
+
0 1
|
337
|
+
TABLE
|
338
|
+
end
|
31
339
|
end
|
32
340
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
class TimestampArrayTest < Test::Unit::TestCase
|
16
|
+
test("#[]") do
|
17
|
+
sec = 1513267750
|
18
|
+
usec = 914509
|
19
|
+
array = Arrow::TimestampArray.new(:micro, [sec * (10 ** 6) + usec])
|
20
|
+
time = Time.at(sec, usec)
|
21
|
+
assert_equal(time, array[0])
|
22
|
+
end
|
23
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: red-arrow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kouhei Sutou
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-01-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gobject-introspection
|
@@ -138,6 +138,7 @@ files:
|
|
138
138
|
- dependency-check/Rakefile
|
139
139
|
- doc/text/apache-2.0.txt
|
140
140
|
- doc/text/news.md
|
141
|
+
- image/red-arrow.png
|
141
142
|
- lib/arrow.rb
|
142
143
|
- lib/arrow/array-builder.rb
|
143
144
|
- lib/arrow/array.rb
|
@@ -146,25 +147,35 @@ files:
|
|
146
147
|
- lib/arrow/chunked-array.rb
|
147
148
|
- lib/arrow/column.rb
|
148
149
|
- lib/arrow/compatibility.rb
|
150
|
+
- lib/arrow/csv-reader.rb
|
149
151
|
- lib/arrow/field.rb
|
150
152
|
- lib/arrow/loader.rb
|
151
153
|
- lib/arrow/record-batch-file-reader.rb
|
152
154
|
- lib/arrow/record-batch-stream-reader.rb
|
153
155
|
- lib/arrow/record-batch.rb
|
154
156
|
- lib/arrow/record.rb
|
157
|
+
- lib/arrow/slicer.rb
|
158
|
+
- lib/arrow/table-formatter.rb
|
155
159
|
- lib/arrow/table.rb
|
156
160
|
- lib/arrow/tensor.rb
|
161
|
+
- lib/arrow/timestamp-array.rb
|
157
162
|
- lib/arrow/version.rb
|
158
163
|
- red-arrow.gemspec
|
164
|
+
- test/fixture/with-header.csv
|
165
|
+
- test/fixture/without-header.csv
|
159
166
|
- test/helper.rb
|
167
|
+
- test/helper/fixture.rb
|
160
168
|
- test/run-test.rb
|
161
169
|
- test/test-array-builder.rb
|
162
170
|
- test/test-array.rb
|
163
171
|
- test/test-chunked-array.rb
|
164
172
|
- test/test-column.rb
|
173
|
+
- test/test-csv-reader.rb
|
165
174
|
- test/test-record-batch-file-reader.rb
|
166
175
|
- test/test-record-batch.rb
|
176
|
+
- test/test-slicer.rb
|
167
177
|
- test/test-table.rb
|
178
|
+
- test/test-timestamp-array.rb
|
168
179
|
homepage: https://github.com/red-data-tools/red-arrow
|
169
180
|
licenses:
|
170
181
|
- Apache-2.0
|
@@ -185,18 +196,24 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
185
196
|
version: '0'
|
186
197
|
requirements: []
|
187
198
|
rubyforge_project:
|
188
|
-
rubygems_version: 2.5.2
|
199
|
+
rubygems_version: 2.5.2.2
|
189
200
|
signing_key:
|
190
201
|
specification_version: 4
|
191
202
|
summary: Red Arrow is a Ruby bindings of Apache Arrow. Red Arrow is based on GObject
|
192
203
|
Introspection.
|
193
204
|
test_files:
|
194
|
-
- test/test-chunked-array.rb
|
195
|
-
- test/test-table.rb
|
196
205
|
- test/test-record-batch-file-reader.rb
|
206
|
+
- test/fixture/with-header.csv
|
207
|
+
- test/fixture/without-header.csv
|
208
|
+
- test/test-csv-reader.rb
|
209
|
+
- test/test-column.rb
|
197
210
|
- test/test-array-builder.rb
|
198
|
-
- test/
|
211
|
+
- test/test-chunked-array.rb
|
199
212
|
- test/test-record-batch.rb
|
200
213
|
- test/helper.rb
|
201
|
-
- test/test
|
214
|
+
- test/run-test.rb
|
215
|
+
- test/test-timestamp-array.rb
|
216
|
+
- test/test-table.rb
|
202
217
|
- test/test-array.rb
|
218
|
+
- test/helper/fixture.rb
|
219
|
+
- test/test-slicer.rb
|