red-arrow 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/doc/text/development.md +15 -0
- data/doc/text/news.md +14 -0
- data/lib/arrow/array-builder.rb +60 -5
- data/lib/arrow/csv-loader.rb +206 -0
- data/lib/arrow/csv-reader.rb +6 -117
- data/lib/arrow/date32-array-builder.rb +29 -0
- data/lib/arrow/date32-array.rb +27 -0
- data/lib/arrow/date64-array-builder.rb +30 -0
- data/lib/arrow/date64-array.rb +26 -0
- data/lib/arrow/loader.rb +10 -1
- data/lib/arrow/record-batch.rb +11 -37
- data/lib/arrow/record-containable.rb +70 -0
- data/lib/arrow/record.rb +20 -5
- data/lib/arrow/table-loader.rb +117 -0
- data/lib/arrow/table-saver.rb +93 -0
- data/lib/arrow/table.rb +18 -25
- data/lib/arrow/timestamp-array-builder.rb +59 -0
- data/lib/arrow/version.rb +2 -2
- data/test/test-array-builder.rb +80 -42
- data/test/test-csv-loader.rb +79 -0
- data/test/test-csv-reader.rb +5 -66
- data/test/test-date32-array.rb +21 -0
- data/test/test-date64-array.rb +22 -0
- data/test/test-table.rb +64 -10
- metadata +18 -2
data/test/test-csv-reader.rb
CHANGED
@@ -15,76 +15,15 @@
|
|
15
15
|
class CSVReaderTest < Test::Unit::TestCase
|
16
16
|
include Helper::Fixture
|
17
17
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
assert_equal(<<-TABLE,
|
22
|
-
name score
|
23
|
-
0 alice 10
|
24
|
-
1 bob 29
|
25
|
-
2 chris -1
|
26
|
-
TABLE
|
27
|
-
end
|
28
|
-
|
29
|
-
test("String: data: without header") do
|
30
|
-
data = fixture_path("without-header.csv").read
|
31
|
-
assert_equal(<<-TABLE, Arrow::CSVReader.read(data).to_s)
|
32
|
-
0 1
|
33
|
-
0 alice 10
|
34
|
-
1 bob 29
|
35
|
-
2 chris -1
|
36
|
-
TABLE
|
37
|
-
end
|
38
|
-
|
39
|
-
test("String: path: with header") do
|
40
|
-
path = fixture_path("with-header.csv").to_s
|
41
|
-
assert_equal(<<-TABLE, Arrow::CSVReader.read(path).to_s)
|
42
|
-
name score
|
43
|
-
0 alice 10
|
44
|
-
1 bob 29
|
45
|
-
2 chris -1
|
46
|
-
TABLE
|
47
|
-
end
|
48
|
-
|
49
|
-
test("String: path: without header") do
|
50
|
-
path = fixture_path("without-header.csv").to_s
|
51
|
-
assert_equal(<<-TABLE, Arrow::CSVReader.read(path).to_s)
|
52
|
-
0 1
|
53
|
-
0 alice 10
|
54
|
-
1 bob 29
|
55
|
-
2 chris -1
|
56
|
-
TABLE
|
57
|
-
end
|
58
|
-
|
59
|
-
test("Pathname: with header") do
|
60
|
-
path = fixture_path("with-header.csv")
|
61
|
-
assert_equal(<<-TABLE, Arrow::CSVReader.read(path).to_s)
|
62
|
-
name score
|
63
|
-
0 alice 10
|
64
|
-
1 bob 29
|
65
|
-
2 chris -1
|
66
|
-
TABLE
|
67
|
-
end
|
68
|
-
|
69
|
-
test("Pathname: without header") do
|
70
|
-
path = fixture_path("without-header.csv")
|
71
|
-
assert_equal(<<-TABLE, Arrow::CSVReader.read(path).to_s)
|
72
|
-
0 1
|
73
|
-
0 alice 10
|
74
|
-
1 bob 29
|
75
|
-
2 chris -1
|
76
|
-
TABLE
|
77
|
-
end
|
78
|
-
|
79
|
-
test("CSV") do
|
80
|
-
CSV.open(fixture_path("with-header.csv").to_s, headers: true) do |csv|
|
81
|
-
assert_equal(<<-TABLE, Arrow::CSVReader.read(csv).to_s)
|
18
|
+
test("#read") do
|
19
|
+
CSV.open(fixture_path("with-header.csv").to_s, headers: true) do |csv|
|
20
|
+
reader = Arrow::CSVReader.new(csv)
|
21
|
+
assert_equal(<<-TABLE, reader.read.to_s)
|
82
22
|
name score
|
83
23
|
0 alice 10
|
84
24
|
1 bob 29
|
85
25
|
2 chris -1
|
86
|
-
|
87
|
-
end
|
26
|
+
TABLE
|
88
27
|
end
|
89
28
|
end
|
90
29
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
class Date32ArrayTest < Test::Unit::TestCase
|
16
|
+
test("#[]") do
|
17
|
+
n_days_since_epoch = 17406 # 2017-08-28
|
18
|
+
array = Arrow::Date32Array.new([n_days_since_epoch])
|
19
|
+
assert_equal(Date.new(2017, 8, 28), array[0])
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
class Date64ArrayTest < Test::Unit::TestCase
|
16
|
+
test("#[]") do
|
17
|
+
n_msecs_since_epoch = 1503878400000 # 2017-08-28T00:00:00Z
|
18
|
+
array = Arrow::Date64Array.new([n_msecs_since_epoch])
|
19
|
+
assert_equal(DateTime.new(2017, 8, 28, 0, 0, 0),
|
20
|
+
array[0])
|
21
|
+
end
|
22
|
+
end
|
data/test/test-table.rb
CHANGED
@@ -13,15 +13,17 @@
|
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
15
|
class TableTest < Test::Unit::TestCase
|
16
|
+
include Helper::Fixture
|
17
|
+
|
16
18
|
def setup
|
17
|
-
@count_field = Arrow::Field.new("count", :
|
19
|
+
@count_field = Arrow::Field.new("count", :uint8)
|
18
20
|
@visible_field = Arrow::Field.new("visible", :boolean)
|
19
21
|
schema = Arrow::Schema.new([@count_field, @visible_field])
|
20
22
|
count_arrays = [
|
21
|
-
Arrow::
|
22
|
-
Arrow::
|
23
|
-
Arrow::
|
24
|
-
Arrow::
|
23
|
+
Arrow::UInt8Array.new([1, 2]),
|
24
|
+
Arrow::UInt8Array.new([4, 8, 16]),
|
25
|
+
Arrow::UInt8Array.new([32, 64]),
|
26
|
+
Arrow::UInt8Array.new([128]),
|
25
27
|
]
|
26
28
|
visible_arrays = [
|
27
29
|
Arrow::BooleanArray.new([true, false, nil]),
|
@@ -280,11 +282,11 @@ class TableTest < Test::Unit::TestCase
|
|
280
282
|
sub_test_case("#select_columns") do
|
281
283
|
def setup
|
282
284
|
raw_table = {
|
283
|
-
:a => Arrow::
|
284
|
-
:b => Arrow::
|
285
|
-
:c => Arrow::
|
286
|
-
:d => Arrow::
|
287
|
-
:e => Arrow::
|
285
|
+
:a => Arrow::UInt8Array.new([1]),
|
286
|
+
:b => Arrow::UInt8Array.new([1]),
|
287
|
+
:c => Arrow::UInt8Array.new([1]),
|
288
|
+
:d => Arrow::UInt8Array.new([1]),
|
289
|
+
:e => Arrow::UInt8Array.new([1]),
|
288
290
|
}
|
289
291
|
@table = Arrow::Table.new(raw_table)
|
290
292
|
end
|
@@ -337,4 +339,56 @@ class TableTest < Test::Unit::TestCase
|
|
337
339
|
TABLE
|
338
340
|
end
|
339
341
|
end
|
342
|
+
|
343
|
+
sub_test_case("#save and .load") do
|
344
|
+
sub_test_case(":format") do
|
345
|
+
test("default") do
|
346
|
+
file = Tempfile.new(["red-arrow", ".arrow"])
|
347
|
+
@table.save(file.path)
|
348
|
+
assert_equal(@table, Arrow::Table.load(file.path))
|
349
|
+
end
|
350
|
+
|
351
|
+
test(":batch") do
|
352
|
+
file = Tempfile.new(["red-arrow", ".arrow"])
|
353
|
+
@table.save(file.path, :format => :batch)
|
354
|
+
assert_equal(@table, Arrow::Table.load(file.path, :format => :batch))
|
355
|
+
end
|
356
|
+
|
357
|
+
test(":stream") do
|
358
|
+
file = Tempfile.new(["red-arrow", ".arrow"])
|
359
|
+
@table.save(file.path, :format => :stream)
|
360
|
+
assert_equal(@table, Arrow::Table.load(file.path, :format => :stream))
|
361
|
+
end
|
362
|
+
|
363
|
+
test(":csv") do
|
364
|
+
file = Tempfile.new(["red-arrow", ".csv"])
|
365
|
+
@table.save(file.path, :format => :csv)
|
366
|
+
assert_equal(@table, Arrow::Table.load(file.path, :format => :csv))
|
367
|
+
end
|
368
|
+
|
369
|
+
sub_test_case("load: auto detect") do
|
370
|
+
test(":batch") do
|
371
|
+
file = Tempfile.new(["red-arrow", ".arrow"])
|
372
|
+
@table.save(file.path, :format => :batch)
|
373
|
+
assert_equal(@table, Arrow::Table.load(file.path))
|
374
|
+
end
|
375
|
+
|
376
|
+
test(":stream") do
|
377
|
+
file = Tempfile.new(["red-arrow", ".arrow"])
|
378
|
+
@table.save(file.path, :format => :stream)
|
379
|
+
assert_equal(@table, Arrow::Table.load(file.path))
|
380
|
+
end
|
381
|
+
|
382
|
+
test(":csv") do
|
383
|
+
path = fixture_path("with-header.csv")
|
384
|
+
assert_equal(<<-TABLE, Arrow::Table.load(path).to_s)
|
385
|
+
name score
|
386
|
+
0 alice 10
|
387
|
+
1 bob 29
|
388
|
+
2 chris -1
|
389
|
+
TABLE
|
390
|
+
end
|
391
|
+
end
|
392
|
+
end
|
393
|
+
end
|
340
394
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: red-arrow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kouhei Sutou
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-01-
|
11
|
+
date: 2018-01-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gobject-introspection
|
@@ -137,6 +137,7 @@ files:
|
|
137
137
|
- Rakefile
|
138
138
|
- dependency-check/Rakefile
|
139
139
|
- doc/text/apache-2.0.txt
|
140
|
+
- doc/text/development.md
|
140
141
|
- doc/text/news.md
|
141
142
|
- image/red-arrow.png
|
142
143
|
- lib/arrow.rb
|
@@ -147,17 +148,26 @@ files:
|
|
147
148
|
- lib/arrow/chunked-array.rb
|
148
149
|
- lib/arrow/column.rb
|
149
150
|
- lib/arrow/compatibility.rb
|
151
|
+
- lib/arrow/csv-loader.rb
|
150
152
|
- lib/arrow/csv-reader.rb
|
153
|
+
- lib/arrow/date32-array-builder.rb
|
154
|
+
- lib/arrow/date32-array.rb
|
155
|
+
- lib/arrow/date64-array-builder.rb
|
156
|
+
- lib/arrow/date64-array.rb
|
151
157
|
- lib/arrow/field.rb
|
152
158
|
- lib/arrow/loader.rb
|
153
159
|
- lib/arrow/record-batch-file-reader.rb
|
154
160
|
- lib/arrow/record-batch-stream-reader.rb
|
155
161
|
- lib/arrow/record-batch.rb
|
162
|
+
- lib/arrow/record-containable.rb
|
156
163
|
- lib/arrow/record.rb
|
157
164
|
- lib/arrow/slicer.rb
|
158
165
|
- lib/arrow/table-formatter.rb
|
166
|
+
- lib/arrow/table-loader.rb
|
167
|
+
- lib/arrow/table-saver.rb
|
159
168
|
- lib/arrow/table.rb
|
160
169
|
- lib/arrow/tensor.rb
|
170
|
+
- lib/arrow/timestamp-array-builder.rb
|
161
171
|
- lib/arrow/timestamp-array.rb
|
162
172
|
- lib/arrow/version.rb
|
163
173
|
- red-arrow.gemspec
|
@@ -170,7 +180,10 @@ files:
|
|
170
180
|
- test/test-array.rb
|
171
181
|
- test/test-chunked-array.rb
|
172
182
|
- test/test-column.rb
|
183
|
+
- test/test-csv-loader.rb
|
173
184
|
- test/test-csv-reader.rb
|
185
|
+
- test/test-date32-array.rb
|
186
|
+
- test/test-date64-array.rb
|
174
187
|
- test/test-record-batch-file-reader.rb
|
175
188
|
- test/test-record-batch.rb
|
176
189
|
- test/test-slicer.rb
|
@@ -209,11 +222,14 @@ test_files:
|
|
209
222
|
- test/test-column.rb
|
210
223
|
- test/test-array-builder.rb
|
211
224
|
- test/test-chunked-array.rb
|
225
|
+
- test/test-date32-array.rb
|
212
226
|
- test/test-record-batch.rb
|
213
227
|
- test/helper.rb
|
214
228
|
- test/run-test.rb
|
215
229
|
- test/test-timestamp-array.rb
|
216
230
|
- test/test-table.rb
|
231
|
+
- test/test-csv-loader.rb
|
217
232
|
- test/test-array.rb
|
218
233
|
- test/helper/fixture.rb
|
234
|
+
- test/test-date64-array.rb
|
219
235
|
- test/test-slicer.rb
|