red-arrow 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +7 -3
- data/Rakefile +2 -0
- data/doc/text/news.md +11 -0
- data/lib/arrow.rb +0 -4
- data/lib/arrow/{ipc/file-reader.rb → ipc-file-reader.rb} +0 -0
- data/lib/arrow/{ipc/stream-reader.rb → ipc-stream-reader.rb} +0 -0
- data/lib/arrow/loader.rb +14 -0
- data/lib/arrow/record-batch.rb +21 -4
- data/lib/arrow/record.rb +9 -4
- data/lib/arrow/version.rb +1 -1
- data/red-arrow.gemspec +1 -1
- data/test/test-record-batch.rb +54 -0
- metadata +7 -8
- data/lib/arrow/block-openable-applicable.rb +0 -29
- data/lib/arrow/io/loader.rb +0 -37
- data/lib/arrow/ipc/loader.rb +0 -39
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9557cac923316c8ff79fe9308b97d9a48dfe7a92
|
4
|
+
data.tar.gz: e8e485563267fefa6219e4890d5de589b79fa186
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4022b9925abe4269ced64d25f2ac5ac24cb4752b58d924e00a2f818f62c70b8f79e458055c0d34b20f8b0ad8077d1bfeadaf530c04d64c1ebc095d146c344106
|
7
|
+
data.tar.gz: 01517b6e438f619b0f8ad3fe6b7d1fbf76195afed624523de4d42896a87a1eb2fcd0d924202952f4bc6160936cbc69c4cb580e3f609bc99ff940c36809deabf3
|
data/README.md
CHANGED
@@ -12,14 +12,18 @@ Red Arrow is a Ruby bindings of Apache Arrow. Red Arrow is based on GObject Intr
|
|
12
12
|
|
13
13
|
[GObject Introspection](https://wiki.gnome.org/action/show/Projects/GObjectIntrospection) is a middleware for language bindings of C library. GObject Introspection can generate language bindings automatically at runtime.
|
14
14
|
|
15
|
-
Red Arrow uses [
|
15
|
+
Red Arrow uses [Arrow GLib](https://github.com/apache/arrow/tree/master/c_glib) and [gobject-introspection gem](https://rubygems.org/gems/gobject-introspection) to generate Ruby bindings of Apache Arrow.
|
16
16
|
|
17
|
-
|
17
|
+
Arrow GLib is a C wrapper for [Arrow C++](https://github.com/apache/arrow/tree/master/cpp). GObject Introspection can't use Arrow C++ directly. Arrow GLib is a bridge between Arrow C++ and GObject Introspection.
|
18
18
|
|
19
19
|
gobject-introspection gem is a Ruby bindings of GObject Introspection. Red Arrow uses GObject Introspection via gobject-introspection gem.
|
20
20
|
|
21
21
|
## Install
|
22
22
|
|
23
|
+
Install Arrow GLib before install Red Arrow. Use [Apache Arrow packages](https://github.com/kou/arrow-packages) for installing Arrow GLib.
|
24
|
+
|
25
|
+
Install Red Arrow after you install Arrow GLib:
|
26
|
+
|
23
27
|
```text
|
24
28
|
% gem install red-arrow
|
25
29
|
```
|
@@ -36,7 +40,7 @@ require "arrow"
|
|
36
40
|
|
37
41
|
* [Apache Arrow](https://arrow.apache.org/)
|
38
42
|
|
39
|
-
* [
|
43
|
+
* [Arrow GLib](https://github.com/apache/arrow/tree/master/c_glib)
|
40
44
|
|
41
45
|
* [gobject-introspection gem](https://rubygems.org/gems/gobject-introspection)
|
42
46
|
|
data/Rakefile
CHANGED
data/doc/text/news.md
CHANGED
@@ -1,5 +1,16 @@
|
|
1
1
|
# News
|
2
2
|
|
3
|
+
## 0.2.2 - 2017-04-24
|
4
|
+
|
5
|
+
### Improvements
|
6
|
+
|
7
|
+
* `Arrow::RecordBatch#each`: Supported reusing record object for
|
8
|
+
performance.
|
9
|
+
|
10
|
+
* ``Arrow::IO`: Unified into `Arrow`.
|
11
|
+
|
12
|
+
* ``Arrow::IPC`: Unified into `Arrow`.
|
13
|
+
|
3
14
|
## 0.2.1 - 2017-03-23
|
4
15
|
|
5
16
|
### Improvements
|
data/lib/arrow.rb
CHANGED
@@ -17,14 +17,10 @@ require "gobject-introspection"
|
|
17
17
|
require "arrow/version"
|
18
18
|
|
19
19
|
require "arrow/loader"
|
20
|
-
require "arrow/io/loader"
|
21
|
-
require "arrow/ipc/loader"
|
22
20
|
|
23
21
|
module Arrow
|
24
22
|
class Error < StandardError
|
25
23
|
end
|
26
24
|
|
27
25
|
Loader.load
|
28
|
-
IO::Loader.load
|
29
|
-
IPC::Loader.load
|
30
26
|
end
|
File without changes
|
File without changes
|
data/lib/arrow/loader.rb
CHANGED
@@ -12,6 +12,8 @@
|
|
12
12
|
# See the License for the specific language governing permissions and
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
|
+
require "arrow/block-openable"
|
16
|
+
|
15
17
|
module Arrow
|
16
18
|
class Loader < GObjectIntrospection::Loader
|
17
19
|
class << self
|
@@ -30,6 +32,18 @@ module Arrow
|
|
30
32
|
require "arrow/array-builder"
|
31
33
|
require "arrow/field"
|
32
34
|
require "arrow/record-batch"
|
35
|
+
|
36
|
+
require "arrow/ipc-file-reader"
|
37
|
+
require "arrow/ipc-stream-reader"
|
38
|
+
end
|
39
|
+
|
40
|
+
def load_object_info(info)
|
41
|
+
super
|
42
|
+
|
43
|
+
klass = @base_module.const_get(rubyish_class_name(info))
|
44
|
+
if klass.respond_to?(:open)
|
45
|
+
klass.singleton_class.prepend(BlockOpenable)
|
46
|
+
end
|
33
47
|
end
|
34
48
|
|
35
49
|
def rubyish_method_name(function_info, options={})
|
data/lib/arrow/record-batch.rb
CHANGED
@@ -18,9 +18,21 @@ module Arrow
|
|
18
18
|
class RecordBatch
|
19
19
|
include Enumerable
|
20
20
|
|
21
|
-
def each
|
22
|
-
|
23
|
-
|
21
|
+
def each(reuse_record: false)
|
22
|
+
unless block_given?
|
23
|
+
return to_enum(__method__, reuse_record: reuse_record)
|
24
|
+
end
|
25
|
+
|
26
|
+
if reuse_record
|
27
|
+
record = Record.new(self, nil)
|
28
|
+
n_rows.times do |i|
|
29
|
+
record.index = i
|
30
|
+
yield(record)
|
31
|
+
end
|
32
|
+
else
|
33
|
+
n_rows.times do |i|
|
34
|
+
yield(Record.new(self, i))
|
35
|
+
end
|
24
36
|
end
|
25
37
|
end
|
26
38
|
|
@@ -32,7 +44,12 @@ module Arrow
|
|
32
44
|
else
|
33
45
|
index = name_or_index
|
34
46
|
end
|
35
|
-
|
47
|
+
columns[index]
|
48
|
+
end
|
49
|
+
|
50
|
+
alias_method :columns_raw, :columns
|
51
|
+
def columns
|
52
|
+
@columns ||= columns_raw
|
36
53
|
end
|
37
54
|
|
38
55
|
private
|
data/lib/arrow/record.rb
CHANGED
@@ -14,13 +14,18 @@
|
|
14
14
|
|
15
15
|
module Arrow
|
16
16
|
class Record
|
17
|
-
|
17
|
+
attr_accessor :index
|
18
|
+
def initialize(record_batch, index)
|
18
19
|
@record_batch = record_batch
|
19
|
-
@
|
20
|
+
@index = index
|
20
21
|
end
|
21
22
|
|
22
|
-
def [](
|
23
|
-
@record_batch.find_column(
|
23
|
+
def [](column_name_or_column_index)
|
24
|
+
@record_batch.find_column(column_name_or_column_index)[@index]
|
25
|
+
end
|
26
|
+
|
27
|
+
def columns
|
28
|
+
@record_batch.columns
|
24
29
|
end
|
25
30
|
end
|
26
31
|
end
|
data/lib/arrow/version.rb
CHANGED
data/red-arrow.gemspec
CHANGED
@@ -24,7 +24,7 @@ require "arrow/version"
|
|
24
24
|
Gem::Specification.new do |spec|
|
25
25
|
spec.name = "red-arrow"
|
26
26
|
spec.version = Arrow::VERSION
|
27
|
-
spec.homepage = "https://github.com/
|
27
|
+
spec.homepage = "https://github.com/red-data-tools/red-arrow"
|
28
28
|
spec.authors = ["Kouhei Sutou"]
|
29
29
|
spec.email = ["kou@clear-code.com"]
|
30
30
|
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
class RecordBatchTest < Test::Unit::TestCase
|
16
|
+
sub_test_case(".each") do
|
17
|
+
setup do
|
18
|
+
fields = [
|
19
|
+
Arrow::Field.new("count", :uint32),
|
20
|
+
]
|
21
|
+
@schema = Arrow::Schema.new(fields)
|
22
|
+
@counts = Arrow::UInt32Array.new([1, 2, 4, 8])
|
23
|
+
@record_batch = Arrow::RecordBatch.new(@schema, @counts.length, [@counts])
|
24
|
+
end
|
25
|
+
|
26
|
+
test("default") do
|
27
|
+
records = []
|
28
|
+
@record_batch.each do |record|
|
29
|
+
records << [record, record.index]
|
30
|
+
end
|
31
|
+
assert_equal([
|
32
|
+
[0, 0],
|
33
|
+
[1, 1],
|
34
|
+
[2, 2],
|
35
|
+
[3, 3],
|
36
|
+
],
|
37
|
+
records.collect {|record, i| [record.index, i]})
|
38
|
+
end
|
39
|
+
|
40
|
+
test("reuse_record: true") do
|
41
|
+
records = []
|
42
|
+
@record_batch.each(reuse_record: true) do |record|
|
43
|
+
records << [record, record.index]
|
44
|
+
end
|
45
|
+
assert_equal([
|
46
|
+
[3, 0],
|
47
|
+
[3, 1],
|
48
|
+
[3, 2],
|
49
|
+
[3, 3],
|
50
|
+
],
|
51
|
+
records.collect {|record, i| [record.index, i]})
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: red-arrow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kouhei Sutou
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-04-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gobject-introspection
|
@@ -111,13 +111,10 @@ files:
|
|
111
111
|
- lib/arrow.rb
|
112
112
|
- lib/arrow/array-builder.rb
|
113
113
|
- lib/arrow/array.rb
|
114
|
-
- lib/arrow/block-openable-applicable.rb
|
115
114
|
- lib/arrow/block-openable.rb
|
116
115
|
- lib/arrow/field.rb
|
117
|
-
- lib/arrow/
|
118
|
-
- lib/arrow/ipc
|
119
|
-
- lib/arrow/ipc/loader.rb
|
120
|
-
- lib/arrow/ipc/stream-reader.rb
|
116
|
+
- lib/arrow/ipc-file-reader.rb
|
117
|
+
- lib/arrow/ipc-stream-reader.rb
|
121
118
|
- lib/arrow/loader.rb
|
122
119
|
- lib/arrow/record-batch.rb
|
123
120
|
- lib/arrow/record.rb
|
@@ -126,7 +123,8 @@ files:
|
|
126
123
|
- test/helper.rb
|
127
124
|
- test/run-test.rb
|
128
125
|
- test/test-array.rb
|
129
|
-
|
126
|
+
- test/test-record-batch.rb
|
127
|
+
homepage: https://github.com/red-data-tools/red-arrow
|
130
128
|
licenses:
|
131
129
|
- Apache-2.0
|
132
130
|
metadata: {}
|
@@ -152,6 +150,7 @@ specification_version: 4
|
|
152
150
|
summary: Red Arrow is a Ruby bindings of Apache Arrow. Red Arrow is based on GObject
|
153
151
|
Introspection.
|
154
152
|
test_files:
|
153
|
+
- test/test-record-batch.rb
|
155
154
|
- test/helper.rb
|
156
155
|
- test/run-test.rb
|
157
156
|
- test/test-array.rb
|
@@ -1,29 +0,0 @@
|
|
1
|
-
# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
|
-
|
15
|
-
require "arrow/block-openable"
|
16
|
-
|
17
|
-
module Arrow
|
18
|
-
module BlockOpenableApplicable
|
19
|
-
private
|
20
|
-
def load_object_info(info)
|
21
|
-
super
|
22
|
-
|
23
|
-
klass = @base_module.const_get(rubyish_class_name(info))
|
24
|
-
if klass.respond_to?(:open)
|
25
|
-
klass.singleton_class.prepend(BlockOpenable)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
data/lib/arrow/io/loader.rb
DELETED
@@ -1,37 +0,0 @@
|
|
1
|
-
# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
|
-
|
15
|
-
require "arrow/block-openable-applicable"
|
16
|
-
|
17
|
-
module Arrow
|
18
|
-
module IO
|
19
|
-
class Loader < GObjectIntrospection::Loader
|
20
|
-
class << self
|
21
|
-
def load
|
22
|
-
super("ArrowIO", IO)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
include BlockOpenableApplicable
|
27
|
-
|
28
|
-
private
|
29
|
-
def post_load(repository, namespace)
|
30
|
-
require_libraries
|
31
|
-
end
|
32
|
-
|
33
|
-
def require_libraries
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
data/lib/arrow/ipc/loader.rb
DELETED
@@ -1,39 +0,0 @@
|
|
1
|
-
# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
|
-
|
15
|
-
require "arrow/block-openable-applicable"
|
16
|
-
|
17
|
-
module Arrow
|
18
|
-
module IPC
|
19
|
-
class Loader < GObjectIntrospection::Loader
|
20
|
-
class << self
|
21
|
-
def load
|
22
|
-
super("ArrowIPC", IPC)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
include BlockOpenableApplicable
|
27
|
-
|
28
|
-
private
|
29
|
-
def post_load(repository, namespace)
|
30
|
-
require_libraries
|
31
|
-
end
|
32
|
-
|
33
|
-
def require_libraries
|
34
|
-
require "arrow/ipc/file-reader"
|
35
|
-
require "arrow/ipc/stream-reader"
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|