red-arrow 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9557cac923316c8ff79fe9308b97d9a48dfe7a92
-  data.tar.gz: e8e485563267fefa6219e4890d5de589b79fa186
+  metadata.gz: d66932ddc8b1edd4d0a00a3c44ace6a54eb23509
+  data.tar.gz: d373ca2d306387a5709e8b2b28946e10ec5529ee
 SHA512:
-  metadata.gz: 4022b9925abe4269ced64d25f2ac5ac24cb4752b58d924e00a2f818f62c70b8f79e458055c0d34b20f8b0ad8077d1bfeadaf530c04d64c1ebc095d146c344106
-  data.tar.gz: 01517b6e438f619b0f8ad3fe6b7d1fbf76195afed624523de4d42896a87a1eb2fcd0d924202952f4bc6160936cbc69c4cb580e3f609bc99ff940c36809deabf3
+  metadata.gz: d018d51d091d4ad37cd608f4bd951a837b385d36cf975a0a0652a5d1173a847aa2f875705e46d66e533b8246d5f3b0b0b75d936fdcdd2e794e1676463759bd66
+  data.tar.gz: 434cd73edc6180de459d9bfa95272703d7d0ffb1a095ea7e7f64462880bae79cdb14ccadbcd8285966462832c80c8837a55c26c79e424c9160ceb9e719989639

data/doc/text/news.md

@@ -1,5 +1,15 @@
 # News
 
+## 0.3.0 - 2017-05-06
+
+### Improvements
+
+  * `Arrow::Record#to_h`: Added.
+
+  * `#to_arrow`: Added convenience methods for polymorphism.
+
+  * Supported Apache Arrow 0.3.0.
+
 ## 0.2.2 - 2017-04-24
 
 ### Improvements

data/lib/arrow/array.rb

@@ -33,5 +33,9 @@ module Arrow
         yield(self[i])
       end
     end
+
+    def to_arrow
+      self
+    end
   end
 end

data/lib/arrow/{block-openable.rb → block-closable.rb}

@@ -13,11 +13,10 @@
 # limitations under the License.
 
 module Arrow
-  module BlockOpenable
+  module BlockClosable
     def open(*args, &block)
-      io = super(*args)
+      io = new(*args)
       return io unless block
-      return yield(io) unless io.respond_to?(:close)
 
       begin
         yield(io)

data/lib/arrow/compatibility.rb

@@ -0,0 +1,28 @@
+# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Arrow
+  if const_defined?(:MemoryMappedInputStream)
+    class MemoryMappedFile
+      class << self
+        def open(path, mode, &block)
+          warn("#{self}.#{__method__}(path, mode, &block): " +
+               "use #{MemoryMappedInputStream}.open(path, &block) instead: " +
+               caller(1, 1)[0])
+          MemoryMappedInputStream.open(path, &block)
+        end
+      end
+    end
+  end
+end

data/lib/arrow/file-reader.rb

@@ -0,0 +1,53 @@
+# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Arrow
+  class FileReader
+    include Enumerable
+
+    # For backward compatibility
+    if respond_to?(:open)
+      class << self
+        alias_method :open_raw, :open
+        def open(input)
+          warn("#{self}.#{__method__}: use #{self}.new instead: #{caller(1, 1)[0]}")
+          reader = open_raw(input)
+          if block_given?
+            yield(reader)
+          else
+            reader
+          end
+        end
+      end
+    else
+      class << self
+        def open(input)
+          warn("#{self}.#{__method__}: use #{self}.new instead #{caller(1, 1)[0]}")
+          reader = new(input)
+          if block_given?
+            yield(reader)
+          else
+            reader
+          end
+        end
+      end
+    end
+
+    def each
+      n_record_batches.times do |i|
+        yield(get_record_batch(i))
+      end
+    end
+  end
+end

data/lib/arrow/loader.rb

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "arrow/block-openable"
+require "arrow/block-closable"
 
 module Arrow
   class Loader < GObjectIntrospection::Loader
@@ -32,17 +32,20 @@ module Arrow
       require "arrow/array-builder"
       require "arrow/field"
       require "arrow/record-batch"
+      require "arrow/tensor"
 
-      require "arrow/ipc-file-reader"
-      require "arrow/ipc-stream-reader"
+      require "arrow/file-reader"
+      require "arrow/stream-reader"
+
+      require "arrow/compatibility"
     end
 
     def load_object_info(info)
       super
 
       klass = @base_module.const_get(rubyish_class_name(info))
-      if klass.respond_to?(:open)
-        klass.singleton_class.prepend(BlockOpenable)
+      if klass.method_defined?(:close)
+        klass.extend(BlockClosable)
       end
     end
 

data/lib/arrow/record.rb

@@ -27,5 +27,13 @@ module Arrow
     def columns
       @record_batch.columns
     end
+
+    def to_h
+      attributes = {}
+      @record_batch.schema.fields.each_with_index do |field, i|
+        attributes[field.name] = self[i]
+      end
+      attributes
+    end
   end
 end

data/lib/arrow/{ipc-file-reader.rb → stream-reader.rb}

@@ -13,14 +13,14 @@
 # limitations under the License.
 
 module Arrow
-  module IPC
-    class FileReader
-      include Enumerable
+  class StreamReader
+    include Enumerable
 
-      def each
-        n_record_batches.times do |i|
-          yield(get_record_batch(i))
-        end
+    def each
+      loop do
+        record_batch = next_record_batch
+        break if record_batch.nil?
+        yield(record_batch)
       end
     end
   end

data/lib/arrow/{ipc-stream-reader.rb → tensor.rb}

@@ -13,17 +13,9 @@
 # limitations under the License.
 
 module Arrow
-  module IPC
-    class StreamReader
-      include Enumerable
-
-      def each
-        loop do
-          record_batch = next_record_batch
-          break if record_batch.nil?
-          yield(record_batch)
-        end
-      end
+  class Tensor
+    def to_arrow
+      self
     end
   end
 end

data/lib/arrow/version.rb

@@ -13,5 +13,5 @@
 # limitations under the License.
 
 module Arrow
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/test/helper.rb

@@ -14,4 +14,6 @@
 
 require "arrow"
 
+require "tempfile"
+
 require "test-unit"

data/test/test-file-reader.rb

@@ -0,0 +1,112 @@
+# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class FileReaderTest < Test::Unit::TestCase
+  test("write/read") do
+    fields = [
+      Arrow::Field.new("uint8",  :uint8),
+      Arrow::Field.new("uint16", :uint16),
+      Arrow::Field.new("uint32", :uint32),
+      Arrow::Field.new("uint64", :uint64),
+      Arrow::Field.new("int8",   :int8),
+      Arrow::Field.new("int16",  :int16),
+      Arrow::Field.new("int32",  :int32),
+      Arrow::Field.new("int64",  :int64),
+      Arrow::Field.new("float",  :float),
+      Arrow::Field.new("double", :double),
+    ]
+    schema = Arrow::Schema.new(fields)
+
+    tempfile = Tempfile.new(["batch", ".arrow"])
+    Arrow::FileOutputStream.open(tempfile.path, false) do |output|
+      Arrow::FileWriter.open(output, schema) do |writer|
+        uints = [1, 2, 4, 8]
+        ints = [1, -2, 4, -8]
+        floats = [1.1, -2.2, 4.4, -8.8]
+        columns = [
+          Arrow::UInt8Array.new(uints),
+          Arrow::UInt16Array.new(uints),
+          Arrow::UInt32Array.new(uints),
+          Arrow::UInt64Array.new(uints),
+          Arrow::Int8Array.new(ints),
+          Arrow::Int16Array.new(ints),
+          Arrow::Int32Array.new(ints),
+          Arrow::Int64Array.new(ints),
+          Arrow::FloatArray.new(floats),
+          Arrow::DoubleArray.new(floats),
+        ]
+
+        record_batch = Arrow::RecordBatch.new(schema, 4, columns)
+        writer.write_record_batch(record_batch)
+      end
+    end
+
+    Arrow::MemoryMappedInputStream.open(tempfile.path) do |input|
+      reader = Arrow::FileReader.new(input)
+      reader.each do |record_batch|
+        assert_equal([
+                       {
+                         "uint8"  => 1,
+                         "uint16" => 1,
+                         "uint32" => 1,
+                         "uint64" => 1,
+                         "int8"   => 1,
+                         "int16"  => 1,
+                         "int32"  => 1,
+                         "int64"  => 1,
+                         "float"  => 1.100000023841858,
+                         "double" => 1.1,
+                       },
+                       {
+                         "uint8"  => 2,
+                         "uint16" => 2,
+                         "uint32" => 2,
+                         "uint64" => 2,
+                         "int8"   => -2,
+                         "int16"  => -2,
+                         "int32"  => -2,
+                         "int64"  => -2,
+                         "float"  => -2.200000047683716,
+                         "double" => -2.2,
+                       },
+                       {
+                         "uint8"  => 4,
+                         "uint16" => 4,
+                         "uint32" => 4,
+                         "uint64" => 4,
+                         "int8"   => 4,
+                         "int16"  => 4,
+                         "int32"  => 4,
+                         "int64"  => 4,
+                         "float"  => 4.400000095367432,
+                         "double" => 4.4,
+                       },
+                       {
+                         "uint8"  => 8,
+                         "uint16" => 8,
+                         "uint32" => 8,
+                         "uint64" => 8,
+                         "int8"   => -8,
+                         "int16"  => -8,
+                         "int32"  => -8,
+                         "int64"  => -8,
+                         "float"  => -8.800000190734863,
+                         "double" => -8.8,
+                       },
+                     ],
+                     record_batch.collect(&:to_h))
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-24 00:00:00.000000000 Z
+date: 2017-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gobject-introspection
@@ -111,18 +111,21 @@ files:
 - lib/arrow.rb
 - lib/arrow/array-builder.rb
 - lib/arrow/array.rb
-- lib/arrow/block-openable.rb
+- lib/arrow/block-closable.rb
+- lib/arrow/compatibility.rb
 - lib/arrow/field.rb
-- lib/arrow/ipc-file-reader.rb
-- lib/arrow/ipc-stream-reader.rb
+- lib/arrow/file-reader.rb
 - lib/arrow/loader.rb
 - lib/arrow/record-batch.rb
 - lib/arrow/record.rb
+- lib/arrow/stream-reader.rb
+- lib/arrow/tensor.rb
 - lib/arrow/version.rb
 - red-arrow.gemspec
 - test/helper.rb
 - test/run-test.rb
 - test/test-array.rb
+- test/test-file-reader.rb
 - test/test-record-batch.rb
 homepage: https://github.com/red-data-tools/red-arrow
 licenses:
@@ -154,3 +157,4 @@ test_files:
 - test/helper.rb
 - test/run-test.rb
 - test/test-array.rb
+- test/test-file-reader.rb