red-arrow-dataset 3.0.0 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60801af8ebd03566fa8fd59bf971834c9d10040a9487d91a39fbb80d8fcb2918
-  data.tar.gz: '07699ad51ea6c39283f090a3683ebd1c70b4f92bd6c0350ba146db94d7bb75db'
+  metadata.gz: 197357bda9355ce343ee13115d4850fcee6730df92d47c6b3def322533317a10
+  data.tar.gz: 45c1e7544c1323d6b7f342ea528372e414e10a0349e296bcc57c0200b1dd5e17
 SHA512:
-  metadata.gz: 201295a8e1ee873a50e34d8fd33d69c30a8bdf375d814430cbc2da1aec79c072f15a22f06ebda632a77b7430a4c5af1f7f54a2178e625e51b07d57ab1be58d14
-  data.tar.gz: b073742b5d1c89a751e264dd342ffd950195075d0d53e87c8fc8cdbdbe68feb5da69415ecdea0855088caf39493e534a02e86193b2ba23491e2c09d778fe33ce
+  metadata.gz: f9f1d0f69620be9967ca0b8b9115eb5ead5791ad1cd81e324047951ac80e5583b0568ae521abd8c491a32b422d8232bed20f1ea42fbc0c480786538f20c43bad
+  data.tar.gz: b12912dd69dd26eb3c3754ce466623b81e86bf6664433feb6ebd95e252e539144250cdffc8d615d3b5b070ae63f82d9f5a0e507be8b13c7f1114797e13140e4d

data/lib/arrow-dataset/arrow-table-loadable.rb

@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module ArrowDataset
+  module ArrowTableLoadable
+    private
+    def path_to_uri(path)
+      absolute_path = ::File.expand_path(path)
+      if absolute_path.start_with?("/")
+        URI("file://#{absolute_path}")
+      else
+        URI("file:///#{absolute_path}")
+      end
+    end
+
+    def load_from_directory
+      internal_load_from_uri(path_to_uri(@input))
+    end
+
+    def load_from_uri
+      internal_load_from_uri(@input)
+    end
+
+    def internal_load_from_uri(uri)
+      format = FileFormat.resolve(@options[:format])
+      dataset = FileSystemDataset.build(format) do |factory|
+        factory.file_system_uri = uri
+      end
+      scanner_builder = dataset.begin_scan
+      @options.each do |key, value|
+        next if key == :format
+        next if value.nil?
+        setter = "#{key}="
+        next unless scanner_builder.respond_to?(setter)
+        scanner_builder.public_send(setter, value)
+      end
+      scanner = scanner_builder.finish
+      scanner.to_table
+    end
+  end
+end
+
+module Arrow
+  class TableLoader
+    include ArrowDataset::ArrowTableLoadable
+  end
+end

data/lib/arrow-dataset/arrow-table-savable.rb

@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module ArrowDataset
+  module ArrowTableSavable
+    private
+    def save_to_uri
+      format = FileFormat.resolve(@options[:format])
+      options = FileSystemDatasetWriteOptions.new
+      options.file_write_options = format.default_write_options
+      path = @output.path
+      if @output.scheme.nil?
+        options.file_system = Arrow::LocalFileSystem.new
+      else
+        options.file_system = Arrow::FileSystem.create(@output.to_s)
+        # /C:/... -> C:/...
+        unless File.expand_path(".").start_with?("/")
+          path = path.gsub(/\A\//, "")
+        end
+      end
+      partitioning = @options[:partitioning]
+      if partitioning
+        # TODO
+        options.base_dir = File.dirname(path)
+        options.base_name_template = File.basename(path)
+        options.partitioning = Partitioning.resolve(@options[:partitioning])
+        scanner_builder = ScannerBuilder.new(@table)
+        scanner_builder.use_async(true)
+        scanner = scanner_builder.finish
+        FileSystemDataset.write_scanner(scanner, options)
+      else
+        dir = File.dirname(path)
+        unless File.exist?(dir)
+          options.file_system.create_dir(dir, true)
+        end
+        options.file_system.open_output_stream(path) do |output_stream|
+          format.open_writer(output_stream,
+                             options.file_system,
+                             path,
+                             @table.schema,
+                             format.default_write_options) do |writer|
+            reader = Arrow::TableBatchReader.new(@table)
+            writer.write_record_batch_reader(reader)
+          end
+        end
+      end
+    end
+  end
+end
+
+module Arrow
+  class TableSaver
+    include ArrowDataset::ArrowTableSavable
+  end
+end

data/lib/arrow-dataset/{scan-options.rb → dataset.rb}

@@ -16,21 +16,13 @@
 # under the License.
 
 module ArrowDataset
-  class ScanOptions
+  class Dataset
     class << self
-      def try_convert(value)
-        case value
-        when Hash
-          return nil unless value.key?(:schema)
-          options = new(value[:schema])
-          value.each do |name, value|
-            next if name == :schema
-            options.__send__("#{name}=", value)
-          end
-          options
-        else
-          nil
-        end
+      def build(*args)
+        factory_class = ArrowDataset.const_get("#{name}Factory")
+        factory = factory_class.new(*args)
+        yield(factory)
+        factory.finish
       end
     end
   end

data/lib/arrow-dataset/file-format.rb

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module ArrowDataset
+  class FileFormat
+    class << self
+      def resolve(format)
+        case format
+        when :arrow, :arrow_file, :arrow_streaming
+          IPCFileFormat.new
+        when :parquet
+          ParquetFileFormat.new
+        when :csv
+          CSVFileFormat.new
+        else
+          available_formats = [
+            :arrow,
+            :arrow_file,
+            :arrow_streaming,
+            :parquet,
+            :csv,
+          ]
+          message = "Arrow::Table load format must be one of ["
+          message << available_formats.join(", ")
+          message << "]: #{@options[:format].inspect}"
+          raise ArgumentError, message
+        end
+      end
+    end
+
+    alias_method :open_writer_raw, :open_writer
+    def open_writer(destination, file_system, path, schema, options)
+      writer = open_writer_raw(destination, file_system, path, schema, options)
+      if block_given?
+        begin
+          yield(writer)
+        ensure
+          writer.finish
+        end
+      else
+        writer
+      end
+    end
+  end
+end

data/lib/arrow-dataset/{in-memory-scan-task.rb → file-system-dataset-factory.rb}

@@ -16,19 +16,24 @@
 # under the License.
 
 module ArrowDataset
-  class InMemoryScanTask
-    alias_method :initialize_raw, :initialize
-    private :initialize_raw
-    def initialize(record_batches, **options)
-      record_batches = record_batches.collect do |record_batch|
-        unless record_batch.is_a?(Arrow::RecordBatch)
-          record_batch = Arrow::RecordBatch.new(record_batch)
+  class FileSystemDatasetFactory
+    alias_method :set_file_system_uri_raw, :set_file_system_uri
+    def set_file_system_uri(uri)
+      if uri.is_a?(URI)
+        if uri.scheme.nil?
+          uri = uri.dup
+          absolute_path = File.expand_path(uri.path)
+          if absolute_path.start_with?("/")
+            uri.path = absolute_path
+          else
+            uri.path = "/#{absolute_path}"
+          end
+          uri.scheme = "file"
         end
-        record_batch
+        uri = uri.to_s
       end
-      context = options.delete(:context) || ScanContext.new
-      options[:schema] ||= record_batches.first.schema
-      initialize_raw(record_batches, options, context)
+      set_file_system_uri_raw(uri)
     end
+    alias_method :file_system_uri=, :set_file_system_uri
   end
 end

data/lib/arrow-dataset/loader.rb

@@ -29,8 +29,11 @@ module ArrowDataset
     end
 
     def require_libraries
-      require "arrow-dataset/in-memory-scan-task"
-      require "arrow-dataset/scan-options"
+      require "arrow-dataset/arrow-table-loadable"
+      require "arrow-dataset/arrow-table-savable"
+      require "arrow-dataset/dataset"
+      require "arrow-dataset/file-format"
+      require "arrow-dataset/file-system-dataset-factory"
     end
   end
 end

data/lib/arrow-dataset/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module ArrowDataset
-  VERSION = "3.0.0"
+  VERSION = "6.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/test/helper.rb

@@ -17,4 +17,6 @@
 
 require "arrow-dataset"
 
+require "tmpdir"
+
 require "test-unit"

data/test/test-arrow-table.rb

@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestArrowTable < Test::Unit::TestCase
+  def setup
+    Dir.mktmpdir do |tmpdir|
+      @dir = tmpdir
+      @path1 = File.join(@dir, "data", "table1.arrow")
+      @table1 = Arrow::Table.new(visible: [true, false, true],
+                                 point: [1, 2, 3])
+      @path2 = File.join(@dir, "data", "table2.arrow")
+      @table2 = Arrow::Table.new(visible: [true],
+                                 point: [10])
+      yield
+    end
+  end
+
+  def build_file_uri(path)
+    absolute_path = File.expand_path(path)
+    if absolute_path.start_with?("/")
+      URI("file://#{absolute_path}")
+    else
+      URI("file:///#{absolute_path}")
+    end
+  end
+
+  sub_test_case("load") do
+    def test_no_scheme
+      Dir.chdir(@dir) do
+        uri = URI(File.basename(@path1))
+        @table1.save(uri)
+        assert_equal(@table1, Arrow::Table.load(uri))
+      end
+    end
+
+    def test_file
+      uri = build_file_uri(@path1)
+      @table1.save(uri)
+      assert_equal(@table1, Arrow::Table.load(uri))
+    end
+
+    def test_directory_uri
+      uri = build_file_uri(@dir)
+      @table1.save(build_file_uri(@path1))
+      @table2.save(build_file_uri(@path2))
+      assert_equal(@table1.concatenate([@table2]),
+                   Arrow::Table.load(uri))
+    end
+
+    def test_directory_path
+      @table1.save(build_file_uri(@path1))
+      @table2.save(build_file_uri(@path2))
+      assert_equal(@table1.concatenate([@table2]),
+                   Arrow::Table.load(@dir))
+    end
+
+    def test_filter
+      @table1.save(build_file_uri(@path1))
+      @table2.save(build_file_uri(@path2))
+      assert_equal(Arrow::Table.new(visible: [true, true, true],
+                                    point: [1, 3, 10]),
+                   Arrow::Table.load(@dir,
+                                     filter: ["equal", :visible, true]))
+    end
+  end
+end

data/test/{test-in-memory-scan-task.rb → test-file-system-dataset.rb}

@@ -15,19 +15,24 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class TestInMemoryScanTask < Test::Unit::TestCase
+class TestFileSystemDataset < Test::Unit::TestCase
   def setup
-    @record_batches = [
-      Arrow::RecordBatch.new(visible: [true, false, true],
-                             point: [1, 2, 3]),
-    ]
+    Dir.mktmpdir do |tmpdir|
+      @dir = tmpdir
+      @path = File.join(@dir, "table.arrow")
+      @table = Arrow::Table.new(visible: [true, false, true],
+                                point: [1, 2, 3])
+      @table.save(@path)
+      @format = ArrowDataset::IPCFileFormat.new
+      yield
+    end
   end
 
-  sub_test_case(".new") do
-    test("[[Arrow::RecordBatch]]") do
-      scan_task = ArrowDataset::InMemoryScanTask.new(@record_batches)
-      assert_equal(@record_batches,
-                   scan_task.execute.to_a)
+  test(".build") do
+    dataset = ArrowDataset::FileSystemDataset.build(@format) do |factory|
+      factory.file_system = Arrow::LocalFileSystem.new
+      factory.add_path(File.expand_path(@path))
     end
+    assert_equal(@table, dataset.to_table)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow-dataset
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 6.0.0
 platform: ruby
 authors:
 - Apache Arrow Developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-25 00:00:00.000000000 Z
+date: 2021-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 6.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 6.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -82,15 +82,18 @@ files:
 - Rakefile
 - dependency-check/Rakefile
 - lib/arrow-dataset.rb
-- lib/arrow-dataset/in-memory-scan-task.rb
+- lib/arrow-dataset/arrow-table-loadable.rb
+- lib/arrow-dataset/arrow-table-savable.rb
+- lib/arrow-dataset/dataset.rb
+- lib/arrow-dataset/file-format.rb
+- lib/arrow-dataset/file-system-dataset-factory.rb
 - lib/arrow-dataset/loader.rb
-- lib/arrow-dataset/scan-options.rb
 - lib/arrow-dataset/version.rb
 - red-arrow-dataset.gemspec
 - test/helper.rb
 - test/run-test.rb
-- test/test-in-memory-scan-task.rb
-- test/test-scan-options.rb
+- test/test-arrow-table.rb
+- test/test-file-system-dataset.rb
 homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
@@ -110,12 +113,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.5
+rubygems_version: 3.2.27
 signing_key: 
 specification_version: 4
 summary: Red Arrow Dataset is the Ruby bindings of Apache Arrow Dataset
 test_files:
 - test/helper.rb
 - test/run-test.rb
-- test/test-scan-options.rb
-- test/test-in-memory-scan-task.rb
+- test/test-arrow-table.rb
+- test/test-file-system-dataset.rb

data/test/test-scan-options.rb

@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class TestScanOptions < Test::Unit::TestCase
-  def setup
-    @record_batches = [
-      Arrow::RecordBatch.new(visible: [true, false, true],
-                             point: [1, 2, 3]),
-    ]
-    @schema = @record_batches.first.schema
-  end
-
-  sub_test_case(".try_convert") do
-    def test_hash
-      batch_size = 1024
-      context = ArrowDataset::ScanOptions.try_convert(schema: @schema,
-                                                      batch_size: batch_size)
-      assert_equal([@schema, batch_size],
-                   [context.schema, context.batch_size])
-    end
-  end
-end