red-arrow-dataset 2.0.0 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1abf8cb2fb601ddcdd8b0f0014a454037259d001c240ecaa7d5257a13d52602
-  data.tar.gz: b592225816719f4fb2b6fa3d88190c6d9897c9104b8a5927c450ced1cd8d5711
+  metadata.gz: bd2475450c901efd8a4a2e311459285098b42a714ada39e41959e8c7d7f477c8
+  data.tar.gz: 7d696272844899dbdedf1d1e4ba379a54c91a1187da33c97a6d5eddb89c5f588
 SHA512:
-  metadata.gz: d8df982d7587d657770f7c293d233f7cbd888591f65495a4f9667e31608625cc31ec15d5751229867f0c599fb24f715ec7159acc030f66ce19973bf525ecf8d2
-  data.tar.gz: fc734ec58deee49fe18aa75065da563a672eb1d830faa6ad6ea19b22f1d8a806d8ee7b2c68fa7919eb5a0f00424480d0405ed0f28a8009935600fcfaa346aa76
+  metadata.gz: 61868f12b9d4b607ebf3e408a81e576c0684a433b9ffbb2245abc34dd5a4f797a78ed05544c74a0bdd0a8caa7da01cc80e9885f32f90c63c788f7b1380d04df1
+  data.tar.gz: dd3022730a0d70182217dc80a7364db74eaba3bab783bb673b0776e1c5b562c8200ec6c10d81fda61627aca3ce458d287141f0c97e2b1e706ddda752deab2b35

data/lib/arrow-dataset/{scan-options.rb → dataset.rb}

@@ -16,21 +16,13 @@
 # under the License.
 
 module ArrowDataset
-  class ScanOptions
+  class Dataset
     class << self
-      def try_convert(value)
-        case value
-        when Hash
-          return nil unless value.key?(:schema)
-          options = new(value[:schema])
-          value.each do |name, value|
-            next if name == :schema
-            options.__send__("#{name}=", value)
-          end
-          options
-        else
-          nil
-        end
+      def build(*args)
+        factory_class = ArrowDataset.const_get("#{name}Factory")
+        factory = factory_class.new(*args)
+        yield(factory)
+        factory.finish
       end
     end
   end

data/lib/arrow-dataset/loader.rb

@@ -29,8 +29,7 @@ module ArrowDataset
     end
 
     def require_libraries
-      require "arrow-dataset/in-memory-scan-task"
-      require "arrow-dataset/scan-options"
+      require "arrow-dataset/dataset"
     end
   end
 end

data/lib/arrow-dataset/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module ArrowDataset
-  VERSION = "2.0.0"
+  VERSION = "5.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/test/helper.rb

@@ -17,4 +17,6 @@
 
 require "arrow-dataset"
 
+require "tmpdir"
+
 require "test-unit"

data/test/{test-in-memory-scan-task.rb → test-file-system-dataset.rb}

@@ -15,19 +15,24 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class TestInMemoryScanTask < Test::Unit::TestCase
+class TestFileSystemDataset < Test::Unit::TestCase
   def setup
-    @record_batches = [
-      Arrow::RecordBatch.new(visible: [true, false, true],
-                             point: [1, 2, 3]),
-    ]
+    Dir.mktmpdir do |tmpdir|
+      @dir = tmpdir
+      @path = File.join(@dir, "table.arrow")
+      @table = Arrow::Table.new(visible: [true, false, true],
+                                point: [1, 2, 3])
+      @table.save(@path)
+      @format = ArrowDataset::IPCFileFormat.new
+      yield
+    end
   end
 
-  sub_test_case(".new") do
-    test("[[Arrow::RecordBatch]]") do
-      scan_task = ArrowDataset::InMemoryScanTask.new(@record_batches)
-      assert_equal(@record_batches,
-                   scan_task.execute.to_a)
+  test(".build") do
+    dataset = ArrowDataset::FileSystemDataset.build(@format) do |factory|
+      factory.file_system = Arrow::LocalFileSystem.new
+      factory.add_path(File.expand_path(@path))
     end
+    assert_equal(@table, dataset.to_table)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow-dataset
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 5.0.0
 platform: ruby
 authors:
 - Apache Arrow Developers
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-19 00:00:00.000000000 Z
+date: 2021-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 5.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 5.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -82,15 +82,13 @@ files:
 - Rakefile
 - dependency-check/Rakefile
 - lib/arrow-dataset.rb
-- lib/arrow-dataset/in-memory-scan-task.rb
+- lib/arrow-dataset/dataset.rb
 - lib/arrow-dataset/loader.rb
-- lib/arrow-dataset/scan-options.rb
 - lib/arrow-dataset/version.rb
 - red-arrow-dataset.gemspec
 - test/helper.rb
 - test/run-test.rb
-- test/test-in-memory-scan-task.rb
-- test/test-scan-options.rb
+- test/test-file-system-dataset.rb
 homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
@@ -110,12 +108,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Red Arrow Dataset is the Ruby bindings of Apache Arrow Dataset
 test_files:
-- test/run-test.rb
 - test/helper.rb
-- test/test-scan-options.rb
-- test/test-in-memory-scan-task.rb
+- test/run-test.rb
+- test/test-file-system-dataset.rb

data/lib/arrow-dataset/in-memory-scan-task.rb

@@ -1,34 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-module ArrowDataset
-  class InMemoryScanTask
-    alias_method :initialize_raw, :initialize
-    private :initialize_raw
-    def initialize(record_batches, **options)
-      record_batches = record_batches.collect do |record_batch|
-        unless record_batch.is_a?(Arrow::RecordBatch)
-          record_batch = Arrow::RecordBatch.new(record_batch)
-        end
-        record_batch
-      end
-      context = options.delete(:context) || ScanContext.new
-      options[:schema] ||= record_batches.first.schema
-      initialize_raw(record_batches, options, context)
-    end
-  end
-end

data/test/test-scan-options.rb

@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class TestScanOptions < Test::Unit::TestCase
-  def setup
-    @record_batches = [
-      Arrow::RecordBatch.new(visible: [true, false, true],
-                             point: [1, 2, 3]),
-    ]
-    @schema = @record_batches.first.schema
-  end
-
-  sub_test_case(".try_convert") do
-    def test_hash
-      batch_size = 1024
-      context = ArrowDataset::ScanOptions.try_convert(schema: @schema,
-                                                      batch_size: batch_size)
-      assert_equal([@schema, batch_size],
-                   [context.schema, context.batch_size])
-    end
-  end
-end