torch-rb 0.12.1 → 0.13.0

data/ext/torch/backends.cpp

@@ -14,4 +14,8 @@ void init_backends(Rice::Module& m) {
   Rice::define_module_under(rb_mBackends, "MKL")
     .add_handler<torch::Error>(handle_error)
     .define_singleton_function("available?", &torch::hasMKL);
+
+  Rice::define_module_under(rb_mBackends, "MPS")
+    .add_handler<torch::Error>(handle_error)
+    .define_singleton_function("available?", &torch::hasMPS);
 }

data/ext/torch/tensor.cpp

@@ -35,17 +35,17 @@ std::vector<TensorIndex> index_vector(Array a) {
     if (obj.is_instance_of(rb_cInteger)) {
       indices.push_back(Rice::detail::From_Ruby<int64_t>().convert(obj.value()));
     } else if (obj.is_instance_of(rb_cRange)) {
-      torch::optional<int64_t> start_index = torch::nullopt;
-      torch::optional<int64_t> stop_index = torch::nullopt;
+      torch::optional<c10::SymInt> start_index = torch::nullopt;
+      torch::optional<c10::SymInt> stop_index = torch::nullopt;
 
       Object begin = obj.call("begin");
       if (!begin.is_nil()) {
-        start_index = Rice::detail::From_Ruby<int64_t>().convert(begin.value());
+        start_index = c10::SymInt(Rice::detail::From_Ruby<int64_t>().convert(begin.value()));
       }
 
       Object end = obj.call("end");
       if (!end.is_nil()) {
-        stop_index = Rice::detail::From_Ruby<int64_t>().convert(end.value());
+        stop_index = c10::SymInt(Rice::detail::From_Ruby<int64_t>().convert(end.value()));
       }
 
       Object exclude_end = obj.call("exclude_end?");

data/ext/torch/utils.h

@@ -6,7 +6,7 @@
 #include <rice/stl.hpp>
 
 static_assert(
-  TORCH_VERSION_MAJOR == 1 && TORCH_VERSION_MINOR == 13,
+  TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR == 0,
   "Incompatible LibTorch version"
 );
 

data/lib/torch/utils/data/data_pipes/filter_iter_data_pipe.rb

@@ -0,0 +1,37 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        class FilterIterDataPipe < IterDataPipe
+          functional_datapipe :filter
+
+          def initialize(datapipe, &block)
+            @datapipe = datapipe
+            @filter_fn = block
+          end
+
+          def each
+            @datapipe.each do |data|
+              filtered = return_if_true(data)
+              if non_empty?(filtered)
+                yield filtered
+              else
+                Iter::StreamWrapper.close_streams(data)
+              end
+            end
+          end
+
+          def return_if_true(data)
+            condition = @filter_fn.call(data)
+
+            data if condition
+          end
+
+          def non_empty?(data)
+            !data.nil?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/file_lister.rb

@@ -0,0 +1,74 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class FileLister < IterDataPipe
+            def initialize(
+              root = ".",
+              masks = "",
+              recursive: false,
+              abspath: false,
+              non_deterministic: false,
+              length: -1
+            )
+              super()
+              if root.is_a?(String)
+                root = [root]
+              end
+              if !root.is_a?(IterDataPipe)
+                root = IterableWrapper.new(root)
+              end
+              @datapipe = root
+              @masks = masks
+              @recursive = recursive
+              @abspath = abspath
+              @non_deterministic = non_deterministic
+              @length = length
+            end
+
+            def each(&block)
+              @datapipe.each do |path|
+                get_file_pathnames_from_root(path, @masks, recursive: @recursive, abspath: @abspath, non_deterministic: @non_deterministic, &block)
+              end
+            end
+
+            private
+
+            def get_file_pathnames_from_root(
+              root,
+              masks,
+              recursive: false,
+              abspath: false,
+              non_deterministic: false
+            )
+              if File.file?(root)
+                raise NotImplementedYet
+              else
+                pattern = recursive ? "**/*" : "*"
+                paths = Dir.glob(pattern, base: root)
+                paths = paths.sort if non_deterministic
+                paths.each do |f|
+                  if abspath
+                    raise NotImplementedYet
+                  end
+                  if match_masks(f, masks)
+                    yield File.join(root, f)
+                  end
+                end
+              end
+            end
+
+            def match_masks(name, masks)
+              if masks.empty?
+                return true
+              end
+
+              raise NotImplementedYet
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/file_opener.rb

@@ -0,0 +1,51 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class FileOpener < IterDataPipe
+            def initialize(datapipe, mode: "r", encoding: nil, length: -1)
+              super()
+              @datapipe = datapipe
+              @mode = mode
+              @encoding = encoding
+
+              if !["b", "t", "rb", "rt", "r"].include?(@mode)
+                raise ArgumentError, "Invalid mode #{mode}"
+              end
+
+              if mode.include?("b") && !encoding.nil?
+                raise ArgumentError, "binary mode doesn't take an encoding argument"
+              end
+
+              @length = length
+            end
+
+            def each(&block)
+              get_file_binaries_from_pathnames(@datapipe, @mode, encoding: @encoding, &block)
+            end
+
+            private
+
+            def get_file_binaries_from_pathnames(pathnames, mode, encoding: nil)
+              if !pathnames.is_a?(Enumerable)
+                pathnames = [pathnames]
+              end
+
+              if ["b", "t"].include?(mode)
+                mode = "r#{mode}"
+              end
+
+              pathnames.each do |pathname|
+                if !pathname.is_a?(String)
+                  raise TypeError, "Expected string type for pathname, but got #{pathname.class.name}"
+                end
+                yield pathname, StreamWrapper.new(File.open(pathname, mode, encoding: encoding))
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/iterable_wrapper.rb

@@ -0,0 +1,30 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class IterableWrapper < IterDataPipe
+            def initialize(iterable, deepcopy: true)
+              @iterable = iterable
+              @deepcopy = deepcopy
+            end
+
+            def each
+              source_data = @iterable
+              if @deepcopy
+                source_data = Marshal.load(Marshal.dump(@iterable))
+              end
+              source_data.each do |data|
+                yield data
+              end
+            end
+
+            def length
+              @iterable.length
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/stream_wrapper.rb

@@ -0,0 +1,30 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class StreamWrapper
+            def initialize(file_obj)
+              @file_obj = file_obj
+            end
+
+            def gets(...)
+              @file_obj.gets(...)
+            end
+
+            def close
+              @file_obj.close
+            end
+
+            # TODO improve
+            def self.close_streams(cls)
+              if cls.is_a?(StreamWrapper)
+                cls.close
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter_data_pipe.rb

@@ -0,0 +1,41 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        class IterDataPipe < IterableDataset
+          def self.functional_datapipe(name)
+            IterDataPipe.register_datapipe_as_function(name, self)
+          end
+
+          def self.functions
+            @functions ||= {}
+          end
+
+          def self.register_datapipe_as_function(function_name, cls_to_register)
+            if functions.include?(function_name)
+              raise Error, "Unable to add DataPipe function name #{function_name} as it is already taken"
+            end
+
+            function = lambda do |source_dp, *args, **options, &block|
+              cls_to_register.new(source_dp, *args, **options, &block)
+            end
+            functions[function_name] = function
+
+            define_method function_name do |*args, **options, &block|
+              IterDataPipe.functions[function_name].call(self, *args, **options, &block)
+            end
+          end
+
+          def reset
+            # no-op, but subclasses can override
+          end
+
+          def each(&block)
+            reset
+            @source_datapipe.each(&block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/iterable_dataset.rb

@@ -0,0 +1,13 @@
+module Torch
+  module Utils
+    module Data
+      class IterableDataset < Dataset
+        include Enumerable
+
+        def each
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/torch/version.rb

@@ -1,3 +1,3 @@
 module Torch
-  VERSION = "0.12.1"
+  VERSION = "0.13.0"
 end

data/lib/torch.rb

@@ -187,6 +187,13 @@ require "torch/nn/init"
 require "torch/utils/data"
 require "torch/utils/data/data_loader"
 require "torch/utils/data/dataset"
+require "torch/utils/data/iterable_dataset"
+require "torch/utils/data/data_pipes/iter_data_pipe"
+require "torch/utils/data/data_pipes/filter_iter_data_pipe"
+require "torch/utils/data/data_pipes/iter/file_lister"
+require "torch/utils/data/data_pipes/iter/file_opener"
+require "torch/utils/data/data_pipes/iter/iterable_wrapper"
+require "torch/utils/data/data_pipes/iter/stream_wrapper"
 require "torch/utils/data/subset"
 require "torch/utils/data/tensor_dataset"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: torch-rb
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.13.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-30 00:00:00.000000000 Z
+date: 2023-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -207,7 +207,14 @@ files:
 - lib/torch/tensor.rb
 - lib/torch/utils/data.rb
 - lib/torch/utils/data/data_loader.rb
+- lib/torch/utils/data/data_pipes/filter_iter_data_pipe.rb
+- lib/torch/utils/data/data_pipes/iter/file_lister.rb
+- lib/torch/utils/data/data_pipes/iter/file_opener.rb
+- lib/torch/utils/data/data_pipes/iter/iterable_wrapper.rb
+- lib/torch/utils/data/data_pipes/iter/stream_wrapper.rb
+- lib/torch/utils/data/data_pipes/iter_data_pipe.rb
 - lib/torch/utils/data/dataset.rb
+- lib/torch/utils/data/iterable_dataset.rb
 - lib/torch/utils/data/subset.rb
 - lib/torch/utils/data/tensor_dataset.rb
 - lib/torch/version.rb
@@ -223,14 +230,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Deep learning for Ruby, powered by LibTorch