torch-rb 0.12.0 → 0.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24617a6191c4d1e42ff0d5885c0762db75d034ee52817d70004ddc52025890dd
-  data.tar.gz: 7036f3c7fac8a1aac22914ec2811b8771464baeb847052e417b069f3897b3977
+  metadata.gz: 8160298f6299869f699201dd9217785da5ac7d2562f05b12a2df2d4537143886
+  data.tar.gz: cb902f45cb72adaaa2008e181af2af0844e926ef62f1d2ccaa67251cb70c5799
 SHA512:
-  metadata.gz: 044166f526bea5fea0c4314abc55dfaf2410cdd5d7bce764ccb78329754fe284675dc0ecb55cf5a4546587221cf2361c8e215f1e9e067668ea5833c445bea961
-  data.tar.gz: e086d0dcc731fb3e810f8fb91d04d95bf3c5efd9cd35f053238be60479d8a47f2126ddad7ab2705d18a26175c6c4126f12be3f19962c2a69f24ad3ee551252b1
+  metadata.gz: c5077c40cb32414ac7d230430fdcf1cd93f199a81100179a1aa26185d6216c049316bbf35d79968442fd4db5e18414991946a366e6e7e6260a904f001774faa2
+  data.tar.gz: b4f1ff33b47596953c0654b07f1916a88ba247af72c1d4693d9b737802a790256d3232c0419c3a603fdebd51f47360833265326a1f27f8242604ff05766ea690

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.12.2 (2023-01-30)
+
+- Added experimental support for DataPipes
+
+## 0.12.1 (2023-01-29)
+
+- Added `Generator` class
+
 ## 0.12.0 (2022-11-05)
 
 - Updated LibTorch to 1.13.0

data/ext/torch/ext.cpp

@@ -12,6 +12,7 @@ void init_torch(Rice::Module& m);
 void init_backends(Rice::Module& m);
 void init_cuda(Rice::Module& m);
 void init_device(Rice::Module& m);
+void init_generator(Rice::Module& m, Rice::Class& rb_cGenerator);
 void init_ivalue(Rice::Module& m, Rice::Class& rb_cIValue);
 void init_random(Rice::Module& m);
 
@@ -23,6 +24,7 @@ void Init_ext()
   // need to define certain classes up front to keep Rice happy
   auto rb_cIValue = Rice::define_class_under<torch::IValue>(m, "IValue")
     .define_constructor(Rice::Constructor<torch::IValue>());
+  auto rb_cGenerator = Rice::define_class_under<torch::Generator>(m, "Generator");
   auto rb_cTensor = Rice::define_class_under<torch::Tensor>(m, "Tensor");
   auto rb_cTensorOptions = Rice::define_class_under<torch::TensorOptions>(m, "TensorOptions")
     .define_constructor(Rice::Constructor<torch::TensorOptions>());
@@ -38,6 +40,7 @@ void Init_ext()
   init_backends(m);
   init_cuda(m);
   init_device(m);
+  init_generator(m, rb_cGenerator);
   init_ivalue(m, rb_cIValue);
   init_random(m);
 }

data/ext/torch/generator.cpp

@@ -0,0 +1,50 @@
+#include <torch/torch.h>
+
+#include <rice/rice.hpp>
+
+#include "utils.h"
+
+void init_generator(Rice::Module& m, Rice::Class& rb_cGenerator) {
+  // https://github.com/pytorch/pytorch/blob/master/torch/csrc/Generator.cpp
+  rb_cGenerator
+    .add_handler<torch::Error>(handle_error)
+    .define_singleton_function(
+      "new",
+      []() {
+        // TODO support more devices
+        return torch::make_generator<torch::CPUGeneratorImpl>();
+      })
+    .define_method(
+      "device",
+      [](torch::Generator& self) {
+        return self.device();
+      })
+    .define_method(
+      "initial_seed",
+      [](torch::Generator& self) {
+        return self.current_seed();
+      })
+    .define_method(
+      "manual_seed",
+      [](torch::Generator& self, uint64_t seed) {
+        self.set_current_seed(seed);
+        return self;
+      })
+    .define_method(
+      "seed",
+      [](torch::Generator& self) {
+        return self.seed();
+      })
+    .define_method(
+      "state",
+      [](torch::Generator& self) {
+        return self.get_state();
+      })
+    .define_method(
+      "state=",
+      [](torch::Generator& self, const torch::Tensor& state) {
+        self.set_state(state);
+      });
+
+  THPGeneratorClass = rb_cGenerator.value();
+}

data/ext/torch/ruby_arg_parser.cpp

@@ -2,6 +2,7 @@
 
 #include "ruby_arg_parser.h"
 
+VALUE THPGeneratorClass = Qnil;
 VALUE THPVariableClass = Qnil;
 
 static std::unordered_map<std::string, ParameterType> type_map = {
@@ -244,7 +245,7 @@ auto FunctionParameter::check(VALUE obj, int argnum) -> bool
       return size > 0 && FIXNUM_P(obj);
     }
     case ParameterType::FLOAT_LIST: return (RB_TYPE_P(obj, T_ARRAY));
-    case ParameterType::GENERATOR: return false; // return THPGenerator_Check(obj);
+    case ParameterType::GENERATOR: return THPGenerator_Check(obj);
     case ParameterType::BOOL: return obj == Qtrue || obj == Qfalse;
     case ParameterType::STORAGE: return false; // return isStorage(obj);
     // case ParameterType::PYOBJECT: return true;

data/ext/torch/ruby_arg_parser.h

@@ -223,7 +223,7 @@ inline c10::OptionalArray<c10::SymInt> RubyArgs::symintlistOptional(int i) {
 
 inline c10::optional<at::Generator> RubyArgs::generator(int i) {
   if (NIL_P(args[i])) return c10::nullopt;
-  throw std::runtime_error("generator not supported yet");
+  return Rice::detail::From_Ruby<torch::Generator>().convert(args[i]);
 }
 
 inline at::Storage RubyArgs::storage(int i) {

data/ext/torch/utils.h

@@ -17,6 +17,7 @@ inline void handle_error(torch::Error const & ex) {
 
 // keep THP prefix for now to make it easier to compare code
 
+extern VALUE THPGeneratorClass;
 extern VALUE THPVariableClass;
 
 inline VALUE THPUtils_internSymbol(const std::string& str) {
@@ -44,6 +45,10 @@ inline bool THPUtils_checkScalar(VALUE obj) {
   return FIXNUM_P(obj) || RB_FLOAT_TYPE_P(obj) || RB_TYPE_P(obj, T_COMPLEX);
 }
 
+inline bool THPGenerator_Check(VALUE obj) {
+  return rb_obj_is_kind_of(obj, THPGeneratorClass);
+}
+
 inline bool THPVariable_Check(VALUE obj) {
   return rb_obj_is_kind_of(obj, THPVariableClass);
 }

data/lib/torch/utils/data/data_pipes/filter_iter_data_pipe.rb

@@ -0,0 +1,37 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        class FilterIterDataPipe < IterDataPipe
+          functional_datapipe :filter
+
+          def initialize(datapipe, &block)
+            @datapipe = datapipe
+            @filter_fn = block
+          end
+
+          def each
+            @datapipe.each do |data|
+              filtered = return_if_true(data)
+              if non_empty?(filtered)
+                yield filtered
+              else
+                Iter::StreamWrapper.close_streams(data)
+              end
+            end
+          end
+
+          def return_if_true(data)
+            condition = @filter_fn.call(data)
+
+            data if condition
+          end
+
+          def non_empty?(data)
+            !data.nil?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/file_lister.rb

@@ -0,0 +1,74 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class FileLister < IterDataPipe
+            def initialize(
+              root = ".",
+              masks = "",
+              recursive: false,
+              abspath: false,
+              non_deterministic: false,
+              length: -1
+            )
+              super()
+              if root.is_a?(String)
+                root = [root]
+              end
+              if !root.is_a?(IterDataPipe)
+                root = IterableWrapper.new(root)
+              end
+              @datapipe = root
+              @masks = masks
+              @recursive = recursive
+              @abspath = abspath
+              @non_deterministic = non_deterministic
+              @length = length
+            end
+
+            def each(&block)
+              @datapipe.each do |path|
+                get_file_pathnames_from_root(path, @masks, recursive: @recursive, abspath: @abspath, non_deterministic: @non_deterministic, &block)
+              end
+            end
+
+            private
+
+            def get_file_pathnames_from_root(
+              root,
+              masks,
+              recursive: false,
+              abspath: false,
+              non_deterministic: false
+            )
+              if File.file?(root)
+                raise NotImplementedYet
+              else
+                pattern = recursive ? "**/*" : "*"
+                paths = Dir.glob(pattern, base: root)
+                paths = paths.sort if non_deterministic
+                paths.each do |f|
+                  if abspath
+                    raise NotImplementedYet
+                  end
+                  if match_masks(f, masks)
+                    yield File.join(root, f)
+                  end
+                end
+              end
+            end
+
+            def match_masks(name, masks)
+              if masks.empty?
+                return true
+              end
+
+              raise NotImplementedYet
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/file_opener.rb

@@ -0,0 +1,51 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class FileOpener < IterDataPipe
+            def initialize(datapipe, mode: "r", encoding: nil, length: -1)
+              super()
+              @datapipe = datapipe
+              @mode = mode
+              @encoding = encoding
+
+              if !["b", "t", "rb", "rt", "r"].include?(@mode)
+                raise ArgumentError, "Invalid mode #{mode}"
+              end
+
+              if mode.include?("b") && !encoding.nil?
+                raise ArgumentError, "binary mode doesn't take an encoding argument"
+              end
+
+              @length = length
+            end
+
+            def each(&block)
+              get_file_binaries_from_pathnames(@datapipe, @mode, encoding: @encoding, &block)
+            end
+
+            private
+
+            def get_file_binaries_from_pathnames(pathnames, mode, encoding: nil)
+              if !pathnames.is_a?(Enumerable)
+                pathnames = [pathnames]
+              end
+
+              if ["b", "t"].include?(mode)
+                mode = "r#{mode}"
+              end
+
+              pathnames.each do |pathname|
+                if !pathname.is_a?(String)
+                  raise TypeError, "Expected string type for pathname, but got #{pathname.class.name}"
+                end
+                yield pathname, StreamWrapper.new(File.open(pathname, mode, encoding: encoding))
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/iterable_wrapper.rb

@@ -0,0 +1,30 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class IterableWrapper < IterDataPipe
+            def initialize(iterable, deepcopy: true)
+              @iterable = iterable
+              @deepcopy = deepcopy
+            end
+
+            def each
+              source_data = @iterable
+              if @deepcopy
+                source_data = Marshal.load(Marshal.dump(@iterable))
+              end
+              source_data.each do |data|
+                yield data
+              end
+            end
+
+            def length
+              @iterable.length
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/stream_wrapper.rb

@@ -0,0 +1,30 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class StreamWrapper
+            def initialize(file_obj)
+              @file_obj = file_obj
+            end
+
+            def gets(...)
+              @file_obj.gets(...)
+            end
+
+            def close
+              @file_obj.close
+            end
+
+            # TODO improve
+            def self.close_streams(cls)
+              if cls.is_a?(StreamWrapper)
+                cls.close
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter_data_pipe.rb

@@ -0,0 +1,41 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        class IterDataPipe < IterableDataset
+          def self.functional_datapipe(name)
+            IterDataPipe.register_datapipe_as_function(name, self)
+          end
+
+          def self.functions
+            @functions ||= {}
+          end
+
+          def self.register_datapipe_as_function(function_name, cls_to_register)
+            if functions.include?(function_name)
+              raise Error, "Unable to add DataPipe function name #{function_name} as it is already taken"
+            end
+
+            function = lambda do |source_dp, *args, **options, &block|
+              cls_to_register.new(source_dp, *args, **options, &block)
+            end
+            functions[function_name] = function
+
+            define_method function_name do |*args, **options, &block|
+              IterDataPipe.functions[function_name].call(self, *args, **options, &block)
+            end
+          end
+
+          def reset
+            # no-op, but subclasses can override
+          end
+
+          def each(&block)
+            reset
+            @source_datapipe.each(&block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/iterable_dataset.rb

@@ -0,0 +1,13 @@
+module Torch
+  module Utils
+    module Data
+      class IterableDataset < Dataset
+        include Enumerable
+
+        def each
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/torch/version.rb

@@ -1,3 +1,3 @@
 module Torch
-  VERSION = "0.12.0"
+  VERSION = "0.12.2"
 end

data/lib/torch.rb

@@ -187,6 +187,13 @@ require "torch/nn/init"
 require "torch/utils/data"
 require "torch/utils/data/data_loader"
 require "torch/utils/data/dataset"
+require "torch/utils/data/iterable_dataset"
+require "torch/utils/data/data_pipes/iter_data_pipe"
+require "torch/utils/data/data_pipes/filter_iter_data_pipe"
+require "torch/utils/data/data_pipes/iter/file_lister"
+require "torch/utils/data/data_pipes/iter/file_opener"
+require "torch/utils/data/data_pipes/iter/iterable_wrapper"
+require "torch/utils/data/data_pipes/iter/stream_wrapper"
 require "torch/utils/data/subset"
 require "torch/utils/data/tensor_dataset"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: torch-rb
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.12.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-05 00:00:00.000000000 Z
+date: 2023-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -44,6 +44,7 @@ files:
 - ext/torch/extconf.rb
 - ext/torch/fft.cpp
 - ext/torch/fft_functions.h
+- ext/torch/generator.cpp
 - ext/torch/ivalue.cpp
 - ext/torch/linalg.cpp
 - ext/torch/linalg_functions.h
@@ -206,7 +207,14 @@ files:
 - lib/torch/tensor.rb
 - lib/torch/utils/data.rb
 - lib/torch/utils/data/data_loader.rb
+- lib/torch/utils/data/data_pipes/filter_iter_data_pipe.rb
+- lib/torch/utils/data/data_pipes/iter/file_lister.rb
+- lib/torch/utils/data/data_pipes/iter/file_opener.rb
+- lib/torch/utils/data/data_pipes/iter/iterable_wrapper.rb
+- lib/torch/utils/data/data_pipes/iter/stream_wrapper.rb
+- lib/torch/utils/data/data_pipes/iter_data_pipe.rb
 - lib/torch/utils/data/dataset.rb
+- lib/torch/utils/data/iterable_dataset.rb
 - lib/torch/utils/data/subset.rb
 - lib/torch/utils/data/tensor_dataset.rb
 - lib/torch/version.rb
@@ -229,7 +237,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: Deep learning for Ruby, powered by LibTorch