torch-rb 0.12.1 → 0.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce853372191c85509a65417abeaa05c484976f24681246babf9bd00f8db16df1
-  data.tar.gz: 3327004180566a194c7de8288c260b8fe9487d80c25493d82e041cf4fc0062e2
+  metadata.gz: 8160298f6299869f699201dd9217785da5ac7d2562f05b12a2df2d4537143886
+  data.tar.gz: cb902f45cb72adaaa2008e181af2af0844e926ef62f1d2ccaa67251cb70c5799
 SHA512:
-  metadata.gz: d8b6b0f7bd8b79963931b6b28b6a6cee59be18b8f185e2acadc22487a27b5793edabf0fb8b80857c5dfc0eb036a27b67a55750b8dd3c8eb1d199f06323e2b919
-  data.tar.gz: ffcafd2e9e99d6654f9689dd021c874cf53847833e26207853b0051056ae7cfe0b5ff202036e9a38264198facfa2c66f83b6ee51815131b63aa350415139b614
+  metadata.gz: c5077c40cb32414ac7d230430fdcf1cd93f199a81100179a1aa26185d6216c049316bbf35d79968442fd4db5e18414991946a366e6e7e6260a904f001774faa2
+  data.tar.gz: b4f1ff33b47596953c0654b07f1916a88ba247af72c1d4693d9b737802a790256d3232c0419c3a603fdebd51f47360833265326a1f27f8242604ff05766ea690

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.12.2 (2023-01-30)
+
+- Added experimental support for DataPipes
+
 ## 0.12.1 (2023-01-29)
 
 - Added `Generator` class

data/lib/torch/utils/data/data_pipes/filter_iter_data_pipe.rb

@@ -0,0 +1,37 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        class FilterIterDataPipe < IterDataPipe
+          functional_datapipe :filter
+
+          def initialize(datapipe, &block)
+            @datapipe = datapipe
+            @filter_fn = block
+          end
+
+          def each
+            @datapipe.each do |data|
+              filtered = return_if_true(data)
+              if non_empty?(filtered)
+                yield filtered
+              else
+                Iter::StreamWrapper.close_streams(data)
+              end
+            end
+          end
+
+          def return_if_true(data)
+            condition = @filter_fn.call(data)
+
+            data if condition
+          end
+
+          def non_empty?(data)
+            !data.nil?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/file_lister.rb

@@ -0,0 +1,74 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class FileLister < IterDataPipe
+            def initialize(
+              root = ".",
+              masks = "",
+              recursive: false,
+              abspath: false,
+              non_deterministic: false,
+              length: -1
+            )
+              super()
+              if root.is_a?(String)
+                root = [root]
+              end
+              if !root.is_a?(IterDataPipe)
+                root = IterableWrapper.new(root)
+              end
+              @datapipe = root
+              @masks = masks
+              @recursive = recursive
+              @abspath = abspath
+              @non_deterministic = non_deterministic
+              @length = length
+            end
+
+            def each(&block)
+              @datapipe.each do |path|
+                get_file_pathnames_from_root(path, @masks, recursive: @recursive, abspath: @abspath, non_deterministic: @non_deterministic, &block)
+              end
+            end
+
+            private
+
+            def get_file_pathnames_from_root(
+              root,
+              masks,
+              recursive: false,
+              abspath: false,
+              non_deterministic: false
+            )
+              if File.file?(root)
+                raise NotImplementedYet
+              else
+                pattern = recursive ? "**/*" : "*"
+                paths = Dir.glob(pattern, base: root)
+                paths = paths.sort if non_deterministic
+                paths.each do |f|
+                  if abspath
+                    raise NotImplementedYet
+                  end
+                  if match_masks(f, masks)
+                    yield File.join(root, f)
+                  end
+                end
+              end
+            end
+
+            def match_masks(name, masks)
+              if masks.empty?
+                return true
+              end
+
+              raise NotImplementedYet
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/file_opener.rb

@@ -0,0 +1,51 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class FileOpener < IterDataPipe
+            def initialize(datapipe, mode: "r", encoding: nil, length: -1)
+              super()
+              @datapipe = datapipe
+              @mode = mode
+              @encoding = encoding
+
+              if !["b", "t", "rb", "rt", "r"].include?(@mode)
+                raise ArgumentError, "Invalid mode #{mode}"
+              end
+
+              if mode.include?("b") && !encoding.nil?
+                raise ArgumentError, "binary mode doesn't take an encoding argument"
+              end
+
+              @length = length
+            end
+
+            def each(&block)
+              get_file_binaries_from_pathnames(@datapipe, @mode, encoding: @encoding, &block)
+            end
+
+            private
+
+            def get_file_binaries_from_pathnames(pathnames, mode, encoding: nil)
+              if !pathnames.is_a?(Enumerable)
+                pathnames = [pathnames]
+              end
+
+              if ["b", "t"].include?(mode)
+                mode = "r#{mode}"
+              end
+
+              pathnames.each do |pathname|
+                if !pathname.is_a?(String)
+                  raise TypeError, "Expected string type for pathname, but got #{pathname.class.name}"
+                end
+                yield pathname, StreamWrapper.new(File.open(pathname, mode, encoding: encoding))
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/iterable_wrapper.rb

@@ -0,0 +1,30 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class IterableWrapper < IterDataPipe
+            def initialize(iterable, deepcopy: true)
+              @iterable = iterable
+              @deepcopy = deepcopy
+            end
+
+            def each
+              source_data = @iterable
+              if @deepcopy
+                source_data = Marshal.load(Marshal.dump(@iterable))
+              end
+              source_data.each do |data|
+                yield data
+              end
+            end
+
+            def length
+              @iterable.length
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter/stream_wrapper.rb

@@ -0,0 +1,30 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        module Iter
+          class StreamWrapper
+            def initialize(file_obj)
+              @file_obj = file_obj
+            end
+
+            def gets(...)
+              @file_obj.gets(...)
+            end
+
+            def close
+              @file_obj.close
+            end
+
+            # TODO improve
+            def self.close_streams(cls)
+              if cls.is_a?(StreamWrapper)
+                cls.close
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/data_pipes/iter_data_pipe.rb

@@ -0,0 +1,41 @@
+module Torch
+  module Utils
+    module Data
+      module DataPipes
+        class IterDataPipe < IterableDataset
+          def self.functional_datapipe(name)
+            IterDataPipe.register_datapipe_as_function(name, self)
+          end
+
+          def self.functions
+            @functions ||= {}
+          end
+
+          def self.register_datapipe_as_function(function_name, cls_to_register)
+            if functions.include?(function_name)
+              raise Error, "Unable to add DataPipe function name #{function_name} as it is already taken"
+            end
+
+            function = lambda do |source_dp, *args, **options, &block|
+              cls_to_register.new(source_dp, *args, **options, &block)
+            end
+            functions[function_name] = function
+
+            define_method function_name do |*args, **options, &block|
+              IterDataPipe.functions[function_name].call(self, *args, **options, &block)
+            end
+          end
+
+          def reset
+            # no-op, but subclasses can override
+          end
+
+          def each(&block)
+            reset
+            @source_datapipe.each(&block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/torch/utils/data/iterable_dataset.rb

@@ -0,0 +1,13 @@
+module Torch
+  module Utils
+    module Data
+      class IterableDataset < Dataset
+        include Enumerable
+
+        def each
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/torch/version.rb

@@ -1,3 +1,3 @@
 module Torch
-  VERSION = "0.12.1"
+  VERSION = "0.12.2"
 end

data/lib/torch.rb

@@ -187,6 +187,13 @@ require "torch/nn/init"
 require "torch/utils/data"
 require "torch/utils/data/data_loader"
 require "torch/utils/data/dataset"
+require "torch/utils/data/iterable_dataset"
+require "torch/utils/data/data_pipes/iter_data_pipe"
+require "torch/utils/data/data_pipes/filter_iter_data_pipe"
+require "torch/utils/data/data_pipes/iter/file_lister"
+require "torch/utils/data/data_pipes/iter/file_opener"
+require "torch/utils/data/data_pipes/iter/iterable_wrapper"
+require "torch/utils/data/data_pipes/iter/stream_wrapper"
 require "torch/utils/data/subset"
 require "torch/utils/data/tensor_dataset"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: torch-rb
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.12.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-30 00:00:00.000000000 Z
+date: 2023-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -207,7 +207,14 @@ files:
 - lib/torch/tensor.rb
 - lib/torch/utils/data.rb
 - lib/torch/utils/data/data_loader.rb
+- lib/torch/utils/data/data_pipes/filter_iter_data_pipe.rb
+- lib/torch/utils/data/data_pipes/iter/file_lister.rb
+- lib/torch/utils/data/data_pipes/iter/file_opener.rb
+- lib/torch/utils/data/data_pipes/iter/iterable_wrapper.rb
+- lib/torch/utils/data/data_pipes/iter/stream_wrapper.rb
+- lib/torch/utils/data/data_pipes/iter_data_pipe.rb
 - lib/torch/utils/data/dataset.rb
+- lib/torch/utils/data/iterable_dataset.rb
 - lib/torch/utils/data/subset.rb
 - lib/torch/utils/data/tensor_dataset.rb
 - lib/torch/version.rb