torch-rb 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ff0920ba955063c03309fdb45ecf228b51c556508bea30b510d6bf652c1d0b18
-  data.tar.gz: 481dccf6a8e929230033f74c82bc9d292ef38ea219e2cb2cc61ca0b0c5457403
+  metadata.gz: 30089078de4039df111087e5c27e0cb10d6f36398c0e8d5cc774e9b642a8e133
+  data.tar.gz: 89eb9e183b395dd67cd9cf228749cf26402993bb561de973f1ba7438bc372b04
 SHA512:
-  metadata.gz: cd6c8fd9db4af15640217c09813c4f86d3f66360202d30711015c4f34552853ef281d5614fd78dc274d405da0d9f46f08a2359475ae1b0721143db49183faf5d
-  data.tar.gz: ee638c08458e0d2a8fac52e29c45d1347a74847ca7d8dab3a9a573afd887814d4c578c4e1a7fb80b204222785b27e21ca3c138f3face681e98e63c1bc02a9a7f
+  metadata.gz: 027a069b00ac1329c007ddaf471a21b57a82a823ad974a937f832d17720b8e26474c64c79e9a29ec71bac433abb3d74d6a7cf407f0a983bb3c0cafb5b5c7532f
+  data.tar.gz: 6d7ef10b53db0df39eda13d07aa9b52b4afac0965674919b5cc517e7b53f59a9010cb647e50d62bc06154f7d8f3ef632d5897e4f7774372d7ab1b44b2cb6ca82

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.1.2 (2019-11-27)
+
+- Added SGD optimizer
+- Added support for gradient to `backward` method
+- Added `argmax`, `eq`, `leaky_relu`, `prelu`, and `reshape` methods
+- Improved indexing
+- Fixed `zero_grad`
+- Fixed error with infinite values
+
 ## 0.1.1 (2019-11-26)
 
 - Added support for `uint8` and `int8` types

data/README.md

@@ -2,7 +2,7 @@
 
 :fire: Deep learning for Ruby, powered by [LibTorch](https://pytorch.org)
 
-This gem is currently experimental. There may be breaking changes between each release.
+This gem is currently experimental. There may be breaking changes between each release. Please report any issues you experience.
 
 [![Build Status](https://travis-ci.org/ankane/torch-rb.svg?branch=master)](https://travis-ci.org/ankane/torch-rb)
 
@@ -223,7 +223,7 @@ class Net < Torch::NN::Module
 end
 ```
 
-And run
+Create an instance of it
 
 ```ruby
 net = Net.new
@@ -231,6 +231,58 @@ input = Torch.randn(1, 1, 32, 32)
 net.call(input)
 ```
 
+Get trainable parameters
+
+```ruby
+net.parameters
+```
+
+Zero the gradient buffers and backprop with random gradients
+
+```ruby
+net.zero_grad
+out.backward(Torch.randn(1, 10))
+```
+
+Define a loss function
+
+```ruby
+output = net.call(input)
+target = Torch.randn(10)
+target = target.view(1, -1)
+criterion = Torch::NN::MSELoss.new
+loss = criterion.call(output, target)
+```
+
+Backprop
+
+```ruby
+net.zero_grad
+p net.conv1.bias.grad
+loss.backward
+p net.conv1.bias.grad
+```
+
+Update the weights
+
+```ruby
+learning_rate = 0.01
+net.parameters.each do |f|
+  f.data.sub!(f.grad.data * learning_rate)
+end
+```
+
+Use an optimizer
+
+```ruby
+optimizer = Torch::Optim::SGD.new(net.parameters, lr: 0.01)
+optimizer.zero_grad
+output = net.call(input)
+loss = criterion.call(output, target)
+loss.backward
+optimizer.step
+```
+
 ### Tensor Creation
 
 Here’s a list of functions to create tensors (descriptions from the [C++ docs](https://pytorch.org/cppdocs/notes/tensor_creation.html)):
@@ -351,7 +403,7 @@ To get started with development:
 
 ```sh
 git clone https://github.com/ankane/torch-rb.git
-cd torch
+cd torch-rb
 bundle install
 bundle exec rake compile
 bundle exec rake test

data/ext/torch/ext.cpp

@@ -196,6 +196,16 @@ void Init_ext()
       *[](torch::Tensor& input, int64_t dim, bool keepdim) {
         return torch::sum(input, dim, keepdim);
       })
+    .define_singleton_method(
+      "_argmax",
+      *[](torch::Tensor& input) {
+        return torch::argmax(input);
+      })
+    .define_singleton_method(
+      "_argmax_dim",
+      *[](torch::Tensor& input, int64_t dim, bool keepdim) {
+        return torch::argmax(input, dim, keepdim);
+      })
     .define_singleton_method(
       "_norm",
       *[](torch::Tensor& input) {
@@ -236,6 +246,11 @@ void Init_ext()
       *[](torch::Tensor& input, torch::Tensor& other) {
         return torch::matmul(input, other);
       })
+    .define_singleton_method(
+      "_eq",
+      *[](torch::Tensor& input, torch::Tensor& other) {
+        return torch::eq(input, other);
+      })
     .define_singleton_method(
       "_add",
       *[](torch::Tensor& input, torch::Tensor& other) {
@@ -301,15 +316,30 @@ void Init_ext()
       *[](torch::Tensor& input) {
         return torch::neg(input);
       })
+    .define_singleton_method(
+      "_reshape",
+      *[](torch::Tensor& input, IntArrayRef shape) {
+        return torch::reshape(input, shape);
+      })
     .define_singleton_method(
       "relu",
       *[](torch::Tensor& input) {
         return torch::relu(input);
       })
+    .define_singleton_method(
+      "prelu",
+      *[](torch::Tensor& input, torch::Tensor& weight) {
+        return torch::prelu(input, weight);
+      })
+    .define_singleton_method(
+      "leaky_relu",
+      *[](torch::Tensor& input, Scalar negative_slope = 0.01) {
+        return torch::leaky_relu(input, negative_slope);
+      })
     .define_singleton_method(
       "conv2d",
-      *[](torch::Tensor& input, torch::Tensor& weight, torch::Tensor& bias) {
-        return torch::conv2d(input, weight, bias);
+      *[](torch::Tensor& input, torch::Tensor& weight, torch::Tensor& bias, IntArrayRef stride, IntArrayRef padding) {
+        return torch::conv2d(input, weight, bias, stride, padding);
       })
     .define_singleton_method(
       "linear",
@@ -321,6 +351,11 @@ void Init_ext()
       *[](torch::Tensor& input, IntArrayRef kernel_size) {
         return torch::max_pool2d(input, kernel_size);
       })
+    .define_singleton_method(
+      "avg_pool2d",
+      *[](torch::Tensor& input, IntArrayRef kernel_size) {
+        return torch::avg_pool2d(input, kernel_size);
+      })
     .define_singleton_method(
       "mse_loss",
       *[](torch::Tensor& input, torch::Tensor& target, std::string reduction) {
@@ -366,9 +401,14 @@ void Init_ext()
         return self.detach_();
       })
     .define_method(
-      "_access",
-      *[](torch::Tensor& self, int64_t index) {
-        return self[index];
+      "_select",
+      *[](torch::Tensor& self, int64_t dim, int64_t index) {
+        return self.select(dim, index);
+      })
+    .define_method(
+      "_slice",
+      *[](torch::Tensor& self, int64_t dim, int64_t start, int64_t end, int64_t step) {
+        return self.slice(dim, start, end, step);
       })
     .define_method(
       "_requires_grad!",
@@ -376,10 +416,15 @@ void Init_ext()
         return self.set_requires_grad(requires_grad);
       })
     .define_method(
-      "backward",
+      "_backward",
       *[](torch::Tensor& self) {
         return self.backward();
       })
+    .define_method(
+      "_backward_gradient",
+      *[](torch::Tensor& self, const torch::Tensor& gradient) {
+        return self.backward(gradient);
+      })
     .define_method(
       "grad",
       *[](torch::Tensor& self) {
@@ -390,6 +435,11 @@ void Init_ext()
       *[](torch::Tensor& self) {
         return (int) at::typeMetaToScalarType(self.dtype());
       })
+    .define_method(
+      "_type",
+      *[](torch::Tensor& self, int dtype) {
+        return self.toType((torch::ScalarType) dtype);
+      })
     .define_method(
       "_layout",
       *[](torch::Tensor& self) {
@@ -434,6 +484,11 @@ void Init_ext()
       *[](torch::Tensor& self, int64_t dim) {
         return self.log_softmax(dim);
       })
+    .define_method(
+      "data",
+      *[](torch::Tensor& self) {
+        return self.data();
+      })
     .define_method(
       "_data",
       *[](torch::Tensor& self) {
@@ -553,9 +608,15 @@ void Init_ext()
       });
 
   Class rb_cParameter = define_class_under<torch::autograd::Variable, torch::Tensor>(rb_mNN, "Parameter")
+    // TODO return grad or nil to remove need for 2nd function
     .define_method(
-      "grad",
+      "_grad",
       *[](torch::autograd::Variable& self) {
         return self.grad();
+      })
+    .define_method(
+      "_grad_defined",
+      *[](torch::autograd::Variable& self) {
+        return self.grad().defined();
       });
 }

data/lib/torch.rb

@@ -6,6 +6,10 @@ require "torch/inspector"
 require "torch/tensor"
 require "torch/version"
 
+# optim
+require "torch/optim/optimizer"
+require "torch/optim/sgd"
+
 # nn
 require "torch/nn/module"
 require "torch/nn/init"
@@ -55,9 +59,13 @@ module Torch
 
   class << self
     # Torch.float, Torch.long, etc
-    DTYPE_TO_ENUM.each_key do |type|
-      define_method(type) do
-        type
+    DTYPE_TO_ENUM.each_key do |dtype|
+      define_method(dtype) do
+        dtype
+      end
+
+      Tensor.define_method(dtype) do
+        type(dtype)
       end
     end
 
@@ -240,6 +248,18 @@ module Torch
       end
     end
 
+    def argmax(input, dim = nil, keepdim: false)
+      if dim
+        _argmax_dim(input, dim, keepdim)
+      else
+        _argmax(input)
+      end
+    end
+
+    def eq(input, other)
+      _eq(input, other)
+    end
+
     def norm(input)
       _norm(input)
     end
@@ -276,6 +296,10 @@ module Torch
       _matmul(input, other)
     end
 
+    def reshape(input, shape)
+      _reshape(input, shape)
+    end
+
     private
 
     def execute_op(op, input, other, out: nil)

data/lib/torch/inspector.rb

@@ -1,5 +1,6 @@
 module Torch
   module Inspector
+    # TODO make more performance, especially when summarizing
     def inspect
       data =
         if numel == 0
@@ -20,7 +21,7 @@ module Torch
           if floating_point?
             sci = max / min.to_f > 1000 || max > 1e8 || min < 1e-4
 
-            all_int = values.all? { |v| v == v.to_i }
+            all_int = values.all? { |v| v.finite? && v == v.to_i }
             decimal = all_int ? 1 : 4
 
             total += sci ? 10 : decimal + 1 + max.to_i.to_s.size
@@ -35,7 +36,9 @@ module Torch
             fmt = "%#{total}d"
           end
 
-          inspect_level(to_a, fmt, dim - 1)
+          summarize = numel > 1000
+
+          inspect_level(to_a, fmt, dim - 1, 0, summarize)
         end
 
       attributes = []
@@ -51,11 +54,30 @@ module Torch
 
     private
 
-    def inspect_level(arr, fmt, total, level = 0)
+    # TODO DRY code
+    def inspect_level(arr, fmt, total, level, summarize)
       if level == total
-        "[#{arr.map { |v| fmt % v }.join(", ")}]"
+        cols =
+          if summarize && arr.size > 7
+            arr[0..2].map { |v| fmt % v } +
+            ["..."] +
+            arr[-3..-1].map { |v| fmt % v }
+          else
+            arr.map { |v| fmt % v }
+          end
+
+        "[#{cols.join(", ")}]"
       else
-        "[#{arr.map { |row| inspect_level(row, fmt, total, level + 1) }.join(",#{"\n" * (total - level)}#{" " * (level + 8)}")}]"
+        rows =
+          if summarize && arr.size > 7
+            arr[0..2].map { |row| inspect_level(row, fmt, total, level + 1, summarize) } +
+            ["..."] +
+            arr[-3..-1].map { |row| inspect_level(row, fmt, total, level + 1, summarize) }
+          else
+            arr.map { |row| inspect_level(row, fmt, total, level + 1, summarize) }
+          end
+
+        "[#{rows.join(",#{"\n" * (total - level)}#{" " * (level + 8)}")}]"
       end
     end
   end

data/lib/torch/nn/conv2d.rb

@@ -3,13 +3,12 @@ module Torch
     class Conv2d < Module
       attr_reader :bias, :weight
 
-      def initialize(in_channels, out_channels, kernel_size) #, stride: 1, padding: 0, dilation: 1, groups: 1)
+      def initialize(in_channels, out_channels, kernel_size, stride: 1, padding: 0) #, dilation: 1, groups: 1)
         @in_channels = in_channels
         @out_channels = out_channels
         @kernel_size = pair(kernel_size)
-        @stride = pair(1)
-        # @stride = pair(stride)
-        # @padding = pair(padding)
+        @stride = pair(stride)
+        @padding = pair(padding)
         # @dilation = pair(dilation)
 
         # TODO divide by groups
@@ -29,7 +28,7 @@ module Torch
       end
 
       def call(input)
-        F.conv2d(input, @weight, @bias) # @stride, self.padding, self.dilation, self.groups)
+        F.conv2d(input, @weight, @bias, stride: @stride, padding: @padding) #, @dilation, @groups)
       end
 
       def inspect

data/lib/torch/nn/functional.rb

@@ -6,8 +6,17 @@ module Torch
           Torch.relu(input)
         end
 
-        def conv2d(input, weight, bias)
-          Torch.conv2d(input, weight, bias)
+        def conv2d(input, weight, bias, stride: 1, padding: 0)
+          # TODO pair stride and padding when needed
+          Torch.conv2d(input, weight, bias, stride, padding)
+        end
+
+        def prelu(input, weight)
+          Torch.prelu(input, weight)
+        end
+
+        def leaky_relu(input, negative_slope = 0.01)
+          Torch.leaky_relu(input, negative_slope)
         end
 
         def max_pool2d(input, kernel_size)
@@ -15,6 +24,11 @@ module Torch
           Torch.max_pool2d(input, kernel_size)
         end
 
+        def avg_pool2d(input, kernel_size)
+          kernel_size = [kernel_size, kernel_size] if kernel_size.is_a?(Integer)
+          Torch.avg_pool2d(input, kernel_size)
+        end
+
         def linear(input, weight, bias)
           Torch.linear(input, weight, bias)
         end

data/lib/torch/nn/module.rb

@@ -26,7 +26,6 @@ module Torch
       def zero_grad
         parameters.each do |param|
           if param.grad
-            raise Error, "Not supported yet"
             param.grad.detach!
             param.grad.zero!
           end

data/lib/torch/nn/parameter.rb

@@ -5,6 +5,10 @@ module Torch
         data = Tensor.new unless data
         Tensor._make_subclass(data, requires_grad)
       end
+
+      def grad
+        _grad if _grad_defined
+      end
     end
   end
 end

data/lib/torch/optim/optimizer.rb

@@ -0,0 +1,6 @@
+module Torch
+  module Optim
+    class Optimizer
+    end
+  end
+end

data/lib/torch/optim/sgd.rb

@@ -0,0 +1,28 @@
+module Torch
+  module Optim
+    class SGD < Optimizer
+      def initialize(params, lr:)
+        @params = params
+        @lr = lr
+      end
+
+      def zero_grad
+        @params.each do |param|
+          if param.grad
+            param.grad.detach!
+            param.grad.zero!
+          end
+        end
+      end
+
+      def step
+        @params.each do |param|
+          next unless param.grad
+          d_p = param.grad.data
+          # same as param.data.add!(-@lr, d_p)
+          param.data.sub!(d_p * @lr)
+        end
+      end
+    end
+  end
+end

data/lib/torch/tensor.rb

@@ -28,7 +28,7 @@ module Torch
     end
 
     def to_a
-      reshape(_data, shape)
+      reshape_arr(_data, shape)
     end
 
     def size(dim = nil)
@@ -54,8 +54,12 @@ module Torch
       _data.first
     end
 
-    def data
-      Torch.tensor(to_a)
+    def backward(gradient = nil)
+      if gradient
+        _backward_gradient(gradient)
+      else
+        _backward
+      end
     end
 
     # TODO read directly from memory
@@ -74,8 +78,14 @@ module Torch
       _requires_grad!(requires_grad)
     end
 
+    def type(dtype)
+      enum = DTYPE_TO_ENUM[dtype]
+      raise Error, "Unknown type: #{dtype}" unless enum
+      _type(enum)
+    end
+
     # operations
-    %w(add sub mul div remainder pow neg sum mean num norm min max dot matmul exp log unsqueeze).each do |op|
+    %w(add sub mul div remainder pow neg sum mean num norm min max dot matmul exp log unsqueeze reshape argmax eq).each do |op|
       define_method(op) do |*args, **options, &block|
         if options.any?
           Torch.send(op, self, *args, **options, &block)
@@ -117,18 +127,27 @@ module Torch
       item <=> other
     end
 
-    # TODO use accessor C++ method
-    def [](index, *args)
-      v = _access(index)
-      args.each do |i|
-        v = v._access(i)
+    def [](*indexes)
+      result = self
+      dim = 0
+      indexes.each_with_index do |index|
+        if index.is_a?(Numeric)
+          result = result._select(dim, index)
+        elsif index.is_a?(Range)
+          finish = index.end
+          finish += 1 unless index.exclude_end?
+          result = result._slice(dim, index.begin, finish, 1)
+          dim += 1
+        else
+          raise Error, "Unsupported index type"
+        end
       end
-      v
+      result
     end
 
     private
 
-    def reshape(arr, dims)
+    def reshape_arr(arr, dims)
       if dims.empty?
         arr
       else

data/lib/torch/utils/data/data_loader.rb

@@ -6,6 +6,15 @@ module Torch
           @dataset = dataset
           @batch_size = batch_size
         end
+
+        def each
+          size = @dataset.size
+          start_index = 0
+          while start_index < size
+            yield @dataset[start_index...(start_index + @batch_size)]
+            start_index += @batch_size
+          end
+        end
       end
     end
   end

data/lib/torch/utils/data/tensor_dataset.rb

@@ -7,7 +7,11 @@ module Torch
         end
 
         def [](index)
-          tensors.map { |t| t[index] }
+          @tensors.map { |t| t[index] }
+        end
+
+        def size
+          @tensors[0].size(0)
         end
       end
     end

data/lib/torch/version.rb

@@ -1,3 +1,3 @@
 module Torch
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: torch-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Andrew Kane
@@ -119,6 +119,8 @@ files:
 - lib/torch/nn/parameter.rb
 - lib/torch/nn/relu.rb
 - lib/torch/nn/sequential.rb
+- lib/torch/optim/optimizer.rb
+- lib/torch/optim/sgd.rb
 - lib/torch/tensor.rb
 - lib/torch/utils/data/data_loader.rb
 - lib/torch/utils/data/tensor_dataset.rb