torch-rb 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30089078de4039df111087e5c27e0cb10d6f36398c0e8d5cc774e9b642a8e133
-  data.tar.gz: 89eb9e183b395dd67cd9cf228749cf26402993bb561de973f1ba7438bc372b04
+  metadata.gz: e7f715179c9a84dc7399b80d93fd61f2bbb58a0156e6084dc4abb23e1d4a1b52
+  data.tar.gz: 6928379ae7c92a77ad9dde4f4224ec33c6f8575a9b77585c0147e4f5361021de
 SHA512:
-  metadata.gz: 027a069b00ac1329c007ddaf471a21b57a82a823ad974a937f832d17720b8e26474c64c79e9a29ec71bac433abb3d74d6a7cf407f0a983bb3c0cafb5b5c7532f
-  data.tar.gz: 6d7ef10b53db0df39eda13d07aa9b52b4afac0965674919b5cc517e7b53f59a9010cb647e50d62bc06154f7d8f3ef632d5897e4f7774372d7ab1b44b2cb6ca82
+  metadata.gz: 9911a9e86d93f1e410776c44fdb3cd9aa06c83d1f0e42fdab8530970bea6520aed7906e96fb8243efd6b957453ebc13678b2b92e4c85b54407030a32c6196e08
+  data.tar.gz: 0d080f5458a5dcf8fee19ce5e2e342bf6269432de6e78d923036232963ebb80daeea993c0bbf4af2d6da46593ac28a72a8232020a9fcb48acc3276c9e1ebebf3

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.1.3 (2019-11-30)
+
+- Changed to BSD 3-Clause license to match PyTorch
+- Added many optimizers
+- Added `StepLR` learning rate scheduler
+- Added dropout
+- Added embedding
+- Added support for `bool` type
+- Improved performance of `from_numo`
+
 ## 0.1.2 (2019-11-27)
 
 - Added SGD optimizer

data/LICENSE.txt

@@ -1,22 +1,46 @@
-Copyright (c) 2019 Andrew Kane
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+BSD 3-Clause License
+
+From Torch-rb:
+
+Copyright (c) 2019-     Andrew Kane
+
+From PyTorch (for ported code):
+
+Copyright (c) 2016-     Facebook, Inc            (Adam Paszke)
+Copyright (c) 2014-     Facebook, Inc            (Soumith Chintala)
+Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
+Copyright (c) 2012-2014 Deepmind Technologies    (Koray Kavukcuoglu)
+Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
+Copyright (c) 2011-2013 NYU                      (Clement Farabet)
+Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston)
+Copyright (c) 2006      Idiap Research Institute (Samy Bengio)
+Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz)
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America
+   and IDIAP Research Institute nor the names of its contributors may be
+   used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -30,7 +30,9 @@ This library follows the [PyTorch API](https://pytorch.org/docs/stable/torch.htm
 
 Many methods and options are missing at the moment. PRs welcome!
 
-Some examples below are from [Deep Learning with PyTorch: A 60 Minutes Blitz](https://pytorch.org/tutorials/beginner/deep_learning_60min_blitz.html).
+## Tutorial
+
+Some examples below are from [Deep Learning with PyTorch: A 60 Minutes Blitz](https://pytorch.org/tutorials/beginner/deep_learning_60min_blitz.html)
 
 ### Tensors
 
@@ -145,7 +147,7 @@ Convert a Numo array to a tensor
 
 ```ruby
 b = Numo::NArray.cast([1, 2, 3])
-Torch.from_numpy(b)
+Torch.from_numo(b)
 ```
 
 ### Autograd
@@ -180,10 +182,10 @@ Stop autograd from tracking history
 
 ```ruby
 x.requires_grad # true
-(x ** 2).requires_grad # true
+(x**2).requires_grad # true
 
 Torch.no_grad do
-  (x ** 2).requires_grad # false
+  (x**2).requires_grad # false
 end
 ```
 
@@ -359,6 +361,13 @@ Here’s a list of functions to create tensors (descriptions from the [C++ docs]
   Torch.zeros(3) # tensor([0, 0, 0])
   ```
 
+## Examples
+
+Here are a few full examples:
+
+- [Image classification with MNIST](examples/mnist)
+- [Collaborative filtering with MovieLens](examples/movielens)
+
 ## LibTorch Installation
 
 [Download LibTorch](https://pytorch.org/). For Linux, use the `cxx11 ABI` version. Then run:
@@ -405,7 +414,7 @@ To get started with development:
 git clone https://github.com/ankane/torch-rb.git
 cd torch-rb
 bundle install
-bundle exec rake compile
+bundle exec rake compile -- --with-torch-dir=/path/to/libtorch
 bundle exec rake test
 ```
 

data/ext/torch/ext.cpp

@@ -88,7 +88,49 @@ IntArrayRef from_ruby<IntArrayRef>(Object x)
 }
 
 // for now
-typedef float Scalar;
+class Scalar {
+  torch::Scalar value;
+  public:
+    Scalar(Object o) {
+      // TODO cast based on Ruby type
+      if (o.rb_type() == T_FIXNUM) {
+        value = torch::Scalar(from_ruby<int64_t>(o));
+      } else {
+        value = torch::Scalar(from_ruby<float>(o));
+      }
+    }
+    operator torch::Scalar() {
+      return value;
+    }
+};
+
+template<>
+inline
+Scalar from_ruby<Scalar>(Object x)
+{
+  return Scalar(x);
+}
+
+class TensorList {
+  std::vector<torch::Tensor> vec;
+  public:
+    TensorList(Object o) {
+      Array a = Array(o);
+      for (size_t i = 0; i < a.size(); i++) {
+        vec.push_back(from_ruby<torch::Tensor>(a[i]));
+      }
+    }
+    operator torch::TensorList() {
+      return torch::TensorList(vec);
+    }
+};
+
+template<>
+inline
+TensorList from_ruby<TensorList>(Object x)
+{
+  return TensorList(x);
+}
 
 extern "C"
 void Init_ext()
@@ -206,6 +248,11 @@ void Init_ext()
       *[](torch::Tensor& input, int64_t dim, bool keepdim) {
         return torch::argmax(input, dim, keepdim);
       })
+    .define_singleton_method(
+      "_cat",
+      *[](TensorList tensors, int64_t dim) {
+        return torch::cat(tensors, dim);
+      })
     .define_singleton_method(
       "_norm",
       *[](torch::Tensor& input) {
@@ -221,6 +268,17 @@ void Init_ext()
       *[](torch::Tensor& input) {
         return torch::max(input);
       })
+    .define_singleton_method(
+      "_max_out",
+      *[](torch::Tensor &max, torch::Tensor &max_indices, const torch::Tensor &input, int64_t dim, bool keepdim) {
+        // TODO add return value
+        torch::_max_out(max, max_indices, input, dim, keepdim);
+      })
+    .define_singleton_method(
+      "_sqrt",
+      *[](torch::Tensor& input) {
+        return torch::sqrt(input);
+      })
     .define_singleton_method(
       "_exp",
       *[](torch::Tensor& input) {
@@ -231,6 +289,11 @@ void Init_ext()
       *[](torch::Tensor& input) {
         return torch::log(input);
       })
+    .define_singleton_method(
+      "_sign",
+      *[](torch::Tensor& input) {
+        return torch::sign(input);
+      })
     .define_singleton_method(
       "_unsqueeze",
       *[](torch::Tensor& input, int64_t dim) {
@@ -251,6 +314,18 @@ void Init_ext()
       *[](torch::Tensor& input, torch::Tensor& other) {
         return torch::eq(input, other);
       })
+    .define_singleton_method(
+      "_gt",
+      // TODO support tensors
+      *[](torch::Tensor& input, Scalar other) {
+        return torch::gt(input, other);
+      })
+    .define_singleton_method(
+      "_lt",
+      // TODO support tensors
+      *[](torch::Tensor& input, Scalar other) {
+        return torch::lt(input, other);
+      })
     .define_singleton_method(
       "_add",
       *[](torch::Tensor& input, torch::Tensor& other) {
@@ -258,7 +333,7 @@ void Init_ext()
       })
     .define_singleton_method(
       "_add_scalar",
-      *[](torch::Tensor& input, float other) {
+      *[](torch::Tensor& input, Scalar other) {
         return torch::add(input, other);
       })
     .define_singleton_method(
@@ -273,7 +348,7 @@ void Init_ext()
       })
     .define_singleton_method(
       "_sub_scalar",
-      *[](torch::Tensor& input, float other) {
+      *[](torch::Tensor& input, Scalar other) {
         return torch::sub(input, other);
       })
     .define_singleton_method(
@@ -283,7 +358,7 @@ void Init_ext()
       })
     .define_singleton_method(
       "_mul_scalar",
-      *[](torch::Tensor& input, float other) {
+      *[](torch::Tensor& input, Scalar other) {
         return torch::mul(input, other);
       })
     .define_singleton_method(
@@ -293,7 +368,7 @@ void Init_ext()
       })
     .define_singleton_method(
       "_div_scalar",
-      *[](torch::Tensor& input, float other) {
+      *[](torch::Tensor& input, Scalar other) {
         return torch::div(input, other);
       })
     .define_singleton_method(
@@ -303,7 +378,7 @@ void Init_ext()
       })
     .define_singleton_method(
       "_remainder_scalar",
-      *[](torch::Tensor& input, float other) {
+      *[](torch::Tensor& input, Scalar other) {
         return torch::remainder(input, other);
       })
     .define_singleton_method(
@@ -311,6 +386,11 @@ void Init_ext()
       *[](torch::Tensor& input, Scalar exponent) {
         return torch::pow(input, exponent);
       })
+    .define_singleton_method(
+      "_abs",
+      *[](torch::Tensor& input) {
+        return torch::abs(input);
+      })
     .define_singleton_method(
       "_neg",
       *[](torch::Tensor& input) {
@@ -321,25 +401,20 @@ void Init_ext()
       *[](torch::Tensor& input, IntArrayRef shape) {
         return torch::reshape(input, shape);
       })
+    .define_singleton_method(
+      "_flatten",
+      *[](torch::Tensor& input, int64_t start_dim, int64_t end_dim) {
+        return torch::flatten(input, start_dim, end_dim);
+      })
     .define_singleton_method(
       "relu",
       *[](torch::Tensor& input) {
         return torch::relu(input);
       })
-    .define_singleton_method(
-      "prelu",
-      *[](torch::Tensor& input, torch::Tensor& weight) {
-        return torch::prelu(input, weight);
-      })
-    .define_singleton_method(
-      "leaky_relu",
-      *[](torch::Tensor& input, Scalar negative_slope = 0.01) {
-        return torch::leaky_relu(input, negative_slope);
-      })
     .define_singleton_method(
       "conv2d",
-      *[](torch::Tensor& input, torch::Tensor& weight, torch::Tensor& bias, IntArrayRef stride, IntArrayRef padding) {
-        return torch::conv2d(input, weight, bias, stride, padding);
+      *[](torch::Tensor& input, torch::Tensor& weight, torch::Tensor& bias, IntArrayRef stride, IntArrayRef padding, IntArrayRef dilation, int64_t groups) {
+        return torch::conv2d(input, weight, bias, stride, padding, dilation, groups);
       })
     .define_singleton_method(
       "linear",
@@ -356,6 +431,52 @@ void Init_ext()
       *[](torch::Tensor& input, IntArrayRef kernel_size) {
         return torch::avg_pool2d(input, kernel_size);
       })
+    .define_singleton_method(
+      "_dropout",
+      *[](torch::Tensor& input, float p, bool train) {
+        return torch::dropout(input, p, train);
+      })
+    .define_singleton_method(
+      "_dropout!",
+      *[](torch::Tensor& input, float p, bool train) {
+        return torch::dropout_(input, p, train);
+      })
+    .define_singleton_method(
+      "_feature_dropout",
+      *[](torch::Tensor& input, float p, bool train) {
+        return torch::feature_dropout(input, p, train);
+      })
+    .define_singleton_method(
+      "_feature_dropout!",
+      *[](torch::Tensor& input, float p, bool train) {
+        return torch::feature_dropout_(input, p, train);
+      })
+    .define_singleton_method(
+      "_alpha_dropout",
+      *[](torch::Tensor& input, float p, bool train) {
+        return torch::alpha_dropout(input, p, train);
+      })
+    .define_singleton_method(
+      "_alpha_dropout!",
+      *[](torch::Tensor& input, float p, bool train) {
+        return torch::alpha_dropout_(input, p, train);
+      })
+    .define_singleton_method(
+      "_feature_alpha_dropout",
+      *[](torch::Tensor& input, float p, bool train) {
+        return torch::feature_alpha_dropout(input, p, train);
+      })
+    .define_singleton_method(
+      "_feature_alpha_dropout!",
+      *[](torch::Tensor& input, float p, bool train) {
+        return torch::feature_alpha_dropout_(input, p, train);
+      })
+    .define_singleton_method(
+      "_embedding",
+      // weight and indices are swapped from Python interface
+      *[](const torch::Tensor &indices, const torch::Tensor &weight, int64_t padding_idx, bool scale_grad_by_freq, bool sparse) {
+        return torch::embedding(weight, indices, padding_idx, scale_grad_by_freq, sparse);
+      })
     .define_singleton_method(
       "mse_loss",
       *[](torch::Tensor& input, torch::Tensor& target, std::string reduction) {
@@ -364,8 +485,16 @@ void Init_ext()
       })
     .define_singleton_method(
       "nll_loss",
-      *[](torch::Tensor& input, torch::Tensor& target) {
-        return torch::nll_loss(input, target);
+      *[](torch::Tensor& input, torch::Tensor& target, std::string reduction) {
+        auto red = reduction == "mean" ? Reduction::Mean : Reduction::Sum;
+        return torch::nll_loss(input, target, {}, red);
+      })
+    .define_singleton_method("numel", &torch::numel)
+    .define_singleton_method(
+      "_from_blob",
+      *[](String s, IntArrayRef size, const torch::TensorOptions &options) {
+        void *data = const_cast<char *>(s.c_str());
+        return torch::from_blob(data, size, options);
       })
     .define_singleton_method(
       "_tensor",
@@ -387,9 +516,19 @@ void Init_ext()
     .define_method("sparse?", &torch::Tensor::is_sparse)
     .define_method("quantized?", &torch::Tensor::is_quantized)
     .define_method("dim", &torch::Tensor::dim)
-    .define_method("numel", &torch::Tensor::numel)
     .define_method("element_size", &torch::Tensor::element_size)
     .define_method("requires_grad", &torch::Tensor::requires_grad)
+    .define_method("view_as", &torch::Tensor::view_as)
+    .define_method(
+      "addcmul!",
+      *[](torch::Tensor& self, Scalar value, const torch::Tensor & tensor1, const torch::Tensor & tensor2) {
+        return self.addcmul_(tensor1, tensor2, value);
+      })
+    .define_method(
+      "addcdiv!",
+      *[](torch::Tensor& self, Scalar value, const torch::Tensor & tensor1, const torch::Tensor & tensor2) {
+        return self.addcdiv_(tensor1, tensor2, value);
+      })
     .define_method(
       "zero!",
       *[](torch::Tensor& self) {
@@ -460,24 +599,74 @@ void Init_ext()
         return self.view(size);
       })
     .define_method(
-      "add!",
+      "resize_as!",
       *[](torch::Tensor& self, torch::Tensor& other) {
-        self.add_(other);
+        return self.resize_as_(other);
+      })
+    .define_method(
+      "fill!",
+      *[](torch::Tensor& self, Scalar value) {
+        return self.fill_(value);
+      })
+    .define_method(
+      "_add!",
+      *[](torch::Tensor& self, torch::Tensor& other) {
+        return self.add_(other);
+      })
+    .define_method(
+      "_add_alpha!",
+      *[](torch::Tensor& self, torch::Tensor& other, Scalar alpha) {
+        return self.add_(other, alpha);
+      })
+    .define_method(
+      "_add_scalar!",
+      *[](torch::Tensor& self, Scalar other) {
+        return self.add_(other);
+      })
+    .define_method(
+      "normal!",
+      *[](torch::Tensor& self, double mean, double std) {
+        return self.normal_(mean, std);
       })
     .define_method(
       "sub!",
       *[](torch::Tensor& self, torch::Tensor& other) {
-        self.sub_(other);
+        return self.sub_(other);
       })
     .define_method(
-      "mul!",
+      "_mul!",
       *[](torch::Tensor& self, torch::Tensor& other) {
-        self.mul_(other);
+        return self.mul_(other);
+      })
+    .define_method(
+      "_mul_scalar!",
+      *[](torch::Tensor& self, Scalar other) {
+        return self.mul_(other);
       })
     .define_method(
       "div!",
       *[](torch::Tensor& self, torch::Tensor& other) {
-        self.div_(other);
+        return self.div_(other);
+      })
+    .define_method(
+      "sqrt!",
+      *[](torch::Tensor& self) {
+        return self.sqrt_();
+      })
+    .define_method(
+      "unsqueeze!",
+      *[](torch::Tensor& self, int64_t dim) {
+        return self.unsqueeze_(dim);
+      })
+    .define_method(
+      "copy!",
+      *[](torch::Tensor& self, torch::Tensor& src) {
+        return self.copy_(src);
+      })
+    .define_method(
+      "clone",
+      *[](torch::Tensor& self) {
+        return self.clone();
       })
     .define_method(
       "log_softmax",
@@ -532,8 +721,10 @@ void Init_ext()
             a.push(data[i]);
           }
         } else if (dtype == torch::kBool) {
-          // bool
-          throw std::runtime_error("Type not supported yet");
+          bool* data = self.data_ptr<bool>();
+          for (int i = 0; i < self.numel(); i++) {
+            a.push(data[i] ? True : False);
+          }
         } else {
           throw std::runtime_error("Unsupported type");
         }
@@ -544,6 +735,11 @@ void Init_ext()
       *[](torch::Tensor& self, int i) {
         return self.size(i);
       })
+    .define_method(
+      "_to",
+      *[](torch::Tensor& self, torch::Device device, int dtype, bool non_blocking, bool copy) {
+        return self.to(device, (torch::ScalarType) dtype, non_blocking, copy);
+      })
     .define_singleton_method(
       "_make_subclass",
       *[](torch::Tensor& rd, bool requires_grad) {
@@ -597,12 +793,17 @@ void Init_ext()
 
   Module rb_mInit = define_module_under(rb_mNN, "Init")
     .define_singleton_method(
-      "kaiming_uniform_",
+      "kaiming_uniform!",
       *[](torch::Tensor& input, double a) {
         return torch::nn::init::kaiming_uniform_(input, a);
       })
     .define_singleton_method(
-      "uniform_",
+      "normal!",
+      *[](torch::Tensor& input) {
+        return torch::nn::init::normal_(input);
+      })
+    .define_singleton_method(
+      "uniform!",
       *[](torch::Tensor& input, double to, double from) {
         return torch::nn::init::uniform_(input, to, from);
       });
@@ -619,4 +820,20 @@ void Init_ext()
       *[](torch::autograd::Variable& self) {
         return self.grad().defined();
       });
+
+  Class rb_cDevice = define_class_under<torch::Device>(rb_mTorch, "Device")
+    .define_constructor(Constructor<torch::Device, std::string>())
+    .define_method("index", &torch::Device::index)
+    .define_method("index?", &torch::Device::has_index)
+    .define_method(
+      "type",
+      *[](torch::Device& self) {
+        std::stringstream s;
+        s << self.type();
+        return s.str();
+      });
+
+  Module rb_mCUDA = define_module_under(rb_mTorch, "CUDA")
+    .define_singleton_method("available?", &torch::cuda::is_available)
+    .define_singleton_method("device_count", &torch::cuda::device_count);
 }