torch-rb 0.1.2 → 0.1.3

data/lib/torch.rb

@@ -8,18 +8,40 @@ require "torch/version"
 
 # optim
 require "torch/optim/optimizer"
+require "torch/optim/adadelta"
+require "torch/optim/adagrad"
+require "torch/optim/adam"
+require "torch/optim/adamax"
+require "torch/optim/adamw"
+require "torch/optim/asgd"
+require "torch/optim/rmsprop"
+require "torch/optim/rprop"
 require "torch/optim/sgd"
 
-# nn
+# optim lr_scheduler
+require "torch/optim/lr_scheduler/lr_scheduler"
+require "torch/optim/lr_scheduler/step_lr"
+
+# nn base classes
 require "torch/nn/module"
-require "torch/nn/init"
+require "torch/nn/convnd"
+require "torch/nn/dropoutnd"
+
+# nn
+require "torch/nn/alpha_dropout"
 require "torch/nn/conv2d"
+require "torch/nn/dropout"
+require "torch/nn/dropout2d"
+require "torch/nn/dropout3d"
+require "torch/nn/embedding"
+require "torch/nn/feature_alpha_dropout"
 require "torch/nn/functional"
+require "torch/nn/init"
 require "torch/nn/linear"
+require "torch/nn/mse_loss"
 require "torch/nn/parameter"
-require "torch/nn/sequential"
 require "torch/nn/relu"
-require "torch/nn/mse_loss"
+require "torch/nn/sequential"
 
 # utils
 require "torch/utils/data/data_loader"
@@ -27,6 +49,11 @@ require "torch/utils/data/tensor_dataset"
 
 module Torch
   class Error < StandardError; end
+  class NotImplementedYet < StandardError
+    def message
+      "This feature has not been implemented yet. Consider submitting a PR."
+    end
+  end
 
   # keys: https://pytorch.org/docs/stable/tensor_attributes.html#torch.torch.dtype
   # values: https://github.com/pytorch/pytorch/blob/master/c10/core/ScalarType.h
@@ -75,11 +102,18 @@ module Torch
       obj.is_a?(Tensor)
     end
 
-    # TODO don't copy
     def from_numo(ndarray)
       dtype = _dtype_to_numo.find { |k, v| ndarray.is_a?(v) }
       raise Error, "Cannot convert #{ndarray.class.name} to tensor" unless dtype
-      tensor(ndarray.to_a, dtype: dtype[0])
+      options = tensor_options(device: "cpu", dtype: dtype[0])
+      # TODO pass pointer to array instead of creating string
+      str = ndarray.to_string
+      tensor = _from_blob(str, ndarray.shape, options)
+      # from_blob does not own the data, so we need to keep
+      # a reference to it for duration of tensor
+      # can remove when passing pointer directly
+      tensor.instance_variable_set("@_numo_str", str)
+      tensor
     end
 
     # private
@@ -197,7 +231,7 @@ module Torch
         high = low
         low = 0
       end
-      rand(input.size, like_options(input, options))
+      randint(low, high, input.size, like_options(input, options))
     end
 
     def randn_like(input, **options)
@@ -272,8 +306,13 @@ module Torch
       _min(input)
     end
 
-    def max(input)
-      _max(input)
+    def max(input, dim = nil, keepdim: false, out: nil)
+      if dim
+        raise NotImplementedYet unless out
+        _max_out(out[0], out[1], input, dim, keepdim)
+      else
+        _max(input)
+      end
     end
 
     def exp(input)
@@ -284,6 +323,18 @@ module Torch
       _log(input)
     end
 
+    def sign(input)
+      _sign(input)
+    end
+
+    def gt(input, other)
+      _gt(input, other)
+    end
+
+    def lt(input, other)
+      _lt(input, other)
+    end
+
     def unsqueeze(input, dim)
       _unsqueeze(input, dim)
     end
@@ -292,6 +343,10 @@ module Torch
       _dot(input, tensor)
     end
 
+    def cat(tensors, dim = 0)
+      _cat(tensors, dim)
+    end
+
     def matmul(input, other)
       _matmul(input, other)
     end
@@ -300,6 +355,22 @@ module Torch
       _reshape(input, shape)
     end
 
+    def flatten(input, start_dim: 0, end_dim: -1)
+      _flatten(input, start_dim, end_dim)
+    end
+
+    def sqrt(input)
+      _sqrt(input)
+    end
+
+    def abs(input)
+      _abs(input)
+    end
+
+    def device(str)
+      Device.new(str)
+    end
+
     private
 
     def execute_op(op, input, other, out: nil)

data/lib/torch/inspector.rb

@@ -1,13 +1,16 @@
 module Torch
   module Inspector
     # TODO make more performance, especially when summarizing
+    # how? only read data that will be displayed
     def inspect
       data =
         if numel == 0
           "[]"
         elsif dim == 0
-          to_a.first
+          item
         else
+          summarize = numel > 1000
+
           values = to_a.flatten
           abs = values.select { |v| v != 0 }.map(&:abs)
           max = abs.max || 1
@@ -36,8 +39,6 @@ module Torch
             fmt = "%#{total}d"
           end
 
-          summarize = numel > 1000
-
           inspect_level(to_a, fmt, dim - 1, 0, summarize)
         end
 

data/lib/torch/nn/alpha_dropout.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class AlphaDropout < DropoutNd
+      def forward(input)
+        F.alpha_dropout(input, p: @p, training: @training, inplace: @inplace)
+      end
+    end
+  end
+end

data/lib/torch/nn/conv2d.rb

@@ -1,36 +1,24 @@
 module Torch
   module NN
-    class Conv2d < Module
+    class Conv2d < ConvNd
       attr_reader :bias, :weight
 
-      def initialize(in_channels, out_channels, kernel_size, stride: 1, padding: 0) #, dilation: 1, groups: 1)
-        @in_channels = in_channels
-        @out_channels = out_channels
-        @kernel_size = pair(kernel_size)
-        @stride = pair(stride)
-        @padding = pair(padding)
-        # @dilation = pair(dilation)
-
-        # TODO divide by groups
-        @weight = Parameter.new(Tensor.new(out_channels, in_channels, *@kernel_size))
-        @bias = Parameter.new(Tensor.new(out_channels))
-
-        reset_parameters
+      def initialize(in_channels, out_channels, kernel_size, stride: 1, padding: 0, dilation: 1, groups: 1, bias: true, padding_mode: "zeros")
+        kernel_size = pair(kernel_size)
+        stride = pair(stride)
+        padding = pair(padding)
+        dilation = pair(dilation)
+        super(in_channels, out_channels, kernel_size, stride, padding, dilation, false, pair(0), groups, bias, padding_mode)
       end
 
-      def reset_parameters
-        Init.kaiming_uniform_(@weight, Math.sqrt(5))
-        if @bias
-          fan_in, _ = Init.calculate_fan_in_and_fan_out(@weight)
-          bound = 1 / Math.sqrt(fan_in)
-          Init.uniform_(@bias, -bound, bound)
+      def forward(input)
+        if @padding_mode == "circular"
+          raise NotImplementedError
         end
+        F.conv2d(input, @weight, @bias, stride: @stride, padding: @padding, dilation: @dilation, groups: @groups)
       end
 
-      def call(input)
-        F.conv2d(input, @weight, @bias, stride: @stride, padding: @padding) #, @dilation, @groups)
-      end
-
+      # TODO add more parameters
       def inspect
         "Conv2d(#{@in_channels}, #{@out_channels}, kernel_size: #{@kernel_size.inspect}, stride: #{@stride.inspect})"
       end

data/lib/torch/nn/convnd.rb

@@ -0,0 +1,41 @@
+module Torch
+  module NN
+    class ConvNd < Module
+      def initialize(in_channels, out_channels, kernel_size, stride, padding, dilation, transposed, output_padding, groups, bias, padding_mode)
+        super()
+        raise ArgumentError, "in_channels must be divisible by groups" if in_channels % groups != 0
+        raise ArgumentError, "out_channels must be divisible by groups" if out_channels % groups != 0
+        @in_channels = in_channels
+        @out_channels = out_channels
+        @kernel_size = kernel_size
+        @stride = stride
+        @padding = padding
+        @dilation = dilation
+        @transposed = transposed
+        @output_padding = output_padding
+        @groups = groups
+        @padding_mode = padding_mode
+        if transposed
+          @weight = Parameter.new(Tensor.new(in_channels, out_channels / groups, *kernel_size))
+        else
+          @weight = Parameter.new(Tensor.new(out_channels, in_channels / groups, *kernel_size))
+        end
+        if bias
+          @bias = Parameter.new(Tensor.new(out_channels))
+        else
+          raise NotImplementedError
+        end
+        reset_parameters
+      end
+
+      def reset_parameters
+        Init.kaiming_uniform!(@weight, Math.sqrt(5))
+        if @bias
+          fan_in, _ = Init.calculate_fan_in_and_fan_out(@weight)
+          bound = 1 / Math.sqrt(fan_in)
+          Init.uniform!(@bias, -bound, bound)
+        end
+      end
+    end
+  end
+end

data/lib/torch/nn/dropout.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class Dropout < DropoutNd
+      def forward(input)
+        F.dropout(input, p: @p, training: @training, inplace: @inplace)
+      end
+    end
+  end
+end

data/lib/torch/nn/dropout2d.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class Dropout2d < DropoutNd
+      def forward(input)
+        F.dropout2d(input, p: @p, training: @training, inplace: @inplace)
+      end
+    end
+  end
+end

data/lib/torch/nn/dropout3d.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class Dropout3d < DropoutNd
+      def forward(input)
+        F.dropout3d(input, p: @p, training: @training, inplace: @inplace)
+      end
+    end
+  end
+end

data/lib/torch/nn/dropoutnd.rb

@@ -0,0 +1,15 @@
+module Torch
+  module NN
+    class DropoutNd < Module
+      def initialize(p: 0.5, inplace: false)
+        super()
+        @p = p
+        @inplace = inplace
+      end
+
+      def inspect
+        "#{self.class.name.split("::").last}(p: #{@p.inspect}, inplace: #{@inplace.inspect})"
+      end
+    end
+  end
+end

data/lib/torch/nn/embedding.rb

@@ -0,0 +1,52 @@
+# ported from https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/sparse.py
+module Torch
+  module NN
+    class Embedding < Module
+      def initialize(num_embeddings, embedding_dim, padding_idx: nil, max_norm: nil,
+        norm_type: 2.0, scale_grad_by_freq: false, sparse: false, _weight: nil)
+
+        super()
+        @num_embeddings = num_embeddings
+        @embedding_dim = embedding_dim
+
+        if padding_idx
+          if padding_idx > 0
+            raise ArgumentError, "Padding_idx must be within num_embeddings" unless padding_idx < @num_embeddings
+          elsif padding_idx < 0
+            raise ArgumentError, "Padding_idx must be within num_embeddings" unless padding_idx >= -@num_embeddings
+            padding_idx = @num_embeddings + padding_idx
+          end
+        end
+        @padding_idx = padding_idx
+        @max_norm = max_norm
+        @norm_type = norm_type
+        @scale_grad_by_freq = scale_grad_by_freq
+        if _weight.nil?
+          @weight = Parameter.new(Tensor.new(num_embeddings, embedding_dim))
+          reset_parameters
+        else
+          raise ArgumentError, "Shape of weight does not match num_embeddings and embedding_dim" unless _weight.shape == [num_embeddings, embedding_dim]
+          @weight = Parameter.new(_weight)
+        end
+        @sparse = sparse
+      end
+
+      def reset_parameters
+        Init.normal!(@weight)
+        if @padding_idx
+          Torch.no_grad do
+            @weight[@padding_idx].fill!(0)
+          end
+        end
+      end
+
+      def forward(input)
+        F.embedding(input, @weight, padding_idx: @padding_idx, max_norm: @max_norm, norm_type: @norm_type, scale_grad_by_freq: @scale_grad_by_freq, sparse: @sparse)
+      end
+
+      def inspect
+        "Embedding(#{@num_embeddings}, #{@embedding_dim})"
+      end
+    end
+  end
+end

data/lib/torch/nn/feature_alpha_dropout.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class FeatureAlphaDropout < DropoutNd
+      def forward(input)
+        F.feature_alpha_dropout(input, p: @p, training: @training, inplace: @inplace)
+      end
+    end
+  end
+end

data/lib/torch/nn/functional.rb

@@ -6,17 +6,9 @@ module Torch
           Torch.relu(input)
         end
 
-        def conv2d(input, weight, bias, stride: 1, padding: 0)
+        def conv2d(input, weight, bias, stride: 1, padding: 0, dilation: 1, groups: 1)
           # TODO pair stride and padding when needed
-          Torch.conv2d(input, weight, bias, stride, padding)
-        end
-
-        def prelu(input, weight)
-          Torch.prelu(input, weight)
-        end
-
-        def leaky_relu(input, negative_slope = 0.01)
-          Torch.leaky_relu(input, negative_slope)
+          Torch.conv2d(input, weight, bias, stride, padding, dilation, groups)
         end
 
         def max_pool2d(input, kernel_size)
@@ -41,14 +33,64 @@ module Torch
           nll_loss(log_softmax(input, 1), target)
         end
 
-        def nll_loss(input, target)
+        def nll_loss(input, target, reduction: "mean")
           # TODO fix for non-1d
-          Torch.nll_loss(input, target)
+          Torch.nll_loss(input, target, reduction)
         end
 
         def log_softmax(input, dim)
           input.log_softmax(dim)
         end
+
+        def dropout(input, p: 0.5, training: true, inplace: false)
+          if inplace
+            Torch._dropout!(input, p, training)
+          else
+            Torch._dropout(input, p, training)
+          end
+        end
+
+        def dropout2d(input, p: 0.5, training: true, inplace: false)
+          raise ArgumentError, "dropout probability has to be between 0 and 1, but got #{p}" if p < 0 || p > 1
+
+          if inplace
+            Torch._feature_dropout!(input, p, training)
+          else
+            Torch._feature_dropout(input, p, training)
+          end
+        end
+
+        def dropout3d(input, p: 0.5, training: true, inplace: false)
+          if inplace
+            Torch._feature_dropout!(input, p, training)
+          else
+            Torch._feature_dropout(input, p, training)
+          end
+        end
+
+        def alpha_dropout(input, p: 0.5, training: true, inplace: false)
+          if inplace
+            Torch._alpha_dropout!(input, p, training)
+          else
+            Torch._alpha_dropout(input, p, training)
+          end
+        end
+
+        def feature_alpha_dropout(input, p: 0.5, training: true, inplace: false)
+          if inplace
+            Torch._feature_alpha_dropout!(input, p, training)
+          else
+            Torch._feature_alpha_dropout(input, p, training)
+          end
+        end
+
+        def embedding(input, weight, padding_idx: nil, max_norm: nil, norm_type: 2.0, scale_grad_by_freq: false, sparse: false)
+          # TODO handle max_norm and norm_type
+          raise NotImplementedYet unless max_norm.nil? && norm_type == 2.0
+
+          padding_idx ||= -1
+          Torch._embedding(input, weight, padding_idx, scale_grad_by_freq, sparse)
+        end
       end
     end