torch-rb 0.1.2 → 0.1.7

data/lib/torch/nn/group_norm.rb

@@ -0,0 +1,36 @@
+module Torch
+  module NN
+    class GroupNorm < Module
+      def initialize(num_groups, num_channels, eps: 1e-5, affine: true)
+        super()
+        @num_groups = num_groups
+        @num_channels = num_channels
+        @eps = eps
+        @affine = affine
+        if @affine
+          @weight = Parameter.new(Torch::Tensor.new(num_channels))
+          @bias = Parameter.new(Torch::Tensor.new(num_channels))
+        else
+          register_parameter("weight", nil)
+          register_parameter("bias", nil)
+        end
+        reset_parameters
+      end
+
+      def reset_parameters
+        if @affine
+          Init.ones!(@weight)
+          Init.zeros!(@bias)
+        end
+      end
+
+      def forward(input)
+        F.group_norm(input, @num_groups, weight: @weight, bias: @bias, eps: @eps)
+      end
+
+      def extra_inspect
+        format("%{num_groups}, %{num_channels}, eps: %{eps}, affine: %{affine}", **dict)
+      end
+    end
+  end
+end

data/lib/torch/nn/gru.rb

@@ -0,0 +1,49 @@
+module Torch
+  module NN
+    class GRU < RNNBase
+      def initialize(*args, **options)
+        super("GRU", *args, **options)
+      end
+
+      def run_impl(input, hx, batch_sizes)
+        if batch_sizes.nil?
+          Torch.gru(input, hx, _get_flat_weights, @bias, @num_layers,
+                             @dropout, @training, @bidirectional, @batch_first)
+        else
+          Torch.gru(input, batch_sizes, hx, _get_flat_weights, @bias,
+                             @num_layers, @dropout, @training, @bidirectional)
+        end
+      end
+
+      def forward_impl(input, hx, batch_sizes, max_batch_size, sorted_indices)
+        if hx.nil?
+          num_directions = @bidirectional ? 2 : 1
+          hx = Torch.zeros(@num_layers * num_directions, max_batch_size, @hidden_size, dtype: input.dtype, device: input.device)
+        else
+          # Each batch of the hidden state should match the input sequence that
+          # the user believes he/she is passing in.
+          hx = permute_hidden(hx, sorted_indices)
+        end
+
+        check_forward_args(input, hx, batch_sizes)
+        result = run_impl(input, hx, batch_sizes)
+        output = result[0]
+        hidden = result[1]
+        [output, hidden]
+      end
+
+      def forward_tensor(input, hx: nil)
+        batch_sizes = nil
+        max_batch_size = @batch_first ? input.size(0) : input.size(1)
+        sorted_indices = nil
+        unsorted_indices = nil
+        output, hidden = forward_impl(input, hx, batch_sizes, max_batch_size, sorted_indices)
+        [output, permute_hidden(hidden, unsorted_indices)]
+      end
+
+      def forward(input, hx: nil)
+        forward_tensor(input, hx: hx)
+      end
+    end
+  end
+end

data/lib/torch/nn/hardshrink.rb

@@ -0,0 +1,18 @@
+module Torch
+  module NN
+    class Hardshrink < Module
+      def initialize(lambd: 0.5)
+        super()
+        @lambd = lambd
+      end
+
+      def forward(input)
+        F.hardshrink(input, @lambd)
+      end
+
+      def extra_inspect
+        @lambd.to_s
+      end
+    end
+  end
+end

data/lib/torch/nn/hinge_embedding_loss.rb

@@ -0,0 +1,14 @@
+module Torch
+  module NN
+    class HingeEmbeddingLoss < Loss
+      def initialize(margin: 1.0, reduction: "mean")
+        super(reduction)
+        @margin = margin
+      end
+
+      def forward(input, target)
+        F.hinge_embedding_loss(input, target, margin: @margin, reduction: @reduction)
+      end
+    end
+  end
+end

data/lib/torch/nn/identity.rb

@@ -0,0 +1,14 @@
+module Torch
+  module NN
+    class Identity < Module
+      # written this way to support unused arguments
+      def initialize(*args, **options)
+        super()
+      end
+
+      def forward(input)
+        input
+      end
+    end
+  end
+end

data/lib/torch/nn/init.rb

@@ -2,7 +2,64 @@ module Torch
   module NN
     module Init
       class << self
-        def calculate_fan_in_and_fan_out(tensor)
+        def calculate_gain(nonlinearity, param: 0.01)
+          _calculate_gain(nonlinearity, param)
+        end
+
+        def uniform!(tensor, a: 0.0, b: 1.0)
+          _uniform!(tensor, a, b)
+        end
+
+        def normal!(tensor, mean: 0.0, std: 1.0)
+          _normal!(tensor, mean, std)
+        end
+
+        def constant!(tensor, val)
+          _constant!(tensor, val)
+        end
+
+        def ones!(tensor)
+          _ones!(tensor)
+        end
+
+        def zeros!(tensor)
+          _zeros!(tensor)
+        end
+
+        def eye!(tensor)
+          _eye!(tensor)
+        end
+
+        def dirac!(tensor)
+          _dirac!(tensor)
+        end
+
+        def xavier_uniform!(tensor, gain: 1.0)
+          _xavier_uniform!(tensor, gain)
+        end
+
+        def xavier_normal!(tensor, gain: 1.0)
+          _xavier_normal!(tensor, gain)
+        end
+
+        def kaiming_uniform!(tensor, a: 0, mode: "fan_in", nonlinearity: "leaky_relu")
+          _kaiming_uniform!(tensor, a, mode, nonlinearity)
+        end
+
+        def kaiming_normal!(tensor, a: 0, mode: "fan_in", nonlinearity: "leaky_relu")
+          _kaiming_normal!(tensor, a, mode, nonlinearity)
+        end
+
+        def orthogonal!(tensor, gain: 1)
+          _orthogonal!(tensor, gain)
+        end
+
+        def sparse!(tensor, sparsity, std: 0.01)
+          _sparse!(tensor, sparsity, std)
+        end
+
+        # TODO move to C++ when released
+        def _calculate_fan_in_and_fan_out(tensor)
           dimensions = tensor.dim
           if dimensions < 2
             raise Error, "Fan in and fan out can not be computed for tensor with fewer than 2 dimensions"

data/lib/torch/nn/instance_norm.rb

@@ -0,0 +1,20 @@
+module Torch
+  module NN
+    class InstanceNorm < BatchNorm
+      def initialize(num_features, eps: 1e-5, momentum: 0.1, affine: false, track_running_stats: false)
+        super(num_features, eps: eps, momentum: momentum, affine: affine, track_running_stats: track_running_stats)
+      end
+
+      def forward(input)
+        _check_input_dim(input)
+
+        F.instance_norm(
+          input, running_mean: @running_mean, running_var: @running_var,
+          weight: @weight, bias: @bias,
+          use_input_stats: @training || !@track_running_stats,
+          momentum: @momentum, eps: @eps
+        )
+      end
+    end
+  end
+end

data/lib/torch/nn/instance_norm1d.rb

@@ -0,0 +1,18 @@
+module Torch
+  module NN
+    class InstanceNorm1d < InstanceNorm
+      def _check_input_dim(input)
+        if input.dim == 2
+          raise ArgumentError,
+            "InstanceNorm1d returns 0-filled tensor to 2D tensor." +
+            "This is because InstanceNorm1d reshapes inputs to" +
+            "(1, N * C, ...) from (N, C,...) and this makes" +
+            "variances 0."
+        end
+        if input.dim != 3
+          raise "expected 3D input (got #{input.dim}D input)"
+        end
+      end
+    end
+  end
+end

data/lib/torch/nn/instance_norm2d.rb

@@ -0,0 +1,11 @@
+module Torch
+  module NN
+    class InstanceNorm2d < InstanceNorm
+      def _check_input_dim(input)
+        if input.dim != 4
+          raise ArgumentError, "expected 4D input (got #{input.dim}D input)"
+        end
+      end
+    end
+  end
+end

data/lib/torch/nn/instance_norm3d.rb

@@ -0,0 +1,11 @@
+module Torch
+  module NN
+    class InstanceNorm3d < InstanceNorm
+      def _check_input_dim(input)
+        if input.dim != 5
+          raise ArgumentError, "expected 5D input (got #{input.dim}D input)"
+        end
+      end
+    end
+  end
+end

data/lib/torch/nn/kl_div_loss.rb

@@ -0,0 +1,13 @@
+module Torch
+  module NN
+    class KLDivLoss < Loss
+      def initialize(reduction: "mean")
+        super(reduction)
+      end
+
+      def forward(input, target)
+        F.kl_div(input, target, reduction: @reduction)
+      end
+    end
+  end
+end

data/lib/torch/nn/l1_loss.rb

@@ -0,0 +1,13 @@
+module Torch
+  module NN
+    class L1Loss < Loss
+      def initialize(reduction: "mean")
+        super(reduction)
+      end
+
+      def forward(input, target)
+        F.l1_loss(input, target, reduction: @reduction)
+      end
+    end
+  end
+end

data/lib/torch/nn/layer_norm.rb

@@ -0,0 +1,35 @@
+module Torch
+  module NN
+    class LayerNorm < Module
+      def initialize(normalized_shape, eps: 1e-5, elementwise_affine: true)
+        super()
+        @normalized_shape = Array(normalized_shape)
+        @eps = eps
+        @elementwise_affine = elementwise_affine
+        if @elementwise_affine
+          @weight = Parameter.new(Torch::Tensor.new(*normalized_shape))
+          @bias = Parameter.new(Torch::Tensor.new(*normalized_shape))
+        else
+          register_parameter("weight", nil)
+          register_parameter("bias", nil)
+        end
+        reset_parameters
+      end
+
+      def reset_parameters
+        if @elementwise_affine
+          Init.ones!(@weight)
+          Init.zeros!(@bias)
+        end
+      end
+
+      def forward(input)
+        F.layer_norm(input, @normalized_shape, weight: @weight, bias: @bias, eps: @eps)
+      end
+
+      def extra_inspect
+        format("%{normalized_shape}, eps: %{eps}, elementwise_affine: %{elementwise_affine}", **dict)
+      end
+    end
+  end
+end

data/lib/torch/nn/leaky_relu.rb

@@ -0,0 +1,20 @@
+module Torch
+  module NN
+    class LeakyReLU < Module
+      def initialize(negative_slope: 1e-2) #, inplace: false)
+        super()
+        @negative_slope = negative_slope
+        # @inplace = inplace
+      end
+
+      def forward(input)
+        F.leaky_relu(input, @negative_slope) #, inplace: @inplace)
+      end
+
+      def extra_inspect
+        inplace_str = @inplace ? ", inplace: true" : ""
+        format("negative_slope: %s%s", @negative_slope, inplace_str)
+      end
+    end
+  end
+end

data/lib/torch/nn/linear.rb

@@ -1,35 +1,36 @@
 module Torch
   module NN
     class Linear < Module
-      attr_reader :bias, :weight
-
       def initialize(in_features, out_features, bias: true)
+        super()
         @in_features = in_features
         @out_features = out_features
 
         @weight = Parameter.new(Tensor.new(out_features, in_features))
         if bias
           @bias = Parameter.new(Tensor.new(out_features))
+        else
+          register_parameter("bias", nil)
         end
 
         reset_parameters
       end
 
-      def call(input)
-        F.linear(input, @weight, @bias)
-      end
-
       def reset_parameters
-        Init.kaiming_uniform_(@weight, Math.sqrt(5))
+        Init.kaiming_uniform!(@weight, a: Math.sqrt(5))
         if @bias
-          fan_in, _ = Init.calculate_fan_in_and_fan_out(@weight)
+          fan_in, _ = Init._calculate_fan_in_and_fan_out(@weight)
           bound = 1 / Math.sqrt(fan_in)
-          Init.uniform_(@bias, -bound, bound)
+          Init.uniform!(@bias, a: -bound, b: bound)
         end
       end
 
-      def inspect
-        "Linear(in_features: #{@in_features.inspect}, out_features: #{@out_features.inspect}, bias: #{(!@bias.nil?).inspect})"
+      def forward(input)
+        F.linear(input, @weight, @bias)
+      end
+
+      def extra_inspect
+        format("in_features: %s, out_features: %s, bias: %s", @in_features, @out_features, !@bias.nil?)
       end
     end
   end

data/lib/torch/nn/local_response_norm.rb

@@ -0,0 +1,21 @@
+module Torch
+  module NN
+    class LocalResponseNorm < Module
+      def initialize(size, alpha: 1e-4, beta: 0.75, k: 1.0)
+        super()
+        @size = size
+        @alpha = alpha
+        @beta = beta
+        @k = k
+      end
+
+      def forward(input)
+        F.local_response_norm(input, @size, alpha: @alpha, beta: @beta, k: @k)
+      end
+
+      def extra_inspect
+        format("%{size}, alpha: %{alpha}, beta: %{beta}, k: %{k}", **dict)
+      end
+    end
+  end
+end

data/lib/torch/nn/log_sigmoid.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class LogSigmoid < Module
+      def forward(input)
+        F.log_sigmoid(input)
+      end
+    end
+  end
+end