torch-rb 0.1.0 → 0.1.5

data/lib/torch/nn/relu.rb

@@ -1,12 +1,17 @@
 module Torch
   module NN
     class ReLU < Module
-      def initialize #(inplace: false)
-        # @inplace = inplace
+      def initialize(inplace: false)
+        super()
+        @inplace = inplace
       end
 
       def forward(input)
-        F.relu(input) #, inplace: @inplace)
+        F.relu(input, inplace: @inplace)
+      end
+
+      def extra_inspect
+        @inplace ? "inplace: true" : ""
       end
     end
   end

data/lib/torch/nn/rnn.rb

@@ -0,0 +1,22 @@
+module Torch
+  module NN
+    class RNN < RNNBase
+      def initialize(*args, **options)
+        if options.key?(:nonlinearity)
+          if options[:nonlinearity] == "tanh"
+            mode = "RNN_TANH"
+          elsif options[:nonlinearity] == "relu"
+            mode = "RNN_RELU"
+          else
+            raise ArgumentError, "Unknown nonlinearity: #{options[:nonlinearity]}"
+          end
+          options.delete(:nonlinearity)
+        else
+          mode = "RNN_TANH"
+        end
+
+        super(mode, *args, **options)
+      end
+    end
+  end
+end

data/lib/torch/nn/rnn_base.rb

@@ -0,0 +1,154 @@
+module Torch
+  module NN
+    class RNNBase < Module
+      def initialize(mode, input_size, hidden_size, num_layers: 1, bias: true,
+        batch_first: false, dropout: 0.0, bidirectional: false)
+
+        super()
+        @mode = mode
+        @input_size = input_size
+        @hidden_size = hidden_size
+        @num_layers = num_layers
+        @bias = bias
+        @batch_first = batch_first
+        @dropout = dropout.to_f
+        @bidirectional = bidirectional
+        num_directions = bidirectional ? 2 : 1
+
+        if !dropout.is_a?(Numeric) || !(dropout >= 0 && dropout <= 1)
+          raise ArgumentError, "dropout should be a number in range [0, 1] " +
+                               "representing the probability of an element being " +
+                               "zeroed"
+        end
+        if dropout > 0 && num_layers == 1
+          warn "dropout option adds dropout after all but last " +
+               "recurrent layer, so non-zero dropout expects " +
+               "num_layers greater than 1, but got dropout=#{dropout} and " +
+               "num_layers=#{num_layers}"
+        end
+
+        gate_size =
+          case mode
+          when "LSTM"
+            4 * hidden_size
+          when "GRU"
+            3 * hidden_size
+          when "RNN_TANH"
+            hidden_size
+          when "RNN_RELU"
+            hidden_size
+          else
+            raise ArgumentError, "Unrecognized RNN mode: #{mode}"
+          end
+
+        @all_weights = []
+        num_layers.times do |layer|
+          num_directions.times do |direction|
+            layer_input_size = layer == 0 ? input_size : hidden_size * num_directions
+
+            w_ih = Parameter.new(Torch::Tensor.new(gate_size, layer_input_size))
+            w_hh = Parameter.new(Torch::Tensor.new(gate_size, hidden_size))
+            b_ih = Parameter.new(Torch::Tensor.new(gate_size))
+            # Second bias vector included for CuDNN compatibility. Only one
+            # bias vector is needed in standard definition.
+            b_hh = Parameter.new(Torch::Tensor.new(gate_size))
+            layer_params = [w_ih, w_hh, b_ih, b_hh]
+
+            suffix = direction == 1 ? "_reverse" : ""
+            param_names = ["weight_ih_l%s%s", "weight_hh_l%s%s"]
+            if bias
+              param_names += ["bias_ih_l%s%s", "bias_hh_l%s%s"]
+            end
+            param_names.map! { |x| x % [layer, suffix] }
+
+            param_names.zip(layer_params) do |name, param|
+              instance_variable_set("@#{name}", param)
+            end
+            @all_weights << param_names
+          end
+        end
+
+        flatten_parameters
+        reset_parameters
+      end
+
+      def flatten_parameters
+        # no-op unless module is on the GPU and cuDNN is enabled
+      end
+
+      def _apply(fn)
+        ret = super
+        flatten_parameters
+        ret
+      end
+
+      def reset_parameters
+        stdv = 1.0 / Math.sqrt(@hidden_size)
+        parameters.each do |weight|
+          Init.uniform!(weight, a: -stdv, b: stdv)
+        end
+      end
+
+      def permute_hidden(hx, permutation)
+        raise NotImplementedYet
+      end
+
+      def forward(input, hx: nil)
+        raise NotImplementedYet
+
+        is_packed = false # TODO isinstance(input, PackedSequence)
+        if is_packed
+          input, batch_sizes, sorted_indices, unsorted_indices = input
+          max_batch_size = batch_sizes[0]
+          max_batch_size = max_batch_size.to_i
+        else
+          batch_sizes = nil
+          max_batch_size = @batch_first ? input.size(0) : input.size(1)
+          sorted_indices = nil
+          unsorted_indices = nil
+        end
+
+        if hx.nil?
+          num_directions = @bidirectional ? 2 : 1
+          hx = Torch.zeros(@num_layers * num_directions, max_batch_size,
+            @hidden_size, dtype: input.dtype, device: input.device)
+        else
+          # Each batch of the hidden state should match the input sequence that
+          # the user believes he/she is passing in.
+          hx = permute_hidden(hx, sorted_indices)
+        end
+
+        check_forward_args(input, hx, batch_sizes)
+        _rnn_impls = {
+          "RNN_TANH" => Torch.method(:_rnn_tanh),
+          "RNN_RELU" => Torch.method(:_rnn_relu)
+        }
+        _impl = _rnn_impls[@mode]
+        if batch_sizes.nil?
+          result = _impl.call(input, hx, _get_flat_weights, @bias, @num_layers,
+                           @dropout, @training, @bidirectional, @batch_first)
+        else
+          result = _impl.call(input, batch_sizes, hx, _get_flat_weights, @bias,
+                           @num_layers, @dropout, @training, @bidirectional)
+        end
+        output = result[0]
+        hidden = result[1]
+
+        if is_packed
+          raise NotImplementedYet
+          # output = PackedSequence(output, batch_sizes, sorted_indices, unsorted_indices)
+        end
+        [output, permute_hidden(hidden, unsorted_indices)]
+      end
+
+      # TODO add more parameters
+      def extra_inspect
+        s = String.new("%{input_size}, %{hidden_size}")
+        if @num_layers != 1
+          s += ", num_layers: %{num_layers}"
+        end
+        format(s, input_size: @input_size, hidden_size: @hidden_size, num_layers: @num_layers)
+      end
+    end
+  end
+end

data/lib/torch/nn/sequential.rb

@@ -2,28 +2,19 @@ module Torch
   module NN
     class Sequential < Module
       def initialize(*args)
-        @modules = {}
+        super()
         # TODO support hash arg (named modules)
         args.each_with_index do |mod, idx|
           add_module(idx.to_s, mod)
         end
       end
 
-      def add_module(name, mod)
-        # TODO add checks
-        @modules[name] = mod
-      end
-
       def forward(input)
         @modules.values.each do |mod|
           input = mod.call(input)
         end
         input
       end
-
-      def parameters
-        @modules.flat_map { |_, mod| mod.parameters }
-      end
     end
   end
 end

data/lib/torch/nn/sigmoid.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class Sigmoid < Module
+      def forward(input)
+        Torch.sigmoid(input)
+      end
+    end
+  end
+end

data/lib/torch/nn/smooth_l1_loss.rb

@@ -0,0 +1,13 @@
+module Torch
+  module NN
+    class SmoothL1Loss < Loss
+      def initialize(reduction: "mean")
+        super(reduction)
+      end
+
+      def forward(input, target)
+        F.smooth_l1_loss(input, target, reduction: @reduction)
+      end
+    end
+  end
+end

data/lib/torch/nn/soft_margin_loss.rb

@@ -0,0 +1,13 @@
+module Torch
+  module NN
+    class SoftMarginLoss < Loss
+      def initialize(reduction: "mean")
+        super(reduction)
+      end
+
+      def forward(input, target)
+        F.soft_margin_loss(input, target, reduction: @reduction)
+      end
+    end
+  end
+end

data/lib/torch/nn/softmax.rb

@@ -0,0 +1,18 @@
+module Torch
+  module NN
+    class Softmax < Module
+      def initialize(dim: nil)
+        super()
+        @dim = dim
+      end
+
+      def forward(input)
+        F.softmax(input, dim: @dim)
+      end
+
+      def extra_inspect
+        format("dim: %s", @dim)
+      end
+    end
+  end
+end

data/lib/torch/nn/softmax2d.rb

@@ -0,0 +1,10 @@
+module Torch
+  module NN
+    class Softmax2d < Module
+      def forward(input)
+        raise ArgumentError, "Softmax2d requires a 4D tensor as input" unless input.dim == 4
+        F.softmax(input, dim: 1)
+      end
+    end
+  end
+end

data/lib/torch/nn/softmin.rb

@@ -0,0 +1,14 @@
+module Torch
+  module NN
+    class Softmin < Module
+      def initialize(dim: nil)
+        super()
+        @dim = dim
+      end
+
+      def forward(input)
+        F.softmin(input, dim: @dim)
+      end
+    end
+  end
+end

data/lib/torch/nn/softplus.rb

@@ -0,0 +1,19 @@
+module Torch
+  module NN
+    class Softplus < Module
+      def initialize(beta: 1, threshold: 20)
+        super()
+        @beta = beta
+        @threshold = threshold
+      end
+
+      def forward(input)
+        F.softplus(input, beta: @beta, threshold: @threshold)
+      end
+
+      def extra_inspect
+        format("beta: %s, threshold: %s", @beta, @threshold)
+      end
+    end
+  end
+end

data/lib/torch/nn/triplet_margin_loss.rb

@@ -0,0 +1,18 @@
+module Torch
+  module NN
+    class TripletMarginLoss < Loss
+      def initialize(margin: 1.0, p: 2.0, eps: 1e-6, swap: false, reduction: "mean")
+        super(reduction)
+        @margin = margin
+        @p = p
+        @eps = eps
+        @swap = swap
+      end
+
+      def forward(anchor, positive, negative)
+        F.triplet_margin_loss(anchor, positive, negative, margin: @margin, p: @p,
+                              eps: @eps, swap: @swap, reduction: @reduction)
+      end
+    end
+  end
+end

data/lib/torch/nn/weighted_loss.rb

@@ -0,0 +1,10 @@
+module Torch
+  module NN
+    class WeightedLoss < Loss
+      def initialize(weight, reduction)
+        super(reduction)
+        register_buffer("weight", weight)
+      end
+    end
+  end
+end

data/lib/torch/optim/adadelta.rb

@@ -0,0 +1,57 @@
+# ported from https://github.com/pytorch/pytorch/blob/master/torch/optim/adadelta.py
+module Torch
+  module Optim
+    class Adadelta < Optimizer
+      def initialize(params, lr: 1.0, rho: 0.9, eps: 1e-6, weight_decay: 0)
+        raise ArgumentError, "Invalid learning rate: #{lr}" if lr < 0
+        raise ArgumentError, "Invalid rho value: #{rho}" if rho < 0 || rho > 1
+        raise ArgumentError, "Invalid epsilon value: #{eps}" if eps < 0
+        raise ArgumentError, "Invalid weight_decay value: #{weight_decay}" if weight_decay < 0
+
+        defaults = {lr: lr, rho: rho, eps: eps, weight_decay: weight_decay}
+        super(params, defaults)
+      end
+
+      def step(closure = nil)
+        loss = nil
+        if closure
+          loss = closure.call
+        end
+
+        @param_groups.each do |group|
+          group[:params].each do |p|
+            next unless p.grad
+            grad = p.grad.data
+            if grad.sparse?
+              raise Error, "Adadelta does not support sparse gradients"
+            end
+            state = @state[p]
+
+            if state.size == 0
+              state[:step] = 0
+              state[:square_avg] = Torch.zeros_like(p.data)
+              state[:acc_delta] = Torch.zeros_like(p.data)
+            end
+
+            square_avg, acc_delta = state[:square_avg], state[:acc_delta]
+            rho, eps = group[:rho], group[:eps]
+
+            state[:step] += 1
+
+            if group[:weight_decay] != 0
+              grad = grad.add(group[:weight_decay], p.data)
+            end
+
+            square_avg.mul!(rho).addcmul!(1 - rho, grad, grad)
+            std = square_avg.add(eps).sqrt!
+            delta = acc_delta.add(eps).sqrt!.div!(std).mul!(grad)
+            p.data.add!(-group[:lr], delta)
+            acc_delta.mul!(rho).addcmul!(1 - rho, delta, delta)
+          end
+        end
+
+        loss
+      end
+    end
+  end
+end

data/lib/torch/optim/adagrad.rb

@@ -0,0 +1,71 @@
+# ported from https://github.com/pytorch/pytorch/blob/master/torch/optim/adagrad.py
+module Torch
+  module Optim
+    class Adagrad < Optimizer
+      def initialize(params, lr: 1e-2, lr_decay: 0, weight_decay: 0, initial_accumulator_value: 0, eps: 1e-10)
+        raise ArgumentError, "Invalid learning rate: #{lr}" if lr < 0
+        raise ArgumentError, "Invalid lr_decay value: #{lr_decay}" if lr_decay < 0
+        raise ArgumentError, "Invalid initial_accumulator_value value: #{initial_accumulator_value}" if initial_accumulator_value < 0
+        raise ArgumentError, "Invalid weight_decay value: #{weight_decay}" if weight_decay < 0
+        raise ArgumentError, "Invalid epsilon value: #{eps}" if eps < 0
+
+        defaults = {lr: lr, lr_decay: lr_decay, eps: eps, weight_decay: weight_decay, initial_accumulator_value: initial_accumulator_value}
+        super(params, defaults)
+
+        @param_groups.each do |group|
+          group[:params].each do |p|
+            state = @state[p]
+            state[:step] = 0
+            state[:sum] = Torch.full_like(p.data, initial_accumulator_value)
+          end
+        end
+      end
+
+      def share_memory
+        @param_groups.each do |group|
+          group[:params].each do |p|
+            state = @state[p]
+            state[:sum].share_memory!
+          end
+        end
+      end
+
+      def step(closure = nil)
+        loss = nil
+        if closure
+          loss = closure.call
+        end
+
+        @param_groups.each do |group|
+          group[:params].each do |p|
+            next unless p.grad
+
+            grad = p.grad.data
+            state = @state[p]
+
+            state[:step] += 1
+
+            if group[:weight_decay] != 0
+              if p.grad.data.sparse?
+                raise Error, "weight_decay option is not compatible with sparse gradients"
+              end
+              grad = grad.add(group[:weight_decay], p.data)
+            end
+
+            clr = group[:lr] / (1 + (state[:step] - 1) * group[:lr_decay])
+
+            if grad.sparse?
+              raise NotImplementedYet
+            else
+              state[:sum].addcmul!(1, grad, grad)
+              std = state[:sum].sqrt.add!(group[:eps])
+              p.data.addcdiv!(-clr, grad, std)
+            end
+          end
+        end
+
+        loss
+      end
+    end
+  end
+end