torch-rb 0.1.2 → 0.1.7

data/lib/torch/nn/sequential.rb

@@ -2,28 +2,19 @@ module Torch
   module NN
     class Sequential < Module
       def initialize(*args)
-        @modules = {}
+        super()
         # TODO support hash arg (named modules)
         args.each_with_index do |mod, idx|
           add_module(idx.to_s, mod)
         end
       end
 
-      def add_module(name, mod)
-        # TODO add checks
-        @modules[name] = mod
-      end
-
       def forward(input)
         @modules.values.each do |mod|
           input = mod.call(input)
         end
         input
       end
-
-      def parameters
-        @modules.flat_map { |_, mod| mod.parameters }
-      end
     end
   end
 end

data/lib/torch/nn/sigmoid.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class Sigmoid < Module
+      def forward(input)
+        Torch.sigmoid(input)
+      end
+    end
+  end
+end

data/lib/torch/nn/smooth_l1_loss.rb

@@ -0,0 +1,13 @@
+module Torch
+  module NN
+    class SmoothL1Loss < Loss
+      def initialize(reduction: "mean")
+        super(reduction)
+      end
+
+      def forward(input, target)
+        F.smooth_l1_loss(input, target, reduction: @reduction)
+      end
+    end
+  end
+end

data/lib/torch/nn/soft_margin_loss.rb

@@ -0,0 +1,13 @@
+module Torch
+  module NN
+    class SoftMarginLoss < Loss
+      def initialize(reduction: "mean")
+        super(reduction)
+      end
+
+      def forward(input, target)
+        F.soft_margin_loss(input, target, reduction: @reduction)
+      end
+    end
+  end
+end

data/lib/torch/nn/softmax.rb

@@ -0,0 +1,18 @@
+module Torch
+  module NN
+    class Softmax < Module
+      def initialize(dim: nil)
+        super()
+        @dim = dim
+      end
+
+      def forward(input)
+        F.softmax(input, dim: @dim)
+      end
+
+      def extra_inspect
+        format("dim: %s", @dim)
+      end
+    end
+  end
+end

data/lib/torch/nn/softmax2d.rb

@@ -0,0 +1,10 @@
+module Torch
+  module NN
+    class Softmax2d < Module
+      def forward(input)
+        raise ArgumentError, "Softmax2d requires a 4D tensor as input" unless input.dim == 4
+        F.softmax(input, dim: 1)
+      end
+    end
+  end
+end

data/lib/torch/nn/softmin.rb

@@ -0,0 +1,14 @@
+module Torch
+  module NN
+    class Softmin < Module
+      def initialize(dim: nil)
+        super()
+        @dim = dim
+      end
+
+      def forward(input)
+        F.softmin(input, dim: @dim)
+      end
+    end
+  end
+end

data/lib/torch/nn/softplus.rb

@@ -0,0 +1,19 @@
+module Torch
+  module NN
+    class Softplus < Module
+      def initialize(beta: 1, threshold: 20)
+        super()
+        @beta = beta
+        @threshold = threshold
+      end
+
+      def forward(input)
+        F.softplus(input, beta: @beta, threshold: @threshold)
+      end
+
+      def extra_inspect
+        format("beta: %s, threshold: %s", @beta, @threshold)
+      end
+    end
+  end
+end

data/lib/torch/nn/softshrink.rb

@@ -0,0 +1,18 @@
+module Torch
+  module NN
+    class Softshrink < Module
+      def initialize(lambd: 0.5)
+        super()
+        @lambd = lambd
+      end
+
+      def forward(input)
+        F.softshrink(input, @lambd)
+      end
+
+      def extra_inspect
+        @lambd.to_s
+      end
+    end
+  end
+end

data/lib/torch/nn/softsign.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class Softsign < Module
+      def forward(input)
+        F.softsign(input)
+      end
+    end
+  end
+end

data/lib/torch/nn/tanh.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class Tanh < Module
+      def forward(input)
+        Torch.tanh(input)
+      end
+    end
+  end
+end

data/lib/torch/nn/tanhshrink.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class Tanhshrink < Module
+      def forward(input)
+        F.tanhshrink(input)
+      end
+    end
+  end
+end

data/lib/torch/nn/triplet_margin_loss.rb

@@ -0,0 +1,18 @@
+module Torch
+  module NN
+    class TripletMarginLoss < Loss
+      def initialize(margin: 1.0, p: 2.0, eps: 1e-6, swap: false, reduction: "mean")
+        super(reduction)
+        @margin = margin
+        @p = p
+        @eps = eps
+        @swap = swap
+      end
+
+      def forward(anchor, positive, negative)
+        F.triplet_margin_loss(anchor, positive, negative, margin: @margin, p: @p,
+                              eps: @eps, swap: @swap, reduction: @reduction)
+      end
+    end
+  end
+end

data/lib/torch/nn/unfold.rb

@@ -0,0 +1,19 @@
+module Torch
+  module NN
+    class Unfold < Module
+      def initialize(kernel_size, dilation: 1, padding: 0, stride: 1)
+        super()
+        @kernel_size = kernel_size
+        @dilation = dilation
+        @padding = padding
+        @stride = stride
+      end
+
+      def forward(input)
+        F.unfold(input, @kernel_size, dilation: @dilation, padding: @padding, stride: @stride)
+      end
+
+      # TODO add extra_inspect
+    end
+  end
+end

data/lib/torch/nn/utils.rb

@@ -0,0 +1,25 @@
+module Torch
+  module NN
+    module Utils
+      def _single(value)
+        _ntuple(1, value)
+      end
+
+      def _pair(value)
+        _ntuple(2, value)
+      end
+
+      def _triple(value)
+        _ntuple(3, value)
+      end
+
+      def _quadrupal(value)
+        _ntuple(4, value)
+      end
+
+      def _ntuple(n, value)
+        value.is_a?(Array) ? value : [value] * n
+      end
+    end
+  end
+end

data/lib/torch/nn/weighted_loss.rb

@@ -0,0 +1,10 @@
+module Torch
+  module NN
+    class WeightedLoss < Loss
+      def initialize(weight, reduction)
+        super(reduction)
+        register_buffer("weight", weight)
+      end
+    end
+  end
+end

data/lib/torch/nn/zero_pad2d.rb

@@ -0,0 +1,9 @@
+module Torch
+  module NN
+    class ZeroPad2d < ConstantPad2d
+      def initialize(padding)
+        super(padding, 0.0)
+      end
+    end
+  end
+end

data/lib/torch/optim/adadelta.rb

@@ -0,0 +1,57 @@
+# ported from https://github.com/pytorch/pytorch/blob/master/torch/optim/adadelta.py
+module Torch
+  module Optim
+    class Adadelta < Optimizer
+      def initialize(params, lr: 1.0, rho: 0.9, eps: 1e-6, weight_decay: 0)
+        raise ArgumentError, "Invalid learning rate: #{lr}" if lr < 0
+        raise ArgumentError, "Invalid rho value: #{rho}" if rho < 0 || rho > 1
+        raise ArgumentError, "Invalid epsilon value: #{eps}" if eps < 0
+        raise ArgumentError, "Invalid weight_decay value: #{weight_decay}" if weight_decay < 0
+
+        defaults = {lr: lr, rho: rho, eps: eps, weight_decay: weight_decay}
+        super(params, defaults)
+      end
+
+      def step(closure = nil)
+        loss = nil
+        if closure
+          loss = closure.call
+        end
+
+        @param_groups.each do |group|
+          group[:params].each do |p|
+            next unless p.grad
+            grad = p.grad.data
+            if grad.sparse?
+              raise Error, "Adadelta does not support sparse gradients"
+            end
+            state = @state[p]
+
+            if state.size == 0
+              state[:step] = 0
+              state[:square_avg] = Torch.zeros_like(p.data)
+              state[:acc_delta] = Torch.zeros_like(p.data)
+            end
+
+            square_avg, acc_delta = state[:square_avg], state[:acc_delta]
+            rho, eps = group[:rho], group[:eps]
+
+            state[:step] += 1
+
+            if group[:weight_decay] != 0
+              grad = grad.add(group[:weight_decay], p.data)
+            end
+
+            square_avg.mul!(rho).addcmul!(1 - rho, grad, grad)
+            std = square_avg.add(eps).sqrt!
+            delta = acc_delta.add(eps).sqrt!.div!(std).mul!(grad)
+            p.data.add!(-group[:lr], delta)
+            acc_delta.mul!(rho).addcmul!(1 - rho, delta, delta)
+          end
+        end
+
+        loss
+      end
+    end
+  end
+end

data/lib/torch/optim/adagrad.rb

@@ -0,0 +1,71 @@
+# ported from https://github.com/pytorch/pytorch/blob/master/torch/optim/adagrad.py
+module Torch
+  module Optim
+    class Adagrad < Optimizer
+      def initialize(params, lr: 1e-2, lr_decay: 0, weight_decay: 0, initial_accumulator_value: 0, eps: 1e-10)
+        raise ArgumentError, "Invalid learning rate: #{lr}" if lr < 0
+        raise ArgumentError, "Invalid lr_decay value: #{lr_decay}" if lr_decay < 0
+        raise ArgumentError, "Invalid initial_accumulator_value value: #{initial_accumulator_value}" if initial_accumulator_value < 0
+        raise ArgumentError, "Invalid weight_decay value: #{weight_decay}" if weight_decay < 0
+        raise ArgumentError, "Invalid epsilon value: #{eps}" if eps < 0
+
+        defaults = {lr: lr, lr_decay: lr_decay, eps: eps, weight_decay: weight_decay, initial_accumulator_value: initial_accumulator_value}
+        super(params, defaults)
+
+        @param_groups.each do |group|
+          group[:params].each do |p|
+            state = @state[p]
+            state[:step] = 0
+            state[:sum] = Torch.full_like(p.data, initial_accumulator_value)
+          end
+        end
+      end
+
+      def share_memory
+        @param_groups.each do |group|
+          group[:params].each do |p|
+            state = @state[p]
+            state[:sum].share_memory!
+          end
+        end
+      end
+
+      def step(closure = nil)
+        loss = nil
+        if closure
+          loss = closure.call
+        end
+
+        @param_groups.each do |group|
+          group[:params].each do |p|
+            next unless p.grad
+
+            grad = p.grad.data
+            state = @state[p]
+
+            state[:step] += 1
+
+            if group[:weight_decay] != 0
+              if p.grad.data.sparse?
+                raise Error, "weight_decay option is not compatible with sparse gradients"
+              end
+              grad = grad.add(group[:weight_decay], p.data)
+            end
+
+            clr = group[:lr] / (1 + (state[:step] - 1) * group[:lr_decay])
+
+            if grad.sparse?
+              raise NotImplementedYet
+            else
+              state[:sum].addcmul!(1, grad, grad)
+              std = state[:sum].sqrt.add!(group[:eps])
+              p.data.addcdiv!(-clr, grad, std)
+            end
+          end
+        end
+
+        loss
+      end
+    end
+  end
+end

data/lib/torch/optim/adam.rb

@@ -0,0 +1,81 @@
+# ported from https://github.com/pytorch/pytorch/blob/master/torch/optim/adam.py
+module Torch
+  module Optim
+    class Adam < Optimizer
+      def initialize(params, lr: 1e-3, betas: [0.9, 0.999], eps: 1e-8, weight_decay: 0, amsgrad: false)
+        raise ArgumentError, "Invalid learning rate: #{lr}" if lr < 0
+        raise ArgumentError, "Invalid epsilon value: #{eps}" if eps < 0
+        raise ArgumentError, "Invalid beta parameter at index 0: #{betas[0]}" if betas[0] < 0 || betas[0] >= 1
+        raise ArgumentError, "Invalid beta parameter at index 1: #{betas[1]}" if betas[1] < 0 || betas[1] >= 1
+
+        defaults = {lr: lr, betas: betas, eps: eps, weight_decay: weight_decay, amsgrad: amsgrad}
+        super(params, defaults)
+      end
+
+      def step(closure = nil)
+        loss = nil
+        if closure
+          loss = closure.call
+        end
+
+        @param_groups.each do |group|
+          group[:params].each do |p|
+            next unless p.grad
+            grad = p.grad.data
+            if grad.sparse?
+              raise Error, "Adam does not support sparse gradients, please consider SparseAdam instead"
+            end
+            amsgrad = group[:amsgrad]
+
+            state = @state[p]
+
+            # State initialization
+            if state.size == 0
+              state[:step] = 0
+              # Exponential moving average of gradient values
+              state[:exp_avg] = Torch.zeros_like(p.data)
+              # Exponential moving average of squared gradient values
+              state[:exp_avg_sq] = Torch.zeros_like(p.data)
+              if amsgrad
+                # Maintains max of all exp. moving avg. of sq. grad. values
+                state[:max_exp_avg_sq] = Torch.zeros_like(p.data)
+              end
+            end
+
+            exp_avg, exp_avg_sq = state[:exp_avg], state[:exp_avg_sq]
+            if amsgrad
+              max_exp_avg_sq = state[:max_exp_avg_sq]
+            end
+            beta1, beta2 = group[:betas]
+
+            state[:step] += 1
+            bias_correction1 = 1 - beta1 ** state[:step]
+            bias_correction2 = 1 - beta2 ** state[:step]
+
+            if group[:weight_decay] != 0
+              grad.add!(group[:weight_decay], p.data)
+            end
+
+            # Decay the first and second moment running average coefficient
+            exp_avg.mul!(beta1).add!(1 - beta1, grad)
+            exp_avg_sq.mul!(beta2).addcmul!(1 - beta2, grad, grad)
+            if amsgrad
+              # Maintains the maximum of all 2nd moment running avg. till now
+              Torch.max(max_exp_avg_sq, exp_avg_sq, out: max_exp_avg_sq)
+              # Use the max. for normalizing running avg. of gradient
+              denom = (max_exp_avg_sq.sqrt / Math.sqrt(bias_correction2)).add!(group[:eps])
+            else
+              denom = (exp_avg_sq.sqrt / Math.sqrt(bias_correction2)).add!(group[:eps])
+            end
+
+            step_size = group[:lr] / bias_correction1
+
+            p.data.addcdiv!(-step_size, exp_avg, denom)
+          end
+        end
+
+        loss
+      end
+    end
+  end
+end