torch-rb 0.1.0 → 0.1.5

data/lib/torch/optim/rprop.rb

@@ -0,0 +1,68 @@
+# ported from https://github.com/pytorch/pytorch/blob/master/torch/optim/rprop.py
+module Torch
+  module Optim
+    class Rprop < Optimizer
+      def initialize(params, lr: 1e-2, etas: [0.5, 1.2], step_sizes: [1e-6, 50])
+        raise ArgumentError, "Invalid learning rate: #{lr}" if lr < 0
+        raise ArgumentError, "Invalid eta values: #{etas[0]}, #{etas[1]}" if etas[0] < 0 || etas[0] >= 1 || etas[1] < 1
+
+        defaults = {lr: lr, etas: etas, step_sizes: step_sizes}
+        super(params, defaults)
+      end
+
+      def step(closure = nil)
+        # TODO implement []=
+        raise NotImplementedYet
+
+        loss = nil
+        if closure
+          loss = closure.call
+        end
+
+        @param_groups.each do |group|
+          group[:params].each do |p|
+            next unless p.grad
+            grad = p.grad.data
+            if grad.sparse?
+              raise Error, "Rprop does not support sparse gradients"
+            end
+            state = @state[p]
+
+            # State initialization
+            if state.size == 0
+              state[:step] = 0
+              state[:prev] = Torch.zeros_like(p.data)
+              state[:step_size] = grad.new.resize_as!(grad).fill!(group[:lr])
+            end
+
+            etaminus, etaplus = group[:etas]
+            step_size_min, step_size_max = group[:step_sizes]
+            step_size = state[:step_size]
+
+            state[:step] += 1
+
+            sign = grad.mul(state[:prev]).sign
+            sign[sign.gt(0)] = etaplus
+            sign[sign.lt(0)] = etaminus
+            sign[sign.eq(0)] = 1
+
+            # update stepsizes with step size updates
+            step_size.mul!(sign).clamp!(step_size_min, step_size_max)
+
+            # for dir<0, dfdx=0
+            # for dir>=0 dfdx=dfdx
+            grad = grad.clone
+            grad[sign.eq(etaminus)] = 0
+
+            # update parameters
+            p.data.addcmul!(-1, grad.sign, step_size)
+
+            state[:prev].copy!(grad)
+          end
+        end
+
+        loss
+      end
+    end
+  end
+end

data/lib/torch/optim/sgd.rb

@@ -0,0 +1,60 @@
+# ported from https://github.com/pytorch/pytorch/blob/master/torch/optim/sgd.py
+module Torch
+  module Optim
+    class SGD < Optimizer
+      def initialize(params, lr:, momentum: 0, dampening: 0, weight_decay: 0, nesterov: false)
+        raise ArgumentError, "Invalid learning rate: #{lr}" if lr < 0.0
+        raise ArgumentError, "Invalid momentum value: #{momentum}" if momentum < 0.0
+        raise ArgumentError, "Invalid weight_decay value: #{weight_decay}" if weight_decay < 0.0
+
+        defaults = {lr: lr, momentum: momentum, dampening: dampening, weight_decay: weight_decay, nesterov: nesterov}
+
+        if nesterov && (momentum <= 0 || dampening != 0)
+          raise ArgumentError, "Nesterov momentum requires a momentum and zero dampening"
+        end
+
+        super(params, defaults)
+      end
+
+      def step(closure = nil)
+        loss = nil
+        if closure
+          loss = closure.call
+        end
+
+        @param_groups.each do |group|
+          weight_decay = group[:weight_decay]
+          momentum = group[:momentum]
+          dampening = group[:dampening]
+          nesterov = group[:nesterov]
+
+          group[:params].each do |p|
+            next unless p.grad
+            d_p = p.grad.data
+            if weight_decay != 0
+              d_p.add!(weight_decay, p.data)
+            end
+            if momentum != 0
+              param_state = @state[p]
+              if !param_state.key(:momentum_buffer)
+                buf = param_state[:momentum_buffer] = Torch.clone(d_p).detach
+              else
+                buf = param_state[:momentum_buffer]
+                buf.mul!(momentum).add!(1 - dampening, d_p)
+              end
+              if nesterov
+                d_p = d_p.add(momentum, buf)
+              else
+                d_p = buf
+              end
+            end
+
+            p.data.add!(-group[:lr], d_p)
+          end
+        end
+
+        loss
+      end
+    end
+  end
+end

data/lib/torch/random.rb

@@ -0,0 +1,10 @@
+module Torch
+  module Random
+    class << self
+      # not available through LibTorch
+      def initial_seed
+        raise NotImplementedYet
+      end
+    end
+  end
+end

data/lib/torch/tensor.rb

@@ -5,12 +5,8 @@ module Torch
 
     alias_method :requires_grad?, :requires_grad
 
-    def self.new(*size)
-      if size.first.is_a?(Tensor)
-        size.first
-      else
-        Torch.rand(*size)
-      end
+    def self.new(*args)
+      FloatTensor.new(*args)
     end
 
     def dtype
@@ -28,12 +24,18 @@ module Torch
     end
 
     def to_a
-      reshape(_data, shape)
+      reshape_arr(_flat_data, shape)
+    end
+
+    # TODO support dtype
+    def to(device, non_blocking: false, copy: false)
+      device = Device.new(device) if device.is_a?(String)
+      _to(device, _dtype, non_blocking, copy)
     end
 
     def size(dim = nil)
       if dim
-        _size(dim)
+        _size_int(dim)
       else
         shape
       end
@@ -51,19 +53,23 @@ module Torch
       if numel != 1
         raise Error, "only one element tensors can be converted to Ruby scalars"
       end
-      _data.first
+      _flat_data.first
     end
 
-    def data
-      Torch.tensor(to_a)
+    # unsure if this is correct
+    def new
+      Torch.empty(0, dtype: dtype)
+    end
+
+    def backward(gradient = nil)
+      _backward(gradient)
     end
 
     # TODO read directly from memory
     def numo
-      raise Error, "Numo not found" unless defined?(Numo::NArray)
       cls = Torch._dtype_to_numo[dtype]
       raise Error, "Cannot convert #{dtype} to Numo" unless cls
-      cls.cast(_data).reshape(*shape)
+      cls.cast(_flat_data).reshape(*shape)
     end
 
     def new_ones(*size, **options)
@@ -74,8 +80,33 @@ module Torch
       _requires_grad!(requires_grad)
     end
 
+    def type(dtype)
+      enum = DTYPE_TO_ENUM[dtype]
+      raise Error, "Unknown type: #{dtype}" unless enum
+      _type(enum)
+    end
+
+    # start temp operations
+
+    def add!(value = 1, other)
+      if other.is_a?(Numeric)
+        _add__scalar(other, value)
+      else
+        # need to use alpha for sparse tensors instead of multiplying
+        _add__tensor(other, value)
+      end
+    end
+
+    def mul!(other)
+      if other.is_a?(Numeric)
+        _mul__scalar(other)
+      else
+        _mul__tensor(other)
+      end
+    end
+
     # operations
-    %w(add sub mul div remainder pow neg sum mean num norm min max dot matmul exp log unsqueeze).each do |op|
+    %w(log_softmax mean softmax sum topk).each do |op|
       define_method(op) do |*args, **options, &block|
         if options.any?
           Torch.send(op, self, *args, **options, &block)
@@ -85,6 +116,8 @@ module Torch
       end
     end
 
+    # end temp operations
+
     def +(other)
       add(other)
     end
@@ -117,18 +150,56 @@ module Torch
       item <=> other
     end
 
-    # TODO use accessor C++ method
-    def [](index, *args)
-      v = _access(index)
-      args.each do |i|
-        v = v._access(i)
+    # based on python_variable_indexing.cpp
+    def [](*indexes)
+      result = self
+      dim = 0
+      indexes.each do |index|
+        if index.is_a?(Numeric)
+          result = result._select_int(dim, index)
+        elsif index.is_a?(Range)
+          finish = index.end
+          finish += 1 unless index.exclude_end?
+          result = result._slice_tensor(dim, index.begin, finish, 1)
+          dim += 1
+        elsif index.nil?
+          result = result.unsqueeze(dim)
+          dim += 1
+        elsif index == true
+          result = result.unsqueeze(dim)
+          # TODO handle false
+        else
+          raise Error, "Unsupported index type: #{index.class.name}"
+        end
+      end
+      result
+    end
+
+    # TODO
+    # based on python_variable_indexing.cpp
+    def []=(index, value)
+      raise ArgumentError, "Tensor does not support deleting items" if value.nil?
+
+      value = Torch.tensor(value) unless value.is_a?(Tensor)
+
+      if index.is_a?(Numeric)
+        copy_to(_select_int(0, index), value)
+      elsif index.is_a?(Range)
+        finish = index.end
+        finish += 1 unless index.exclude_end?
+        copy_to(_slice_tensor(0, index.begin, finish, 1), value)
+      else
+        raise Error, "Unsupported index type: #{index.class.name}"
       end
-      v
     end
 
     private
 
-    def reshape(arr, dims)
+    def copy_to(dst, src)
+      dst.copy!(src)
+    end
+
+    def reshape_arr(arr, dims)
       if dims.empty?
         arr
       else

data/lib/torch/utils/data/data_loader.rb

@@ -2,10 +2,25 @@ module Torch
   module Utils
     module Data
       class DataLoader
+        include Enumerable
+
+        attr_reader :dataset
+
         def initialize(dataset, batch_size: 1)
           @dataset = dataset
           @batch_size = batch_size
         end
+
+        def each
+          size.times do |i|
+            start_index = i * @batch_size
+            yield @dataset[start_index...(start_index + @batch_size)]
+          end
+        end
+
+        def size
+          (@dataset.size / @batch_size.to_f).ceil
+        end
       end
     end
   end

data/lib/torch/utils/data/tensor_dataset.rb

@@ -3,11 +3,18 @@ module Torch
     module Data
       class TensorDataset
         def initialize(*tensors)
+          unless tensors.all? { |t| t.size(0) == tensors[0].size(0) }
+            raise Error, "Tensors must all have same dim 0 size"
+          end
           @tensors = tensors
         end
 
         def [](index)
-          tensors.map { |t| t[index] }
+          @tensors.map { |t| t[index] }
+        end
+
+        def size
+          @tensors[0].size(0)
         end
       end
     end

data/lib/torch/version.rb

@@ -1,3 +1,3 @@
 module Torch
-  VERSION = "0.1.0"
+  VERSION = "0.1.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: torch-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-26 00:00:00.000000000 Z
+date: 2019-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -106,26 +106,97 @@ files:
 - README.md
 - ext/torch/ext.cpp
 - ext/torch/extconf.rb
+- ext/torch/nn_functions.cpp
+- ext/torch/nn_functions.hpp
+- ext/torch/templates.hpp
+- ext/torch/tensor_functions.cpp
+- ext/torch/tensor_functions.hpp
+- ext/torch/torch_functions.cpp
+- ext/torch/torch_functions.hpp
 - lib/torch-rb.rb
 - lib/torch.rb
 - lib/torch/ext.bundle
 - lib/torch/inspector.rb
+- lib/torch/native/dispatcher.rb
+- lib/torch/native/function.rb
+- lib/torch/native/generator.rb
+- lib/torch/native/native_functions.yaml
+- lib/torch/native/parser.rb
+- lib/torch/nn/alpha_dropout.rb
+- lib/torch/nn/avg_pool2d.rb
+- lib/torch/nn/avg_poolnd.rb
+- lib/torch/nn/bce_loss.rb
+- lib/torch/nn/bce_with_logits_loss.rb
+- lib/torch/nn/bilinear.rb
 - lib/torch/nn/conv2d.rb
+- lib/torch/nn/convnd.rb
+- lib/torch/nn/cosine_embedding_loss.rb
+- lib/torch/nn/cosine_similarity.rb
+- lib/torch/nn/cross_entropy_loss.rb
+- lib/torch/nn/ctc_loss.rb
+- lib/torch/nn/dropout.rb
+- lib/torch/nn/dropout2d.rb
+- lib/torch/nn/dropout3d.rb
+- lib/torch/nn/dropoutnd.rb
+- lib/torch/nn/embedding.rb
+- lib/torch/nn/embedding_bag.rb
+- lib/torch/nn/feature_alpha_dropout.rb
 - lib/torch/nn/functional.rb
+- lib/torch/nn/hinge_embedding_loss.rb
+- lib/torch/nn/identity.rb
 - lib/torch/nn/init.rb
+- lib/torch/nn/kl_div_loss.rb
+- lib/torch/nn/l1_loss.rb
+- lib/torch/nn/leaky_relu.rb
 - lib/torch/nn/linear.rb
+- lib/torch/nn/log_softmax.rb
+- lib/torch/nn/loss.rb
+- lib/torch/nn/margin_ranking_loss.rb
+- lib/torch/nn/max_pool2d.rb
+- lib/torch/nn/max_poolnd.rb
 - lib/torch/nn/module.rb
 - lib/torch/nn/mse_loss.rb
+- lib/torch/nn/multi_label_margin_loss.rb
+- lib/torch/nn/multi_label_soft_margin_loss.rb
+- lib/torch/nn/multi_margin_loss.rb
+- lib/torch/nn/nll_loss.rb
+- lib/torch/nn/pairwise_distance.rb
 - lib/torch/nn/parameter.rb
+- lib/torch/nn/poisson_nll_loss.rb
+- lib/torch/nn/prelu.rb
 - lib/torch/nn/relu.rb
+- lib/torch/nn/rnn.rb
+- lib/torch/nn/rnn_base.rb
 - lib/torch/nn/sequential.rb
+- lib/torch/nn/sigmoid.rb
+- lib/torch/nn/smooth_l1_loss.rb
+- lib/torch/nn/soft_margin_loss.rb
+- lib/torch/nn/softmax.rb
+- lib/torch/nn/softmax2d.rb
+- lib/torch/nn/softmin.rb
+- lib/torch/nn/softplus.rb
+- lib/torch/nn/triplet_margin_loss.rb
+- lib/torch/nn/weighted_loss.rb
+- lib/torch/optim/adadelta.rb
+- lib/torch/optim/adagrad.rb
+- lib/torch/optim/adam.rb
+- lib/torch/optim/adamax.rb
+- lib/torch/optim/adamw.rb
+- lib/torch/optim/asgd.rb
+- lib/torch/optim/lr_scheduler/lr_scheduler.rb
+- lib/torch/optim/lr_scheduler/step_lr.rb
+- lib/torch/optim/optimizer.rb
+- lib/torch/optim/rmsprop.rb
+- lib/torch/optim/rprop.rb
+- lib/torch/optim/sgd.rb
+- lib/torch/random.rb
 - lib/torch/tensor.rb
 - lib/torch/utils/data/data_loader.rb
 - lib/torch/utils/data/tensor_dataset.rb
 - lib/torch/version.rb
 homepage: https://github.com/ankane/torch-rb
 licenses:
-- MIT
+- BSD-3-Clause
 metadata: {}
 post_install_message: 
 rdoc_options: []