torch-rb 0.1.1 → 0.1.6

data/ext/torch/torch_functions.hpp

@@ -0,0 +1,6 @@
+// generated by rake generate:functions
+// do not edit by hand
+
+#pragma once
+
+void add_torch_functions(Module m);

data/lib/torch.rb

@@ -1,32 +1,181 @@
 # ext
 require "torch/ext"
 
+# native functions
+require "torch/native/generator"
+require "torch/native/parser"
+require "torch/native/dispatcher"
+
 # modules
 require "torch/inspector"
 require "torch/tensor"
 require "torch/version"
 
-# nn
+# optim
+require "torch/optim/optimizer"
+require "torch/optim/adadelta"
+require "torch/optim/adagrad"
+require "torch/optim/adam"
+require "torch/optim/adamax"
+require "torch/optim/adamw"
+require "torch/optim/asgd"
+require "torch/optim/rmsprop"
+require "torch/optim/rprop"
+require "torch/optim/sgd"
+
+# optim lr_scheduler
+require "torch/optim/lr_scheduler/lr_scheduler"
+require "torch/optim/lr_scheduler/step_lr"
+
+# nn parameters
+require "torch/nn/parameter"
+require "torch/nn/utils"
+
+# nn containers
 require "torch/nn/module"
-require "torch/nn/init"
+require "torch/nn/sequential"
+
+# nn convolution layers
+require "torch/nn/convnd"
+require "torch/nn/conv1d"
 require "torch/nn/conv2d"
-require "torch/nn/functional"
+require "torch/nn/conv3d"
+require "torch/nn/unfold"
+require "torch/nn/fold"
+
+# nn pooling layers
+require "torch/nn/max_poolnd"
+require "torch/nn/max_pool1d"
+require "torch/nn/max_pool2d"
+require "torch/nn/max_pool3d"
+require "torch/nn/max_unpoolnd"
+require "torch/nn/max_unpool1d"
+require "torch/nn/max_unpool2d"
+require "torch/nn/max_unpool3d"
+require "torch/nn/avg_poolnd"
+require "torch/nn/avg_pool1d"
+require "torch/nn/avg_pool2d"
+require "torch/nn/avg_pool3d"
+require "torch/nn/lp_poolnd"
+require "torch/nn/lp_pool1d"
+require "torch/nn/lp_pool2d"
+
+# nn padding layers
+require "torch/nn/reflection_padnd"
+require "torch/nn/reflection_pad1d"
+require "torch/nn/reflection_pad2d"
+require "torch/nn/replication_padnd"
+require "torch/nn/replication_pad1d"
+require "torch/nn/replication_pad2d"
+require "torch/nn/replication_pad3d"
+require "torch/nn/constant_padnd"
+require "torch/nn/constant_pad1d"
+require "torch/nn/constant_pad2d"
+require "torch/nn/constant_pad3d"
+require "torch/nn/zero_pad2d"
+
+# nn normalization layers
+require "torch/nn/batch_norm"
+require "torch/nn/batch_norm1d"
+require "torch/nn/batch_norm2d"
+require "torch/nn/batch_norm3d"
+require "torch/nn/group_norm"
+require "torch/nn/instance_norm"
+require "torch/nn/instance_norm1d"
+require "torch/nn/instance_norm2d"
+require "torch/nn/instance_norm3d"
+require "torch/nn/layer_norm"
+require "torch/nn/local_response_norm"
+
+# nn recurrent layers
+require "torch/nn/rnn_base"
+require "torch/nn/rnn"
+require "torch/nn/lstm"
+require "torch/nn/gru"
+
+# nn linear layers
+require "torch/nn/bilinear"
+require "torch/nn/identity"
 require "torch/nn/linear"
-require "torch/nn/parameter"
-require "torch/nn/sequential"
+
+# nn dropout layers
+require "torch/nn/dropoutnd"
+require "torch/nn/alpha_dropout"
+require "torch/nn/dropout"
+require "torch/nn/dropout2d"
+require "torch/nn/dropout3d"
+require "torch/nn/feature_alpha_dropout"
+
+# nn activations
+require "torch/nn/hardshrink"
+require "torch/nn/leaky_relu"
+require "torch/nn/log_sigmoid"
+require "torch/nn/prelu"
 require "torch/nn/relu"
+require "torch/nn/sigmoid"
+require "torch/nn/softplus"
+require "torch/nn/softshrink"
+require "torch/nn/softsign"
+require "torch/nn/tanh"
+require "torch/nn/tanhshrink"
+
+# nn activations other
+require "torch/nn/log_softmax"
+require "torch/nn/softmax"
+require "torch/nn/softmax2d"
+require "torch/nn/softmin"
+
+# nn sparse layers
+require "torch/nn/embedding"
+require "torch/nn/embedding_bag"
+
+# nn distance functions
+require "torch/nn/cosine_similarity"
+require "torch/nn/pairwise_distance"
+
+# nn loss functions
+require "torch/nn/loss"
+require "torch/nn/weighted_loss"
+require "torch/nn/bce_loss"
+require "torch/nn/bce_with_logits_loss"
+require "torch/nn/cosine_embedding_loss"
+require "torch/nn/cross_entropy_loss"
+require "torch/nn/ctc_loss"
+require "torch/nn/hinge_embedding_loss"
+require "torch/nn/kl_div_loss"
+require "torch/nn/l1_loss"
+require "torch/nn/margin_ranking_loss"
 require "torch/nn/mse_loss"
+require "torch/nn/multi_label_margin_loss"
+require "torch/nn/multi_label_soft_margin_loss"
+require "torch/nn/multi_margin_loss"
+require "torch/nn/nll_loss"
+require "torch/nn/poisson_nll_loss"
+require "torch/nn/smooth_l1_loss"
+require "torch/nn/soft_margin_loss"
+require "torch/nn/triplet_margin_loss"
+
+# nn other
+require "torch/nn/functional"
+require "torch/nn/init"
 
 # utils
 require "torch/utils/data/data_loader"
 require "torch/utils/data/tensor_dataset"
 
+# random
+require "torch/random"
+
 module Torch
   class Error < StandardError; end
+  class NotImplementedYet < StandardError
+    def message
+      "This feature has not been implemented yet. Consider submitting a PR."
+    end
+  end
 
   # keys: https://pytorch.org/docs/stable/tensor_attributes.html#torch.torch.dtype
   # values: https://github.com/pytorch/pytorch/blob/master/c10/core/ScalarType.h
-  # complex and quantized types not supported by PyTorch yet
   DTYPE_TO_ENUM = {
     uint8: 0,
     int8: 1,
@@ -42,22 +191,61 @@ module Torch
     float32: 6,
     double: 7,
     float64: 7,
-    # complex_half: 8,
-    # complex_float: 9,
-    # complex_double: 10,
+    complex_half: 8,
+    complex_float: 9,
+    complex_double: 10,
     bool: 11,
-    # qint8: 12,
-    # quint8: 13,
-    # qint32: 14,
-    # bfloat16: 15
+    qint8: 12,
+    quint8: 13,
+    qint32: 14,
+    bfloat16: 15
   }
   ENUM_TO_DTYPE = DTYPE_TO_ENUM.map(&:reverse).to_h
 
+  def self._make_tensor_class(dtype, cuda = false)
+    cls = Class.new
+    device = cuda ? "cuda" : "cpu"
+    cls.define_singleton_method("new") do |*args|
+      if args.size == 1 && args.first.is_a?(Tensor)
+        args.first.send(dtype).to(device)
+      elsif args.size == 1 && args.first.is_a?(Array)
+        Torch.tensor(args.first, dtype: dtype, device: device)
+      else
+        Torch.empty(*args, dtype: dtype, device: device)
+      end
+    end
+    cls
+  end
+
+  FloatTensor = _make_tensor_class(:float32)
+  DoubleTensor = _make_tensor_class(:float64)
+  HalfTensor = _make_tensor_class(:float16)
+  ByteTensor = _make_tensor_class(:uint8)
+  CharTensor = _make_tensor_class(:int8)
+  ShortTensor = _make_tensor_class(:int16)
+  IntTensor = _make_tensor_class(:int32)
+  LongTensor = _make_tensor_class(:int64)
+  BoolTensor = _make_tensor_class(:bool)
+
+  CUDA::FloatTensor = _make_tensor_class(:float32, true)
+  CUDA::DoubleTensor = _make_tensor_class(:float64, true)
+  CUDA::HalfTensor = _make_tensor_class(:float16, true)
+  CUDA::ByteTensor = _make_tensor_class(:uint8, true)
+  CUDA::CharTensor = _make_tensor_class(:int8, true)
+  CUDA::ShortTensor = _make_tensor_class(:int16, true)
+  CUDA::IntTensor = _make_tensor_class(:int32, true)
+  CUDA::LongTensor = _make_tensor_class(:int64, true)
+  CUDA::BoolTensor = _make_tensor_class(:bool, true)
+
   class << self
     # Torch.float, Torch.long, etc
-    DTYPE_TO_ENUM.each_key do |type|
-      define_method(type) do
-        type
+    DTYPE_TO_ENUM.each_key do |dtype|
+      define_method(dtype) do
+        dtype
+      end
+
+      Tensor.define_method(dtype) do
+        type(dtype)
       end
     end
 
@@ -67,17 +255,26 @@ module Torch
       obj.is_a?(Tensor)
     end
 
-    # TODO don't copy
     def from_numo(ndarray)
       dtype = _dtype_to_numo.find { |k, v| ndarray.is_a?(v) }
       raise Error, "Cannot convert #{ndarray.class.name} to tensor" unless dtype
-      tensor(ndarray.to_a, dtype: dtype[0])
+      options = tensor_options(device: "cpu", dtype: dtype[0])
+      # TODO pass pointer to array instead of creating string
+      str = ndarray.to_string
+      tensor = _from_blob(str, ndarray.shape, options)
+      # from_blob does not own the data, so we need to keep
+      # a reference to it for duration of tensor
+      # can remove when passing pointer directly
+      tensor.instance_variable_set("@_numo_str", str)
+      tensor
     end
 
     # private
     # use method for cases when Numo not available
     # or available after Torch loaded
     def _dtype_to_numo
+      raise Error, "Numo not found" unless defined?(Numo::NArray)
+
       {
         uint8: Numo::UInt8,
         int8: Numo::Int8,
@@ -89,6 +286,20 @@ module Torch
       }
     end
 
+    def no_grad
+      previous_value = grad_enabled?
+      begin
+        _set_grad_enabled(false)
+        yield
+      ensure
+        _set_grad_enabled(previous_value)
+      end
+    end
+
+    def device(str)
+      Device.new(str)
+    end
+
     # --- begin tensor creation: https://pytorch.org/cppdocs/notes/tensor_creation.html ---
 
     def arange(start, finish = nil, step = 1, **options)
@@ -158,8 +369,12 @@ module Torch
         data = [data].compact
       end
 
-      if options[:dtype].nil? && data.all? { |v| v.is_a?(Integer) }
-        options[:dtype] = :int64
+      if options[:dtype].nil?
+        if data.all? { |v| v.is_a?(Integer) }
+          options[:dtype] = :int64
+        elsif data.all? { |v| v == true || v == false }
+          options[:dtype] = :bool
+        end
       end
 
       _tensor(data, size, tensor_options(**options))
@@ -189,7 +404,7 @@ module Torch
         high = low
         low = 0
       end
-      rand(input.size, like_options(input, options))
+      randint(low, high, input.size, like_options(input, options))
     end
 
     def randn_like(input, **options)
@@ -200,99 +415,8 @@ module Torch
       zeros(input.size, like_options(input, options))
     end
 
-    # --- begin operations ---
-
-    %w(add sub mul div remainder).each do |op|
-      define_method(op) do |input, other, **options|
-        execute_op(op, input, other, **options)
-      end
-    end
-
-    def neg(input)
-      _neg(input)
-    end
-
-    def no_grad
-      previous_value = grad_enabled?
-      begin
-        _set_grad_enabled(false)
-        yield
-      ensure
-        _set_grad_enabled(previous_value)
-      end
-    end
-
-    # TODO support out
-    def mean(input, dim = nil, keepdim: false)
-      if dim
-        _mean_dim(input, dim, keepdim)
-      else
-        _mean(input)
-      end
-    end
-
-    # TODO support dtype
-    def sum(input, dim = nil, keepdim: false)
-      if dim
-        _sum_dim(input, dim, keepdim)
-      else
-        _sum(input)
-      end
-    end
-
-    def norm(input)
-      _norm(input)
-    end
-
-    def pow(input, exponent)
-      _pow(input, exponent)
-    end
-
-    def min(input)
-      _min(input)
-    end
-
-    def max(input)
-      _max(input)
-    end
-
-    def exp(input)
-      _exp(input)
-    end
-
-    def log(input)
-      _log(input)
-    end
-
-    def unsqueeze(input, dim)
-      _unsqueeze(input, dim)
-    end
-
-    def dot(input, tensor)
-      _dot(input, tensor)
-    end
-
-    def matmul(input, other)
-      _matmul(input, other)
-    end
-
     private
 
-    def execute_op(op, input, other, out: nil)
-      scalar = other.is_a?(Numeric)
-      if out
-        # TODO make work with scalars
-        raise Error, "out not supported with scalar yet" if scalar
-        send("_#{op}_out", out, input, other)
-      else
-        if scalar
-          send("_#{op}_scalar", input, other)
-        else
-          send("_#{op}", input, other)
-        end
-      end
-    end
-
     def tensor_size(size)
       size.flatten
     end

data/lib/torch/inspector.rb

@@ -1,41 +1,49 @@
 module Torch
   module Inspector
+    # TODO make more performance, especially when summarizing
+    # how? only read data that will be displayed
     def inspect
       data =
         if numel == 0
           "[]"
         elsif dim == 0
-          to_a.first
+          item
         else
-          values = to_a.flatten
-          abs = values.select { |v| v != 0 }.map(&:abs)
-          max = abs.max || 1
-          min = abs.min || 1
-
-          total = 0
-          if values.any? { |v| v < 0 }
-            total += 1
-          end
+          summarize = numel > 1000
+
+          if dtype == :bool
+            fmt = "%s"
+          else
+            values = to_a.flatten
+            abs = values.select { |v| v != 0 }.map(&:abs)
+            max = abs.max || 1
+            min = abs.min || 1
 
-          if floating_point?
-            sci = max / min.to_f > 1000 || max > 1e8 || min < 1e-4
+            total = 0
+            if values.any? { |v| v < 0 }
+              total += 1
+            end
+
+            if floating_point?
+              sci = max / min.to_f > 1000 || max > 1e8 || min < 1e-4
 
-            all_int = values.all? { |v| v == v.to_i }
-            decimal = all_int ? 1 : 4
+              all_int = values.all? { |v| v.finite? && v == v.to_i }
+              decimal = all_int ? 1 : 4
 
-            total += sci ? 10 : decimal + 1 + max.to_i.to_s.size
+              total += sci ? 10 : decimal + 1 + max.to_i.to_s.size
 
-            if sci
-              fmt = "%#{total}.4e"
+              if sci
+                fmt = "%#{total}.4e"
+              else
+                fmt = "%#{total}.#{decimal}f"
+              end
             else
-              fmt = "%#{total}.#{decimal}f"
+              total += max.to_s.size
+              fmt = "%#{total}d"
             end
-          else
-            total += max.to_s.size
-            fmt = "%#{total}d"
           end
 
-          inspect_level(to_a, fmt, dim - 1)
+          inspect_level(to_a, fmt, dim - 1, 0, summarize)
         end
 
       attributes = []
@@ -51,11 +59,30 @@ module Torch
 
     private
 
-    def inspect_level(arr, fmt, total, level = 0)
+    # TODO DRY code
+    def inspect_level(arr, fmt, total, level, summarize)
       if level == total
-        "[#{arr.map { |v| fmt % v }.join(", ")}]"
+        cols =
+          if summarize && arr.size > 7
+            arr[0..2].map { |v| fmt % v } +
+            ["..."] +
+            arr[-3..-1].map { |v| fmt % v }
+          else
+            arr.map { |v| fmt % v }
+          end
+
+        "[#{cols.join(", ")}]"
       else
-        "[#{arr.map { |row| inspect_level(row, fmt, total, level + 1) }.join(",#{"\n" * (total - level)}#{" " * (level + 8)}")}]"
+        rows =
+          if summarize && arr.size > 7
+            arr[0..2].map { |row| inspect_level(row, fmt, total, level + 1, summarize) } +
+            ["..."] +
+            arr[-3..-1].map { |row| inspect_level(row, fmt, total, level + 1, summarize) }
+          else
+            arr.map { |row| inspect_level(row, fmt, total, level + 1, summarize) }
+          end
+
+        "[#{rows.join(",#{"\n" * (total - level)}#{" " * (level + 8)}")}]"
       end
     end
   end