rubyzero 0.1.0

data/lib/rubyzero/core/tensor.rb

@@ -0,0 +1,56 @@
+module RubyZero::Core
+    # Tensor class
+    class Tensor
+        attr_accessor :data, :grad_fn, :grad, :device, :requires_grad
+        def initialize(data, dtype: nil, device: nil)
+            @device = device || RubyZero.device(:cpu)
+            @grad_fn = nil
+            @grad = nil
+            @requires_grad = false
+
+            # check data type
+            if data.is_a?(Array)
+                if dtype
+                    @data = dtype.get_type_on_calculator(device)[*data]
+                else
+                    @data = @device.caluculator::NArray[*data]
+                end
+            elsif data.is_a?(Numeric)
+                if dtype
+                    @data = dtype.get_type_on_calculator(device)[data]
+                else
+                    @data = @device.caluculator::NArray[data]
+                end
+            elsif data.is_a?(Numo::NArray)
+                @data = data
+            else 
+                raise Execptions::TypeNotSupported, "Tensor data type not supported: #{data.class}"
+            end
+            @dtype = DataTypes.from_xmo_dtype(@data.class)
+        end
+        # get data type
+        # @return [DataTypes::DataType]
+        def dtype
+            @dtype
+        end
+        # get shape
+        # @return [Array<Integer>]
+        def shape
+            @data.shape
+        end
+        # get tensor summary
+        # @return [String]
+        def inspect
+            numo_inspect = @data.inspect.split("\n")[1..nil].join("\n")
+            return "#{dtype}#shape=#{shape.to_a}\n#{numo_inspect}\ngrad_fn=#{@grad_fn.class}"
+        end
+        # @return [Boolean]
+        def requires_grad?
+            @requires_grad
+        end
+        # detach from graph
+        def detach
+            @grad_fn = nil
+        end
+    end
+end

data/lib/rubyzero/core/tensor_backward.rb

@@ -0,0 +1,17 @@
+module RubyZero::Core
+    class Tensor
+        def backward()
+            return unless @grad_fn
+            @grad ||= ones_like
+            dx = @grad_fn.backward(@grad)
+            self.grad_fn.inputs.each_with_index do |input, i|
+                if input.grad
+                    input.grad += dx[i]
+                else
+                    input.grad = dx[i]
+                end
+                input.backward()
+            end
+        end
+    end
+end

data/lib/rubyzero/core/tensor_initialize_methods.rb

@@ -0,0 +1,78 @@
+module RubyZero::Core
+    class Tensor
+        # initialize new tensor with given shape which filled with zeros.
+        def self.zeros(shape, dtype: RubyZero::Core::DataTypes::Float64, device: RubyZero.device(:cpu))
+            dtype = dtype.get_type_on_calculator(device)
+            data = dtype.zeros(shape)
+            Tensor.new(data, dtype:dtype, device:device)
+        end
+
+
+        # initialize new tensor with given another tensor's shape which filled with zeros.
+        def zeros_like()
+            Tensor.zeros(self.shape, dtype:self.dtype, device:self.device)
+        end
+
+        # initialize new tensor with given shape which filled with ones.
+        def self.ones(shape, dtype: RubyZero::Core::DataTypes::Float64, device: RubyZero.device(:cpu))
+            dtype = dtype.get_type_on_calculator(device)
+            data = dtype.ones(shape)
+            Tensor.new(data, dtype:dtype, device:device)
+        end
+
+        # initialize new tensor with given another tensor's shape which filled with ones.
+        def ones_like()
+            Tensor.ones(self.shape, dtype:self.dtype, device:self.device)
+        end
+
+        # initialize new tensor with given shape which filled with random values.
+        def self.randn(shape, dtype: RubyZero::Core::DataTypes::Float64, device: RubyZero.device(:cpu))
+            dtype = dtype.get_type_on_calculator(device)
+            data = dtype.new(shape).rand_norm
+            Tensor.new(data, dtype:dtype, device:device)
+        end
+
+        # initialize new tensor with given another tensor's shape which filled with random values.
+        def randn_like()
+            Tensor.randn(self.shape, dtype:self.dtype, device:self.device)
+        end
+
+        # initialize new tensor with given shape which filled with random values.
+        def self.rand(shape, dtype: RubyZero::Core::DataTypes::Float64, device: RubyZero.device(:cpu))
+            dtype = dtype.get_type_on_calculator(device)
+            data = dtype.new(shape).rand
+            Tensor.new(data, dtype:dtype, device:device)
+        end
+
+        # initialize new tensor with given another tensor's shape which filled with random values.
+        def rand_like()
+            Tensor.rand(self.shape, dtype: self.dtype, device:self.device)
+        end
+
+        def self.stack(tensors, axis:0)
+            t = Tensor.new(tensors.map{|t| t.data})
+            return t
+        end
+    end
+
+    module DataTypes
+        class DataType
+            def self.zeros(*args)
+                Tensor.zeros(*args, dtype: self)
+            end
+            def self.ones(*args)
+                Tensor.ones(*args, dtype: self)
+            end
+            def self.randn(*args)
+                Tensor.randn(*args, dtype: self)
+            end
+        end
+    end
+end
+
+module RubyZero
+    FloatTensor = RubyZero::Core::DataTypes::Float32
+    DoubleTensor = RubyZero::Core::DataTypes::Float64
+    IntTensor = RubyZero::Core::DataTypes::Int32
+    LongTensor = RubyZero::Core::DataTypes::Int64
+end

data/lib/rubyzero/core/tensor_operators.rb

@@ -0,0 +1,27 @@
+module RubyZero::Core
+    class Tensor
+        def +(other)
+            RubyZero::Core::Functions::Add.new().call(self, other)
+        end
+        def -(other)
+            RubyZero::Core::Functions::Sub.new().call(self, other)
+        end
+        def *(other)
+            if other.is_a?(Numeric)
+                RubyZero::Core::Functions::MulScalar.new(other).call(self)
+            else
+                RubyZero::Core::Functions::Mul.new().call(self, other)
+            end
+        end
+        def /(other)
+            if other.is_a?(Numeric)
+                RubyZero::Core::Functions::DivScalar.new(other).call(self)
+            else
+                RubyZero::Core::Functions::Div.new().call(self, other)
+            end
+        end
+        def -@
+            RubyZero::Core::Functions::Neg.new().call(self)
+        end
+    end
+end

data/lib/rubyzero/data/data.rb

@@ -0,0 +1,9 @@
+module RubyZero
+    module Data
+        
+    end
+end
+
+require_relative './dataset.rb'
+require_relative './dataloader.rb'
+require_relative './presets/presets.rb'

data/lib/rubyzero/data/dataloader.rb

@@ -0,0 +1,34 @@
+module RubyZero::Data
+    class DataLoader
+        def initialize(dataset, batch_size: 1, shuffle: false)
+            @dataset = dataset
+            @batch_size = batch_size > dataset.size ? dataset.size : batch_size
+            @shuffle = shuffle
+        end
+        include Enumerable
+        def each
+            shuffled_index = nil
+            shuffled_index = (0..(@dataset.length-1)).to_a.shuffle if @shuffle
+            batched_index = shuffled_index.each_slice(@batch_size).to_a
+            batched_index.each do |batch_idx_arr|
+                datas = []
+                batch_idx_arr.each do |idx|
+                    datas << @dataset[idx]
+                end
+                # transpose 2d array
+                args = datas.transpose
+                args = args.map do |arg|
+                    if arg.class == Array
+                        if arg[0].is_a?(RubyZero::Core::Tensor)
+                            next RubyZero::Core::Tensor.stack(arg)
+                        elsif arg[0].is_a?(Array)
+                            next RubyZero::Core::Tensor.new(arg)
+                        end
+                    end
+                    next arg
+                end
+                yield(*args)
+            end
+        end
+    end
+end

data/lib/rubyzero/data/dataset.rb

@@ -0,0 +1,19 @@
+module RubyZero::Data
+    class Dataset
+        def initialize()
+
+        end
+        def getitem(index)
+
+        end
+        def length()
+
+        end
+        def [](index)
+            getitem(index)
+        end
+        def size
+            length()
+        end
+    end
+end

data/lib/rubyzero/data/presets/presets.rb

@@ -0,0 +1,7 @@
+module RubyZero::Data
+    module Presets
+
+    end
+end
+
+require_relative './xor.rb'

data/lib/rubyzero/data/presets/xor.rb

@@ -0,0 +1,24 @@
+module RubyZero::Data::Presets
+    class Xor < RubyZero::Data::Dataset
+        def initialize
+            @inputs = [
+                [0, 0],
+                [0, 1],
+                [1, 0],
+                [1, 1]
+            ]
+            @targets = [
+                [0],
+                [1],
+                [1],
+                [0]
+            ]
+        end
+        def getitem(idx)
+            return @inputs[idx], @targets[idx]
+        end
+        def length
+            return 4
+        end
+    end
+end

data/lib/rubyzero/nn/functional.rb

@@ -0,0 +1,21 @@
+module RubyZero::NN
+    module Functional
+        def self.relu(x)
+            return RubyZero::Core::Functions::ReLU.new().call(x)
+        end
+        def self.log(x)
+            return RubyZero::Core::Functions::Log.new().call(x)
+        end
+        def self.sigmoid(x)
+            return RubyZero::Core::Functions::Sigmoid.new().call(x)
+        end
+        def self.exp(x)
+            return RubyZero::Core::Functions::Exp.new().call(x)
+        end
+    end
+end
+
+
+module RubyZero
+    F = RubyZero::NN::Functional
+end

data/lib/rubyzero/nn/layers/affine.rb

@@ -0,0 +1,21 @@
+module RubyZero::NN::Layers
+    # Affine layer
+    class Affine < Layer
+        # @param input_size [Integer] input size
+        # @param output_size [Integer] output size
+        # @param bias [Boolean] whether to use bias
+        def initialize(input_size, output_size, bias: true)
+            @weight = RubyZero::Float32.randn([input_size, output_size])
+            @bias = RubyZero::Float32.randn([output_size]) if bias
+        end
+        # Forward pass
+        # @param x [RubyZero::Float32] input
+        # @return [RubyZero::Float32] output
+        def forward(x)
+            x = x.dot(@weight)
+            if @bias
+                x += @bias.repeat(x.shape[0], axis: 0)
+            end
+        end
+    end
+end

data/lib/rubyzero/nn/layers/embedding.rb

@@ -0,0 +1,7 @@
+module RubyZero::NN::Layers
+    class Embedding < Affine
+        def initialize(vocab_size, hidden_dim)
+            super(vocab_size, hidden_dim, bias: false)
+        end
+    end
+end

data/lib/rubyzero/nn/layers/layer.rb

@@ -0,0 +1,5 @@
+module RubyZero::NN::Layers
+    class Layer < RubyZero::NN::Model
+
+    end
+end

data/lib/rubyzero/nn/layers/layers.rb

@@ -0,0 +1,44 @@
+module RubyZero::NN
+    module Layers
+        
+    end
+end
+
+
+require_relative './layer.rb'
+require_relative './affine.rb'
+require_relative './relu.rb'
+require_relative './modellist.rb'
+require_relative './modelstack.rb'
+require_relative './multi_layer_perceptron.rb'
+require_relative './embedding.rb'
+
+module RubyZero::NN
+    module LayersInitializeAliases
+        def self.affine(*args)
+            RubyZero::NN::Layers::Affine.new(*args)
+        end
+        def self.relu(*args)
+            RubyZero::NN::Layers::ReLU.new(*args)
+        end
+        def self.model_list(*args)
+            RubyZero::NN::Layers::ModelList.new(*args)
+        end
+        def self.model_stack(*args)
+            RubyZero::NN::Layers::ModelStack.new(*args)
+        end
+        def self.multi_layer_perceptron(*args)
+            RubyZero::NN::Layers::MultiLayerPerceptron.new(*args)
+        end
+        def self.mlp(*args)
+            RubyZero::NN::Layers::MultiLayerPerceptron.new(*args)
+        end
+        def embedding(*args)
+            RubyZero::NN::Layers::Embedding.new(*args)
+        end
+    end
+end
+
+module RubyZero
+    L = RubyZero::NN::LayersInitializeAliases
+end

data/lib/rubyzero/nn/layers/modellist.rb

@@ -0,0 +1,40 @@
+module RubyZero::NN::Layers
+    # The list of models that can be used in the neural network.
+    # This class supports to containing different Model classes.
+    class ModelList < Layer
+        def initialize(list=nil)
+            if list
+                @models = list
+            else
+                @models = []
+            end
+        end
+        def parameters
+            params = []
+            @models.each do |model|
+                params += model.parameters.elements
+            end
+            return RubyZero::NN::Parameters.new(params)
+        end
+        def <<(model)
+            @models << model
+        end
+        def elements
+            return @models
+        end
+        def +(modellist)
+            return ModelList.new(@models + modellist.elements)
+        end
+        def each(&block)
+            @models.each(&block)
+        end
+        def __get_str__(num_indents)
+            indents = "  " * num_indents
+            s = "#{indents}#{self.class.name} #{parameters.size} params\n"
+            @models.each do |child|
+                s += child.__get_str__(num_indents + 1)
+            end
+            return s
+        end
+    end
+end

data/lib/rubyzero/nn/layers/modelstack.rb

@@ -0,0 +1,20 @@
+module RubyZero::NN::Layers
+    # The stack of layers
+    # This class don't supports to containing different Model classes.
+    class ModelStack < ModelList
+        def <<(layer)
+            if @models.size > 0
+                if layer.class != @models.last.class
+                    raise TypeError, "Layer type mismatch"
+                end
+            end
+            super(layer)
+        end
+        def forward(*args)
+            @models.each do |model|
+                args = model.forward(*args)
+            end
+            return args
+        end
+    end
+end

data/lib/rubyzero/nn/layers/multi_layer_perceptron.rb

@@ -0,0 +1,26 @@
+module RubyZero::NN::Layers
+    class MultiLayerPerceptron < ModelStack
+        # @option opts [Integer] :dims The dimensions of the input, hidden and output
+        # 
+        # @note [5, 3] -> 5 inputs, 3 outputs
+        #   [5, 3, 2] -> 5 inputs, 3 hidden, 2 outputs
+        # 
+        def initialize(*dims, activation: RubyZero::NN::Layers::ReLU)
+            super()
+            if dims.length < 2
+                raise ArgumentError, "Layer must have at least 2 dimensions"
+            end
+            @dims = dims
+            @activation_function = activation
+
+            (@dims.size-1).times do |i|
+                dim_i = @dims[i]
+                dim_o = @dims[i+1]
+                @models << RubyZero::NN::Layers::Affine.new(dim_i, dim_o)
+                if i != @dims.size-2
+                    @models << activation.new
+                end
+            end
+        end
+    end
+end

data/lib/rubyzero/nn/layers/relu.rb

@@ -0,0 +1,10 @@
+module RubyZero::NN::Layers
+    # ReLU Layer
+    class ReLU < Layer
+        def initialize()
+        end
+        def forward(x)
+            return RubyZero::Core::Functions::ReLU.new().call(x)
+        end
+    end
+end

data/lib/rubyzero/nn/load.rb

@@ -0,0 +1,5 @@
+module RubyZero::NN
+    def self.load(filename)
+        return RubyZero::NN::Model.load(filename)
+    end
+end

data/lib/rubyzero/nn/losses/loss.rb

@@ -0,0 +1,5 @@
+module RubyZero::NN::Losses
+    class Loss < RubyZero::NN::Model
+
+    end
+end

data/lib/rubyzero/nn/losses/losses.rb

@@ -0,0 +1,8 @@
+module RubyZero::NN
+    module Losses
+
+    end
+end
+
+require_relative './loss.rb'
+require_relative './mse.rb'

data/lib/rubyzero/nn/losses/mse.rb

@@ -0,0 +1,13 @@
+module RubyZero::NN::Losses
+    class MSE < Loss
+        def forward(input, target)
+            err = input - target
+            err = err * err
+            while err.shape.size > 1
+                err = err.mean(axis:0)
+            end
+            return err
+        end
+    end
+    MeanSquaredError = MSE
+end

data/lib/rubyzero/nn/model.rb

@@ -0,0 +1,75 @@
+module RubyZero::NN
+    # Model class
+    class Model
+        def initialize
+        end
+        def forward
+            raise RubyZero::Core::Exceptions::NoInplementError, "#{self.class}.forward() method is not implemented"
+        end
+        def call(*args)
+            return forward(*args)
+        end
+        def parameters
+            param_keys = instance_variables
+            params = []
+            param_keys.each do |key|
+                obj = instance_variable_get(key)
+                if obj.is_a?(RubyZero::Core::Tensor)
+                    obj.requires_grad = true
+                    params << obj
+                elsif obj.is_a?(RubyZero::NN::Parameters)
+                    params += obj.elements
+                else obj.is_a?(RubyZero::NN::Model)
+                    params += obj.parameters.elements
+                end
+            end
+            return Parameters.new(params)
+        end
+        def __get_str__(num_indents)
+            keys = instance_variables
+            children = []
+            keys.each do |key|
+                obj = instance_variable_get(key)
+                if obj.is_a?(RubyZero::NN::Model)
+                    children << obj
+                end
+            end
+            indents = "  " * num_indents
+            s = "#{indents}#{self.class.name} #{parameters.size} params\n"
+            children.each do |child|
+                s += child.__get_str__(num_indents + 1)
+            end
+            return s
+        end
+        def inspect
+            return __get_str__(0)
+        end
+        def train()
+            self.parameters.map do |param|
+                param.requires_grad = true
+            end
+            return self
+        end
+        def eval()
+            def train()
+                self.parameters.map do |param|
+                    param.requires_grad = false
+                end
+                return self
+            end
+        end
+        def save(path)
+            self.parameters.map do |param|
+                param.grad = nil
+            end
+            File.open(path, "wb") do |f|
+                Marshal.dump(self, f)
+            end
+        end
+        def self.load(path)
+            File.open(path, "rb") do |f|
+                return Marshal.load(f)
+            end
+        end
+    end
+end

data/lib/rubyzero/nn/nn.rb

@@ -0,0 +1,11 @@
+module RubyZero
+    module NN
+
+    end
+end
+require_relative './parameters.rb'
+require_relative './model.rb'
+require_relative './layers/layers.rb'
+require_relative './losses/losses.rb'
+require_relative './optimizers/optimizers.rb'
+require_relative './functional.rb'

data/lib/rubyzero/nn/optimizers/momentum.rb

@@ -0,0 +1,22 @@
+module RubyZero::NN::Optimizers
+    class Momentum < Optimizer
+        def initialize(parameters, lr: 0.001, alpha: 0.9)
+            @lr = lr
+            @parameters = parameters
+            @alpha = alpha
+            @velocities = []
+        end
+        def update()
+            @parameters.each_with_index do |tensor, index|
+                if @velocities[index].nil?
+                    @velocities[index] = 0
+                else
+                    vel = @velocities[index] * @alpha
+                    uparam = tensor.grad.data + vel
+                    tensor.data -= uparam * @lr
+                end
+            end
+        end
+        alias :step :update
+    end
+end

data/lib/rubyzero/nn/optimizers/optimizer.rb

@@ -0,0 +1,16 @@
+module RubyZero::NN::Optimizers
+    # base of optimizer classes
+    class Optimizer
+        def initialize(parameters, lr:0.01)
+
+        end
+        def update()
+
+        end
+        def zero_grad()
+            @parameters.each do |parameter|
+                parameter.grad = parameter.zeros_like()
+            end
+        end
+    end
+end

data/lib/rubyzero/nn/optimizers/optimizers.rb

@@ -0,0 +1,9 @@
+module RubyZero::NN
+    module Optimizers
+
+    end
+end
+
+require_relative './optimizer.rb'
+require_relative './sgd.rb'
+require_relative './momentum.rb'

data/lib/rubyzero/nn/optimizers/sgd.rb

@@ -0,0 +1,14 @@
+module RubyZero::NN::Optimizers
+    class SGD < Optimizer
+        def initialize(parameters, lr: 0.01)
+            @lr = lr
+            @parameters = parameters
+        end
+        def update()
+            @parameters.each do |tensor|
+                tensor.data -= tensor.grad.data * @lr
+            end
+        end
+        alias :step :update
+    end
+end