rumale-linear_model 0.24.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3a7999fbdb27dc6ed43710083da0cecf2336bfb08277fa4240be7b56c2603c9f
+  data.tar.gz: 6bc61b3d80fe71c1d7ed806c810a0eb454244033506ab2078ccd0987e7891455
+SHA512:
+  metadata.gz: 4c73e2b03dfb0f14c94b880769103bbb05f77de7ac08f8b9af2dddc0ab6bcada758c0c7c51353a79e052e051999b4afc7610012caf4dbb7c4edc505152fdf1d6
+  data.tar.gz: 91ce194539b8abc95fb3ec7335bd4843aea126774ebb566eb9953955f775427afd032f9d02894c8507337d20f03baafa39903eb9dd9f9157ccf6a8d22028fe8b

data/LICENSE.txt

@@ -0,0 +1,27 @@
+Copyright (c) 2022 Atsushi Tatsuma
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,34 @@
+# Rumale::LinearModel
+
+[![Gem Version](https://badge.fury.io/rb/rumale-linear_model.svg)](https://badge.fury.io/rb/rumale-linear_model)
+[![BSD 3-Clause License](https://img.shields.io/badge/License-BSD%203--Clause-orange.svg)](https://github.com/yoshoku/rumale/blob/main/rumale-linear_model/LICENSE.txt)
+[![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://yoshoku.github.io/rumale/doc/Rumale/LinearModel.html)
+
+Rumale is a machine learning library in Ruby.
+Rumale::LinearModel provides linear model algorithms,
+such as Logistic Regression, Support Vector Machine, Lasso, and Ridge Regression
+with Rumale interface.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rumale-linear_model'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install rumale-linear_model
+
+## Documentation
+
+- [Rumale API Documentation - LinearModel](https://yoshoku.github.io/rumale/doc/Rumale/LinearModel.html)
+
+## License
+
+The gem is available as open source under the terms of the [BSD-3-Clause License](https://opensource.org/licenses/BSD-3-Clause).

data/lib/rumale/linear_model/base_sgd.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+require 'rumale/base/estimator'
+
+module Rumale
+  module LinearModel
+    # @!visibility private
+    # This module consists of the classes that implement penalty (regularization) term.
+    module Penalty
+      # @!visibility private
+      # L2Penalty is a class that applies L2 penalty to weight vector of linear model.
+      # This class is used internally.
+      class L2Penalty
+        # @!visibility private
+        def initialize(reg_param:)
+          @reg_param = reg_param
+        end
+
+        # @!visibility private
+        def call(weight, lr)
+          weight - @reg_param * lr * weight
+        end
+      end
+
+      # @!visibility private
+      # L1Penalty is a class that applies L1 penalty to weight vector of linear model.
+      # This class is used internally.
+      class L1Penalty
+        # @!visibility private
+        def initialize(reg_param:)
+          @u = 0.0
+          @reg_param = reg_param
+        end
+
+        # @!visibility private
+        def call(weight, lr)
+          @q_vec ||= Numo::DFloat.zeros(weight.shape[0])
+          @u += @reg_param * lr
+          z = weight.dup
+          gt = weight.gt(0)
+          lt = weight.lt(0)
+          weight[gt] = Numo::DFloat.maximum(0.0, weight[gt] - (@u + @q_vec[gt])) if gt.count.positive?
+          weight[lt] = Numo::DFloat.minimum(0.0, weight[lt] + (@u - @q_vec[lt])) if lt.count.positive?
+          @q_vec += weight - z
+          weight
+        end
+      end
+    end
+
+    # @!visibility private
+    # This module consists of the class that implements stochastic gradient descent (SGD) optimizer.
+    module Optimizer
+      # @!visibility private
+      # SGD is a class that implements SGD optimizer.
+      # This class is used internally.
+      class SGD
+        # @!visibility private
+        # Create a new SGD optimizer.
+        # @param learning_rate [Float] The initial value of learning rate.
+        # @param momentum [Float] The initial value of momentum.
+        # @param decay [Float] The smooting parameter.
+        def initialize(learning_rate: 0.01, momentum: 0.0, decay: 0.0)
+          @learning_rate = learning_rate
+          @momentum = momentum
+          @decay = decay
+          @iter = 0
+        end
+
+        # @!visibility private
+        def current_learning_rate
+          @learning_rate / (1.0 + @decay * @iter)
+        end
+
+        # @!visibility private
+        def call(weight, gradient)
+          @update ||= Numo::DFloat.zeros(weight.shape[0])
+          @update = @momentum * @update - current_learning_rate * gradient
+          @iter += 1
+          weight + @update
+        end
+      end
+    end
+
+    # @!visibility private
+    # This module consists of the classes that implement loss function for linear model.
+    module Loss
+      # @!visibility private
+      # MeanSquaredError is a class that calculates mean squared error for linear regression model.
+      class MeanSquaredError
+        # @!visibility private
+        def loss(out, y)
+          ((out - y)**2).sum.fdiv(y.shape[0])
+        end
+
+        # @!visibility private
+        def dloss(out, y)
+          2.fdiv(y.shape[0]) * (out - y)
+        end
+      end
+
+      # @!visibility private
+      # LogLoss is a class that calculates logistic loss for logistic regression.
+      class LogLoss
+        # @!visibility private
+        def loss(out, y)
+          Numo::NMath.log(1 + Numo::NMath.exp(-y * out)).sum.fdiv(y.shape[0])
+        end
+
+        # @!visibility private
+        def dloss(out, y)
+          y / (1 + Numo::NMath.exp(-y * out)) - y
+        end
+      end
+
+      # @!visibility private
+      # HingeLoss is a class that calculates hinge loss for support vector classifier.
+      class HingeLoss
+        # @!visibility private
+        def loss(out, y)
+          out.class.maximum(0.0, 1 - y * out).sum.fdiv(y.shape[0])
+        end
+
+        # @!visibility private
+        def dloss(out, y)
+          tids = (y * out).lt(1)
+          d = Numo::DFloat.zeros(y.shape[0])
+          d[tids] = -y[tids] if tids.count.positive?
+          d
+        end
+      end
+
+      # @!visibility private
+      # EpsilonInsensitive is a class that calculates epsilon insensitive for support vector regressor.
+      class EpsilonInsensitive
+        # @!visibility private
+        def initialize(epsilon: 0.1)
+          @epsilon = epsilon
+        end
+
+        # @!visibility private
+        def loss(out, y)
+          out.class.maximum(0.0, (y - out).abs - @epsilon).sum.fdiv(y.shape[0])
+        end
+
+        # @!visibility private
+        def dloss(out, y)
+          d = Numo::DFloat.zeros(y.shape[0])
+          tids = (out - y).gt(@epsilon)
+          d[tids] = 1 if tids.count.positive?
+          tids = (y - out).gt(@epsilon)
+          d[tids] = -1 if tids.count.positive?
+          d
+        end
+      end
+    end
+
+    # BaseSGD is an abstract class for implementation of linear model with mini-batch stochastic gradient descent (SGD) optimization.
+    # This class is used internally.
+    class BaseSGD < ::Rumale::Base::Estimator
+      # Create an initial linear model.
+      def initialize
+        super()
+        @params = {
+          learning_rate: 0.01,
+          decay: nil,
+          momentum: 0.0,
+          bias_scale: 1.0,
+          fit_bias: true,
+          reg_param: 0.0,
+          l1_ratio: 0.0,
+          max_iter: 1000,
+          batch_size: 50,
+          tol: 0.0001,
+          verbose: false
+        }
+      end
+
+      private
+
+      L2_PENALTY = 'l2'
+      L1_PENALTY = 'l1'
+      ELASTICNET_PENALTY = 'elasticnet'
+
+      private_constant :L2_PENALTY, :L1_PENALTY, :ELASTICNET_PENALTY
+
+      def partial_fit(x, y)
+        class_name = self.class.to_s.split('::').last if @params[:verbose]
+        narr = x.class
+        # Expand feature vectors for bias term.
+        x = expand_feature(x) if fit_bias?
+        # Initialize some variables.
+        sub_rng = @rng.dup
+        n_samples, n_features = x.shape
+        weight = Numo::DFloat.zeros(n_features)
+        optimizer = ::Rumale::LinearModel::Optimizer::SGD.new(
+          learning_rate: @params[:learning_rate], momentum: @params[:momentum], decay: @params[:decay]
+        )
+        l2_penalty = ::Rumale::LinearModel::Penalty::L2Penalty.new(reg_param: l2_reg_param) if apply_l2_penalty?
+        l1_penalty = ::Rumale::LinearModel::Penalty::L1Penalty.new(reg_param: l1_reg_param) if apply_l1_penalty?
+        # Optimization.
+        @params[:max_iter].times do |t|
+          sample_ids = Array(0...n_samples)
+          sample_ids.shuffle!(random: sub_rng)
+          until (subset_ids = sample_ids.shift(@params[:batch_size])).empty?
+            # sampling
+            sub_x = x[subset_ids, true]
+            sub_y = y[subset_ids]
+            # calculate gradient
+            dloss = @loss_func.dloss(sub_x.dot(weight), sub_y)
+            dloss = narr.minimum(1e12, narr.maximum(-1e12, dloss))
+            gradient = dloss.dot(sub_x)
+            # update weight
+            lr = optimizer.current_learning_rate
+            weight = optimizer.call(weight, gradient)
+            # l2 regularization
+            weight = l2_penalty.call(weight, lr) if apply_l2_penalty?
+            # l1 regularization
+            weight = l1_penalty.call(weight, lr) if apply_l1_penalty?
+          end
+          loss = @loss_func.loss(x.dot(weight), y)
+          puts "[#{class_name}] Loss after #{t + 1} epochs: #{loss}" if @params[:verbose]
+          break if loss < @params[:tol]
+        end
+        split_weight(weight)
+      end
+
+      def expand_feature(x)
+        n_samples = x.shape[0]
+        Numo::NArray.hstack([x, Numo::DFloat.ones([n_samples, 1]) * @params[:bias_scale]])
+      end
+
+      def split_weight(weight)
+        if fit_bias?
+          [weight[0...-1].dup, weight[-1]]
+        else
+          [weight, 0.0]
+        end
+      end
+
+      def fit_bias?
+        @params[:fit_bias] == true
+      end
+
+      def apply_l2_penalty?
+        @penalty_type == L2_PENALTY || @penalty_type == ELASTICNET_PENALTY
+      end
+
+      def apply_l1_penalty?
+        @penalty_type == L1_PENALTY || @penalty_type == ELASTICNET_PENALTY
+      end
+
+      def l2_reg_param
+        case @penalty_type
+        when ELASTICNET_PENALTY
+          @params[:reg_param] * (1.0 - @params[:l1_ratio])
+        when L2_PENALTY
+          @params[:reg_param]
+        else
+          0.0
+        end
+      end
+
+      def l1_reg_param
+        case @penalty_type
+        when ELASTICNET_PENALTY
+          @params[:reg_param] * @params[:l1_ratio]
+        when L1_PENALTY
+          @params[:reg_param]
+        else
+          0.0
+        end
+      end
+    end
+  end
+end

data/lib/rumale/linear_model/elastic_net.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require 'rumale/base/regressor'
+require 'rumale/validation'
+require 'rumale/linear_model/base_sgd'
+
+module Rumale
+  module LinearModel
+    # ElasticNet is a class that implements Elastic-net Regression
+    # with stochastic gradient descent (SGD) optimization.
+    #
+    # @example
+    #   require 'rumale/linear_model/elastic_net'
+    #
+    #   estimator =
+    #     Rumale::LinearModel::ElasticNet.new(reg_param: 0.1, l1_ratio: 0.5, max_iter: 1000, batch_size: 50, random_seed: 1)
+    #   estimator.fit(training_samples, traininig_values)
+    #   results = estimator.predict(testing_samples)
+    #
+    # *Reference*
+    # - Shalev-Shwartz, S., and Singer, Y., "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
+    # - Tsuruoka, Y., Tsujii, J., and Ananiadou, S., "Stochastic Gradient Descent Training for L1-regularized Log-linear Models with Cumulative Penalty," Proc. ACL'09, pp. 477--485, 2009.
+    # - Bottou, L., "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
+    class ElasticNet < BaseSGD
+      include ::Rumale::Base::Regressor
+
+      # Return the weight vector.
+      # @return [Numo::DFloat] (shape: [n_outputs, n_features])
+      attr_reader :weight_vec
+
+      # Return the bias term (a.k.a. intercept).
+      # @return [Numo::DFloat] (shape: [n_outputs])
+      attr_reader :bias_term
+
+      # Return the random generator for random sampling.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new Elastic-net regressor.
+      #
+      # @param learning_rate [Float] The initial value of learning rate.
+      #   The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
+      # @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
+      #   If nil is given, the decay sets to 'reg_param * learning_rate'.
+      # @param momentum [Float] The momentum factor.
+      # @param reg_param [Float] The regularization parameter.
+      # @param l1_ratio [Float] The elastic-net mixing parameter.
+      #   If l1_ratio = 1, the regularization is similar to Lasso.
+      #   If l1_ratio = 0, the regularization is similar to Ridge.
+      #   If 0 < l1_ratio < 1, the regularization is a combination of L1 and L2.
+      # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
+      # @param bias_scale [Float] The scale of the bias term.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
+      # @param batch_size [Integer] The size of the mini batches.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
+      # @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
+      #   If nil is given, the method does not execute in parallel.
+      #   If zero or less is given, it becomes equal to the number of processors.
+      #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
+                     reg_param: 1.0, l1_ratio: 0.5, fit_bias: true, bias_scale: 1.0,
+                     max_iter: 1000, batch_size: 50, tol: 1e-4,
+                     n_jobs: nil, verbose: false, random_seed: nil)
+        super()
+        @params.merge!(method(:initialize).parameters.to_h { |_t, arg| [arg, binding.local_variable_get(arg)] })
+        @params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
+        @params[:random_seed] ||= srand
+        @rng = Random.new(@params[:random_seed])
+        @penalty_type = ELASTICNET_PENALTY
+        @loss_func = ::Rumale::LinearModel::Loss::MeanSquaredError.new
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values to be used for fitting the model.
+      # @return [ElasticNet] The learned regressor itself.
+      def fit(x, y)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+        y = ::Rumale::Validation.check_convert_target_value_array(y)
+        ::Rumale::Validation.check_sample_size(x, y)
+
+        n_outputs = y.shape[1].nil? ? 1 : y.shape[1]
+        n_features = x.shape[1]
+
+        if n_outputs > 1
+          @weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
+          @bias_term = Numo::DFloat.zeros(n_outputs)
+          if enable_parallel?
+            models = parallel_map(n_outputs) { |n| partial_fit(x, y[true, n]) }
+            n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = models[n] }
+          else
+            n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
+          end
+        else
+          @weight_vec, @bias_term = partial_fit(x, y)
+        end
+        self
+      end
+
+      # Predict values for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
+      # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted values per sample.
+      def predict(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        x.dot(@weight_vec.transpose) + @bias_term
+      end
+    end
+  end
+end

data/lib/rumale/linear_model/lasso.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'rumale/base/regressor'
+require 'rumale/validation'
+require 'rumale/linear_model/base_sgd'
+
+module Rumale
+  module LinearModel
+    # Lasso is a class that implements Lasso Regression
+    # with stochastic gradient descent (SGD) optimization.
+    #
+    # @example
+    #   require 'rumale/linear_model/lasso'
+    #
+    #   estimator =
+    #     Rumale::LinearModel::Lasso.new(reg_param: 0.1, max_iter: 1000, batch_size: 20, random_seed: 1)
+    #   estimator.fit(training_samples, traininig_values)
+    #   results = estimator.predict(testing_samples)
+    #
+    # *Reference*
+    # - Shalev-Shwartz, S., and Singer, Y., "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
+    # - Tsuruoka, Y., Tsujii, J., and Ananiadou, S., "Stochastic Gradient Descent Training for L1-regularized Log-linear Models with Cumulative Penalty," Proc. ACL'09, pp. 477--485, 2009.
+    # - Bottou, L., "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
+    class Lasso < BaseSGD
+      include ::Rumale::Base::Regressor
+
+      # Return the weight vector.
+      # @return [Numo::DFloat] (shape: [n_outputs, n_features])
+      attr_reader :weight_vec
+
+      # Return the bias term (a.k.a. intercept).
+      # @return [Numo::DFloat] (shape: [n_outputs])
+      attr_reader :bias_term
+
+      # Return the random generator for random sampling.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new Lasso regressor.
+      #
+      # @param learning_rate [Float] The initial value of learning rate.
+      #   The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
+      # @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
+      #   If nil is given, the decay sets to 'reg_param * learning_rate'.
+      # @param momentum [Float] The momentum factor.
+      # @param reg_param [Float] The regularization parameter.
+      # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
+      # @param bias_scale [Float] The scale of the bias term.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
+      # @param batch_size [Integer] The size of the mini batches.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
+      # @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
+      #   If nil is given, the method does not execute in parallel.
+      #   If zero or less is given, it becomes equal to the number of processors.
+      #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
+                     reg_param: 1.0, fit_bias: true, bias_scale: 1.0,
+                     max_iter: 1000, batch_size: 50, tol: 1e-4,
+                     n_jobs: nil, verbose: false, random_seed: nil)
+        super()
+        @params.merge!(method(:initialize).parameters.to_h { |_t, arg| [arg, binding.local_variable_get(arg)] })
+        @params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
+        @params[:random_seed] ||= srand
+        @rng = Random.new(@params[:random_seed])
+        @penalty_type = L1_PENALTY
+        @loss_func = ::Rumale::LinearModel::Loss::MeanSquaredError.new
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values to be used for fitting the model.
+      # @return [Lasso] The learned regressor itself.
+      def fit(x, y)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+        y = ::Rumale::Validation.check_convert_target_value_array(y)
+        ::Rumale::Validation.check_sample_size(x, y)
+
+        n_outputs = y.shape[1].nil? ? 1 : y.shape[1]
+        n_features = x.shape[1]
+
+        if n_outputs > 1
+          @weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
+          @bias_term = Numo::DFloat.zeros(n_outputs)
+          if enable_parallel?
+            models = parallel_map(n_outputs) { |n| partial_fit(x, y[true, n]) }
+            n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = models[n] }
+          else
+            n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
+          end
+        else
+          @weight_vec, @bias_term = partial_fit(x, y)
+        end
+        self
+      end
+
+      # Predict values for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
+      # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted values per sample.
+      def predict(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        x.dot(@weight_vec.transpose) + @bias_term
+      end
+    end
+  end
+end