alglib 0.1.1

data/ext/alglib/mlpe.h

@@ -0,0 +1,552 @@
+/*************************************************************************
+Copyright (c) 2007-2008, Sergey Bochkanov (ALGLIB project).
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer listed
+  in this license in the documentation and/or other materials
+  provided with the distribution.
+
+- Neither the name of the copyright holders nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*************************************************************************/
+
+#ifndef _mlpe_h
+#define _mlpe_h
+
+#include "ap.h"
+#include "ialglib.h"
+
+#include "mlpbase.h"
+#include "trinverse.h"
+#include "lbfgs.h"
+#include "cholesky.h"
+#include "spdsolve.h"
+#include "mlptrain.h"
+#include "tsort.h"
+#include "descriptivestatistics.h"
+#include "bdss.h"
+
+
+struct mlpensemble
+{
+    ap::integer_1d_array structinfo;
+    int ensemblesize;
+    int nin;
+    int nout;
+    int wcount;
+    bool issoftmax;
+    bool postprocessing;
+    ap::real_1d_array weights;
+    ap::real_1d_array columnmeans;
+    ap::real_1d_array columnsigmas;
+    int serializedlen;
+    ap::real_1d_array serializedmlp;
+    ap::real_1d_array tmpweights;
+    ap::real_1d_array tmpmeans;
+    ap::real_1d_array tmpsigmas;
+    ap::real_1d_array neurons;
+    ap::real_1d_array dfdnet;
+    ap::real_1d_array y;
+};
+
+
+/*************************************************************************
+Like MLPCreate0, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreate0(int nin, int nout, int ensemblesize, mlpensemble& ensemble);
+
+
+/*************************************************************************
+Like MLPCreate1, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreate1(int nin,
+     int nhid,
+     int nout,
+     int ensemblesize,
+     mlpensemble& ensemble);
+
+
+/*************************************************************************
+Like MLPCreate2, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreate2(int nin,
+     int nhid1,
+     int nhid2,
+     int nout,
+     int ensemblesize,
+     mlpensemble& ensemble);
+
+
+/*************************************************************************
+Like MLPCreateB0, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreateb0(int nin,
+     int nout,
+     double b,
+     double d,
+     int ensemblesize,
+     mlpensemble& ensemble);
+
+
+/*************************************************************************
+Like MLPCreateB1, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreateb1(int nin,
+     int nhid,
+     int nout,
+     double b,
+     double d,
+     int ensemblesize,
+     mlpensemble& ensemble);
+
+
+/*************************************************************************
+Like MLPCreateB2, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreateb2(int nin,
+     int nhid1,
+     int nhid2,
+     int nout,
+     double b,
+     double d,
+     int ensemblesize,
+     mlpensemble& ensemble);
+
+
+/*************************************************************************
+Like MLPCreateR0, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreater0(int nin,
+     int nout,
+     double a,
+     double b,
+     int ensemblesize,
+     mlpensemble& ensemble);
+
+
+/*************************************************************************
+Like MLPCreateR1, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreater1(int nin,
+     int nhid,
+     int nout,
+     double a,
+     double b,
+     int ensemblesize,
+     mlpensemble& ensemble);
+
+
+/*************************************************************************
+Like MLPCreateR2, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreater2(int nin,
+     int nhid1,
+     int nhid2,
+     int nout,
+     double a,
+     double b,
+     int ensemblesize,
+     mlpensemble& ensemble);
+
+
+/*************************************************************************
+Like MLPCreateC0, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreatec0(int nin, int nout, int ensemblesize, mlpensemble& ensemble);
+
+
+/*************************************************************************
+Like MLPCreateC1, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreatec1(int nin,
+     int nhid,
+     int nout,
+     int ensemblesize,
+     mlpensemble& ensemble);
+
+
+/*************************************************************************
+Like MLPCreateC2, but for ensembles.
+
+  -- ALGLIB --
+     Copyright 18.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreatec2(int nin,
+     int nhid1,
+     int nhid2,
+     int nout,
+     int ensemblesize,
+     mlpensemble& ensemble);
+
+
+/*************************************************************************
+Creates ensemble from network. Only network geometry is copied.
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecreatefromnetwork(const multilayerperceptron& network,
+     int ensemblesize,
+     mlpensemble& ensemble);
+
+
+/*************************************************************************
+Copying of MLPEnsemble strucure
+
+INPUT PARAMETERS:
+    Ensemble1 -   original
+
+OUTPUT PARAMETERS:
+    Ensemble2 -   copy
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpecopy(const mlpensemble& ensemble1, mlpensemble& ensemble2);
+
+
+/*************************************************************************
+Serialization of MLPEnsemble strucure
+
+INPUT PARAMETERS:
+    Ensemble-   original
+
+OUTPUT PARAMETERS:
+    RA      -   array of real numbers which stores ensemble,
+                array[0..RLen-1]
+    RLen    -   RA lenght
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpeserialize(mlpensemble& ensemble, ap::real_1d_array& ra, int& rlen);
+
+
+/*************************************************************************
+Unserialization of MLPEnsemble strucure
+
+INPUT PARAMETERS:
+    RA      -   real array which stores ensemble
+
+OUTPUT PARAMETERS:
+    Ensemble-   restored structure
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpeunserialize(const ap::real_1d_array& ra, mlpensemble& ensemble);
+
+
+/*************************************************************************
+Randomization of MLP ensemble
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlperandomize(mlpensemble& ensemble);
+
+
+/*************************************************************************
+Return ensemble properties (number of inputs and outputs).
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpeproperties(const mlpensemble& ensemble, int& nin, int& nout);
+
+
+/*************************************************************************
+Return normalization type (whether ensemble is SOFTMAX-normalized or not).
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+bool mlpeissoftmax(const mlpensemble& ensemble);
+
+
+/*************************************************************************
+Procesing
+
+INPUT PARAMETERS:
+    Ensemble-   neural networks ensemble
+    X       -   input vector,  array[0..NIn-1].
+
+OUTPUT PARAMETERS:
+    Y       -   result. Regression estimate when solving regression  task,
+                vector of posterior probabilities for classification task.
+                Subroutine does not allocate memory for this vector, it is
+                responsibility of a caller to allocate it. Array  must  be
+                at least [0..NOut-1].
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpeprocess(mlpensemble& ensemble,
+     const ap::real_1d_array& x,
+     ap::real_1d_array& y);
+
+
+/*************************************************************************
+Relative classification error on the test set
+
+INPUT PARAMETERS:
+    Ensemble-   ensemble
+    XY      -   test set
+    NPoints -   test set size
+
+RESULT:
+    percent of incorrectly classified cases.
+    Works both for classifier betwork and for regression networks which
+are used as classifiers.
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+double mlperelclserror(mlpensemble& ensemble,
+     const ap::real_2d_array& xy,
+     int npoints);
+
+
+/*************************************************************************
+Average cross-entropy (in bits per element) on the test set
+
+INPUT PARAMETERS:
+    Ensemble-   ensemble
+    XY      -   test set
+    NPoints -   test set size
+
+RESULT:
+    CrossEntropy/(NPoints*LN(2)).
+    Zero if ensemble solves regression task.
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+double mlpeavgce(mlpensemble& ensemble,
+     const ap::real_2d_array& xy,
+     int npoints);
+
+
+/*************************************************************************
+RMS error on the test set
+
+INPUT PARAMETERS:
+    Ensemble-   ensemble
+    XY      -   test set
+    NPoints -   test set size
+
+RESULT:
+    root mean square error.
+    Its meaning for regression task is obvious. As for classification task
+RMS error means error when estimating posterior probabilities.
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+double mlpermserror(mlpensemble& ensemble,
+     const ap::real_2d_array& xy,
+     int npoints);
+
+
+/*************************************************************************
+Average error on the test set
+
+INPUT PARAMETERS:
+    Ensemble-   ensemble
+    XY      -   test set
+    NPoints -   test set size
+
+RESULT:
+    Its meaning for regression task is obvious. As for classification task
+it means average error when estimating posterior probabilities.
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+double mlpeavgerror(mlpensemble& ensemble,
+     const ap::real_2d_array& xy,
+     int npoints);
+
+
+/*************************************************************************
+Average relative error on the test set
+
+INPUT PARAMETERS:
+    Ensemble-   ensemble
+    XY      -   test set
+    NPoints -   test set size
+
+RESULT:
+    Its meaning for regression task is obvious. As for classification task
+it means average relative error when estimating posterior probabilities.
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+double mlpeavgrelerror(mlpensemble& ensemble,
+     const ap::real_2d_array& xy,
+     int npoints);
+
+
+/*************************************************************************
+Training neural networks ensemble using  bootstrap  aggregating (bagging).
+Modified Levenberg-Marquardt algorithm is used as base training method.
+
+INPUT PARAMETERS:
+    Ensemble    -   model with initialized geometry
+    XY          -   training set
+    NPoints     -   training set size
+    Decay       -   weight decay coefficient, >=0.001
+    Restarts    -   restarts, >0.
+
+OUTPUT PARAMETERS:
+    Ensemble    -   trained model
+    Info        -   return code:
+                    * -2, if there is a point with class number
+                          outside of [0..NClasses-1].
+                    * -1, if incorrect parameters was passed
+                          (NPoints<0, Restarts<1).
+                    *  2, if task has been solved.
+    Rep         -   training report.
+    OOBErrors   -   out-of-bag generalization error estimate
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpebagginglm(mlpensemble& ensemble,
+     const ap::real_2d_array& xy,
+     int npoints,
+     double decay,
+     int restarts,
+     int& info,
+     mlpreport& rep,
+     mlpcvreport& ooberrors);
+
+
+/*************************************************************************
+Training neural networks ensemble using  bootstrap  aggregating (bagging).
+L-BFGS algorithm is used as base training method.
+
+INPUT PARAMETERS:
+    Ensemble    -   model with initialized geometry
+    XY          -   training set
+    NPoints     -   training set size
+    Decay       -   weight decay coefficient, >=0.001
+    Restarts    -   restarts, >0.
+    WStep       -   stopping criterion, same as in MLPTrainLBFGS
+    MaxIts      -   stopping criterion, same as in MLPTrainLBFGS
+
+OUTPUT PARAMETERS:
+    Ensemble    -   trained model
+    Info        -   return code:
+                    * -8, if both WStep=0 and MaxIts=0
+                    * -2, if there is a point with class number
+                          outside of [0..NClasses-1].
+                    * -1, if incorrect parameters was passed
+                          (NPoints<0, Restarts<1).
+                    *  2, if task has been solved.
+    Rep         -   training report.
+    OOBErrors   -   out-of-bag generalization error estimate
+
+  -- ALGLIB --
+     Copyright 17.02.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpebagginglbfgs(mlpensemble& ensemble,
+     const ap::real_2d_array& xy,
+     int npoints,
+     double decay,
+     int restarts,
+     double wstep,
+     int maxits,
+     int& info,
+     mlpreport& rep,
+     mlpcvreport& ooberrors);
+
+
+/*************************************************************************
+Training neural networks ensemble using early stopping.
+
+INPUT PARAMETERS:
+    Ensemble    -   model with initialized geometry
+    XY          -   training set
+    NPoints     -   training set size
+    Decay       -   weight decay coefficient, >=0.001
+    Restarts    -   restarts, >0.
+
+OUTPUT PARAMETERS:
+    Ensemble    -   trained model
+    Info        -   return code:
+                    * -2, if there is a point with class number
+                          outside of [0..NClasses-1].
+                    * -1, if incorrect parameters was passed
+                          (NPoints<0, Restarts<1).
+                    *  2, if task has been solved.
+    Rep         -   training report.
+    OOBErrors   -   out-of-bag generalization error estimate
+
+  -- ALGLIB --
+     Copyright 10.03.2009 by Bochkanov Sergey
+*************************************************************************/
+void mlpetraines(mlpensemble& ensemble,
+     const ap::real_2d_array& xy,
+     int npoints,
+     double decay,
+     int restarts,
+     int& info,
+     mlpreport& rep);
+
+
+#endif