alglib 0.1.1

data/ext/alglib/lbfgs.h

@@ -0,0 +1,218 @@
+/*************************************************************************
+Copyright (c) 2007-2008, Sergey Bochkanov (ALGLIB project).
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer listed
+  in this license in the documentation and/or other materials
+  provided with the distribution.
+
+- Neither the name of the copyright holders nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*************************************************************************/
+
+#ifndef _lbfgs_h
+#define _lbfgs_h
+
+#include "ap.h"
+#include "ialglib.h"
+
+struct lbfgsstate
+{
+    int n;
+    int m;
+    double epsg;
+    double epsf;
+    double epsx;
+    int maxits;
+    int flags;
+    int nfev;
+    int mcstage;
+    int k;
+    int q;
+    int p;
+    ap::real_1d_array rho;
+    ap::real_2d_array y;
+    ap::real_2d_array s;
+    ap::real_1d_array theta;
+    ap::real_1d_array d;
+    double stp;
+    ap::real_1d_array work;
+    double fold;
+    double gammak;
+    ap::real_1d_array x;
+    double f;
+    ap::real_1d_array g;
+    bool xupdated;
+    ap::rcommstate rstate;
+    int repiterationscount;
+    int repnfev;
+    int repterminationtype;
+    bool brackt;
+    bool stage1;
+    int infoc;
+    double dg;
+    double dgm;
+    double dginit;
+    double dgtest;
+    double dgx;
+    double dgxm;
+    double dgy;
+    double dgym;
+    double finit;
+    double ftest1;
+    double fm;
+    double fx;
+    double fxm;
+    double fy;
+    double fym;
+    double stx;
+    double sty;
+    double stmin;
+    double stmax;
+    double width;
+    double width1;
+    double xtrapf;
+};
+struct lbfgsreport
+{
+    int iterationscount;
+    int nfev;
+    int terminationtype;
+};
+
+
+/*************************************************************************
+        LIMITED MEMORY BFGS METHOD FOR LARGE SCALE OPTIMIZATION
+
+The subroutine minimizes function F(x) of N arguments by  using  a  quasi-
+Newton method (LBFGS scheme) which is optimized to use  a  minimum  amount
+of memory.
+
+The subroutine generates the approximation of an inverse Hessian matrix by
+using information about the last M steps of the algorithm  (instead of N).
+It lessens a required amount of memory from a value  of  order  N^2  to  a
+value of order 2*N*M.
+
+Input parameters:
+    N   -   problem dimension. N>0
+    M   -   number of corrections in the BFGS scheme of Hessian
+            approximation update. Recommended value:  3<=M<=7. The smaller
+            value causes worse convergence, the bigger will  not  cause  a
+            considerably better convergence, but will cause a fall in  the
+            performance. M<=N.
+    X   -   initial solution approximation, array[0..N-1].
+    EpsG -  positive number which  defines  a  precision  of  search.  The
+            subroutine finishes its work if the condition ||G|| < EpsG  is
+            satisfied, where ||.|| means Euclidian norm, G - gradient, X -
+            current approximation.
+    EpsF -  positive number which  defines  a  precision  of  search.  The
+            subroutine finishes its work if on iteration  number  k+1  the
+            condition |F(k+1)-F(k)| <= EpsF*max{|F(k)|, |F(k+1)|, 1}    is
+            satisfied.
+    EpsX -  positive number which  defines  a  precision  of  search.  The
+            subroutine finishes its work if on iteration number k+1    the
+            condition |X(k+1)-X(k)| <= EpsX is fulfilled.
+    MaxIts- maximum number of iterations. If MaxIts=0, the number of
+            iterations is unlimited.
+    Flags - additional settings:
+            * Flags = 0     means no additional settings
+            * Flags = 1     "do not allocate memory". used when solving
+                            a many subsequent tasks with  same N/M  values.
+                            First  call MUST  be without this flag bit set,
+                            subsequent calls of MinLBFGS with same LBFGSState
+                            structure can set Flags to 1.
+
+Output parameters:
+    State - structure used for reverse communication.
+    
+See also MinLBFGSIteration, MinLBFGSResults
+
+  -- ALGLIB --
+     Copyright 14.11.2007 by Bochkanov Sergey
+*************************************************************************/
+void minlbfgs(const int& n,
+     const int& m,
+     ap::real_1d_array& x,
+     const double& epsg,
+     const double& epsf,
+     const double& epsx,
+     const int& maxits,
+     int flags,
+     lbfgsstate& state);
+
+
+/*************************************************************************
+One L-BFGS iteration
+
+Called after initialization with MinLBFGS.
+See HTML documentation for examples.
+
+Input parameters:
+    State   -   structure which stores algorithm state between calls and
+                which is used for reverse communication. Must be initialized
+                with MinLBFGS.
+
+If suborutine returned False, iterative proces has converged.
+
+If subroutine returned True, caller should calculate function value
+State.F an gradient State.G[0..N-1] at State.X[0..N-1] and call
+MinLBFGSIteration again.
+
+  -- ALGLIB --
+     Copyright 20.04.2009 by Bochkanov Sergey
+*************************************************************************/
+bool minlbfgsiteration(lbfgsstate& state);
+
+
+/*************************************************************************
+L-BFGS algorithm results
+
+Called after MinLBFGSIteration returned False.
+
+Input parameters:
+    State   -   algorithm state (used by MinLBFGSIteration).
+
+Output parameters:
+    X       -   array[0..N-1], solution
+    Rep     -   optimization report:
+                * Rep.TerminationType completetion code:
+                    * -2    rounding errors prevent further improvement.
+                            X contains best point found.
+                    * -1    incorrect parameters were specified
+                    *  1    relative function improvement is no more than
+                            EpsF.
+                    *  2    relative step is no more than EpsX.
+                    *  4    gradient norm is no more than EpsG
+                    *  5    MaxIts steps was taken
+                * Rep.IterationsCount contains iterations count
+                * NFEV countains number of function calculations
+
+  -- ALGLIB --
+     Copyright 14.11.2007 by Bochkanov Sergey
+*************************************************************************/
+void minlbfgsresults(const lbfgsstate& state,
+     ap::real_1d_array& x,
+     lbfgsreport& rep);
+
+
+#endif

data/ext/alglib/lda.cpp

@@ -0,0 +1,434 @@
+/*************************************************************************
+Copyright (c) 2008, Sergey Bochkanov (ALGLIB project).
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer listed
+  in this license in the documentation and/or other materials
+  provided with the distribution.
+
+- Neither the name of the copyright holders nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*************************************************************************/
+
+#include <stdafx.h>
+#include "lda.h"
+
+/*************************************************************************
+Multiclass Fisher LDA
+
+Subroutine finds coefficients of linear combination which optimally separates
+training set on classes.
+
+INPUT PARAMETERS:
+    XY          -   training set, array[0..NPoints-1,0..NVars].
+                    First NVars columns store values of independent
+                    variables, next column stores number of class (from 0
+                    to NClasses-1) which dataset element belongs to. Fractional
+                    values are rounded to nearest integer.
+    NPoints     -   training set size, NPoints>=0
+    NVars       -   number of independent variables, NVars>=1
+    NClasses    -   number of classes, NClasses>=2
+
+
+OUTPUT PARAMETERS:
+    Info        -   return code:
+                    * -4, if internal EVD subroutine hasn't converged
+                    * -2, if there is a point with class number
+                          outside of [0..NClasses-1].
+                    * -1, if incorrect parameters was passed (NPoints<0,
+                          NVars<1, NClasses<2)
+                    *  1, if task has been solved
+                    *  2, if there was a multicollinearity in training set,
+                          but task has been solved.
+    W           -   linear combination coefficients, array[0..NVars-1]
+
+  -- ALGLIB --
+     Copyright 31.05.2008 by Bochkanov Sergey
+*************************************************************************/
+void fisherlda(const ap::real_2d_array& xy,
+     int npoints,
+     int nvars,
+     int nclasses,
+     int& info,
+     ap::real_1d_array& w)
+{
+    ap::real_2d_array w2;
+
+    fisherldan(xy, npoints, nvars, nclasses, info, w2);
+    if( info>0 )
+    {
+        w.setbounds(0, nvars-1);
+        ap::vmove(w.getvector(0, nvars-1), w2.getcolumn(0, 0, nvars-1));
+    }
+}
+
+
+/*************************************************************************
+N-dimensional multiclass Fisher LDA
+
+Subroutine finds coefficients of linear combinations which optimally separates
+training set on classes. It returns N-dimensional basis whose vector are sorted
+by quality of training set separation (in descending order).
+
+INPUT PARAMETERS:
+    XY          -   training set, array[0..NPoints-1,0..NVars].
+                    First NVars columns store values of independent
+                    variables, next column stores number of class (from 0
+                    to NClasses-1) which dataset element belongs to. Fractional
+                    values are rounded to nearest integer.
+    NPoints     -   training set size, NPoints>=0
+    NVars       -   number of independent variables, NVars>=1
+    NClasses    -   number of classes, NClasses>=2
+
+
+OUTPUT PARAMETERS:
+    Info        -   return code:
+                    * -4, if internal EVD subroutine hasn't converged
+                    * -2, if there is a point with class number
+                          outside of [0..NClasses-1].
+                    * -1, if incorrect parameters was passed (NPoints<0,
+                          NVars<1, NClasses<2)
+                    *  1, if task has been solved
+                    *  2, if there was a multicollinearity in training set,
+                          but task has been solved.
+    W           -   basis, array[0..NVars-1,0..NVars-1]
+                    columns of matrix stores basis vectors, sorted by
+                    quality of training set separation (in descending order)
+
+  -- ALGLIB --
+     Copyright 31.05.2008 by Bochkanov Sergey
+*************************************************************************/
+void fisherldan(const ap::real_2d_array& xy,
+     int npoints,
+     int nvars,
+     int nclasses,
+     int& info,
+     ap::real_2d_array& w)
+{
+    int i;
+    int j;
+    int k;
+    int m;
+    double v;
+    ap::integer_1d_array c;
+    ap::real_1d_array mu;
+    ap::real_2d_array muc;
+    ap::integer_1d_array nc;
+    ap::real_2d_array sw;
+    ap::real_2d_array st;
+    ap::real_2d_array z;
+    ap::real_2d_array z2;
+    ap::real_2d_array tm;
+    ap::real_2d_array sbroot;
+    ap::real_2d_array a;
+    ap::real_2d_array xyproj;
+    ap::real_2d_array wproj;
+    ap::real_1d_array tf;
+    ap::real_1d_array d;
+    ap::real_1d_array d2;
+    ap::real_1d_array work;
+
+    
+    //
+    // Test data
+    //
+    if( npoints<0||nvars<1||nclasses<2 )
+    {
+        info = -1;
+        return;
+    }
+    for(i = 0; i <= npoints-1; i++)
+    {
+        if( ap::round(xy(i,nvars))<0||ap::round(xy(i,nvars))>=nclasses )
+        {
+            info = -2;
+            return;
+        }
+    }
+    info = 1;
+    
+    //
+    // Special case: NPoints<=1
+    // Degenerate task.
+    //
+    if( npoints<=1 )
+    {
+        info = 2;
+        w.setbounds(0, nvars-1, 0, nvars-1);
+        for(i = 0; i <= nvars-1; i++)
+        {
+            for(j = 0; j <= nvars-1; j++)
+            {
+                if( i==j )
+                {
+                    w(i,j) = 1;
+                }
+                else
+                {
+                    w(i,j) = 0;
+                }
+            }
+        }
+        return;
+    }
+    
+    //
+    // Prepare temporaries
+    //
+    tf.setbounds(0, nvars-1);
+    work.setbounds(1, ap::maxint(nvars, npoints));
+    
+    //
+    // Convert class labels from reals to integers (just for convenience)
+    //
+    c.setbounds(0, npoints-1);
+    for(i = 0; i <= npoints-1; i++)
+    {
+        c(i) = ap::round(xy(i,nvars));
+    }
+    
+    //
+    // Calculate class sizes and means
+    //
+    mu.setbounds(0, nvars-1);
+    muc.setbounds(0, nclasses-1, 0, nvars-1);
+    nc.setbounds(0, nclasses-1);
+    for(j = 0; j <= nvars-1; j++)
+    {
+        mu(j) = 0;
+    }
+    for(i = 0; i <= nclasses-1; i++)
+    {
+        nc(i) = 0;
+        for(j = 0; j <= nvars-1; j++)
+        {
+            muc(i,j) = 0;
+        }
+    }
+    for(i = 0; i <= npoints-1; i++)
+    {
+        ap::vadd(&mu(0), &xy(i, 0), ap::vlen(0,nvars-1));
+        ap::vadd(&muc(c(i), 0), &xy(i, 0), ap::vlen(0,nvars-1));
+        nc(c(i)) = nc(c(i))+1;
+    }
+    for(i = 0; i <= nclasses-1; i++)
+    {
+        v = double(1)/double(nc(i));
+        ap::vmul(&muc(i, 0), ap::vlen(0,nvars-1), v);
+    }
+    v = double(1)/double(npoints);
+    ap::vmul(&mu(0), ap::vlen(0,nvars-1), v);
+    
+    //
+    // Create ST matrix
+    //
+    st.setbounds(0, nvars-1, 0, nvars-1);
+    for(i = 0; i <= nvars-1; i++)
+    {
+        for(j = 0; j <= nvars-1; j++)
+        {
+            st(i,j) = 0;
+        }
+    }
+    for(k = 0; k <= npoints-1; k++)
+    {
+        ap::vmove(&tf(0), &xy(k, 0), ap::vlen(0,nvars-1));
+        ap::vsub(&tf(0), &mu(0), ap::vlen(0,nvars-1));
+        for(i = 0; i <= nvars-1; i++)
+        {
+            v = tf(i);
+            ap::vadd(&st(i, 0), &tf(0), ap::vlen(0,nvars-1), v);
+        }
+    }
+    
+    //
+    // Create SW matrix
+    //
+    sw.setbounds(0, nvars-1, 0, nvars-1);
+    for(i = 0; i <= nvars-1; i++)
+    {
+        for(j = 0; j <= nvars-1; j++)
+        {
+            sw(i,j) = 0;
+        }
+    }
+    for(k = 0; k <= npoints-1; k++)
+    {
+        ap::vmove(&tf(0), &xy(k, 0), ap::vlen(0,nvars-1));
+        ap::vsub(&tf(0), &muc(c(k), 0), ap::vlen(0,nvars-1));
+        for(i = 0; i <= nvars-1; i++)
+        {
+            v = tf(i);
+            ap::vadd(&sw(i, 0), &tf(0), ap::vlen(0,nvars-1), v);
+        }
+    }
+    
+    //
+    // Maximize ratio J=(w'*ST*w)/(w'*SW*w).
+    //
+    // First, make transition from w to v such that w'*ST*w becomes v'*v:
+    //    v  = root(ST)*w = R*w
+    //    R  = root(D)*Z'
+    //    w  = (root(ST)^-1)*v = RI*v
+    //    RI = Z*inv(root(D))
+    //    J  = (v'*v)/(v'*(RI'*SW*RI)*v)
+    //    ST = Z*D*Z'
+    //
+    //    so we have
+    //
+    //    J = (v'*v) / (v'*(inv(root(D))*Z'*SW*Z*inv(root(D)))*v)  =
+    //      = (v'*v) / (v'*A*v)
+    //
+    if( !smatrixevd(st, nvars, 1, true, d, z) )
+    {
+        info = -4;
+        return;
+    }
+    w.setbounds(0, nvars-1, 0, nvars-1);
+    if( d(nvars-1)<=0||d(0)<=1000*ap::machineepsilon*d(nvars-1) )
+    {
+        
+        //
+        // Special case: D[NVars-1]<=0
+        // Degenerate task (all variables takes the same value).
+        //
+        if( d(nvars-1)<=0 )
+        {
+            info = 2;
+            for(i = 0; i <= nvars-1; i++)
+            {
+                for(j = 0; j <= nvars-1; j++)
+                {
+                    if( i==j )
+                    {
+                        w(i,j) = 1;
+                    }
+                    else
+                    {
+                        w(i,j) = 0;
+                    }
+                }
+            }
+            return;
+        }
+        
+        //
+        // Special case: degenerate ST matrix, multicollinearity found.
+        // Since we know ST eigenvalues/vectors we can translate task to
+        // non-degenerate form.
+        //
+        // Let WG is orthogonal basis of the non zero variance subspace
+        // of the ST and let WZ is orthogonal basis of the zero variance
+        // subspace.
+        //
+        // Projection on WG allows us to use LDA on reduced M-dimensional
+        // subspace, N-M vectors of WZ allows us to update reduced LDA
+        // factors to full N-dimensional subspace.
+        //
+        m = 0;
+        for(k = 0; k <= nvars-1; k++)
+        {
+            if( d(k)<=1000*ap::machineepsilon*d(nvars-1) )
+            {
+                m = k+1;
+            }
+        }
+        ap::ap_error::make_assertion(m!=0, "FisherLDAN: internal error #1");
+        xyproj.setbounds(0, npoints-1, 0, nvars-m);
+        matrixmatrixmultiply(xy, 0, npoints-1, 0, nvars-1, false, z, 0, nvars-1, m, nvars-1, false, 1.0, xyproj, 0, npoints-1, 0, nvars-m-1, 0.0, work);
+        for(i = 0; i <= npoints-1; i++)
+        {
+            xyproj(i,nvars-m) = xy(i,nvars);
+        }
+        fisherldan(xyproj, npoints, nvars-m, nclasses, info, wproj);
+        if( info<0 )
+        {
+            return;
+        }
+        matrixmatrixmultiply(z, 0, nvars-1, m, nvars-1, false, wproj, 0, nvars-m-1, 0, nvars-m-1, false, 1.0, w, 0, nvars-1, 0, nvars-m-1, 0.0, work);
+        for(k = nvars-m; k <= nvars-1; k++)
+        {
+            ap::vmove(w.getcolumn(k, 0, nvars-1), z.getcolumn(k-(nvars-m), 0, nvars-1));
+        }
+        info = 2;
+    }
+    else
+    {
+        
+        //
+        // General case: no multicollinearity
+        //
+        tm.setbounds(0, nvars-1, 0, nvars-1);
+        a.setbounds(0, nvars-1, 0, nvars-1);
+        matrixmatrixmultiply(sw, 0, nvars-1, 0, nvars-1, false, z, 0, nvars-1, 0, nvars-1, false, 1.0, tm, 0, nvars-1, 0, nvars-1, 0.0, work);
+        matrixmatrixmultiply(z, 0, nvars-1, 0, nvars-1, true, tm, 0, nvars-1, 0, nvars-1, false, 1.0, a, 0, nvars-1, 0, nvars-1, 0.0, work);
+        for(i = 0; i <= nvars-1; i++)
+        {
+            for(j = 0; j <= nvars-1; j++)
+            {
+                a(i,j) = a(i,j)/sqrt(d(i)*d(j));
+            }
+        }
+        if( !smatrixevd(a, nvars, 1, true, d2, z2) )
+        {
+            info = -4;
+            return;
+        }
+        for(k = 0; k <= nvars-1; k++)
+        {
+            for(i = 0; i <= nvars-1; i++)
+            {
+                tf(i) = z2(i,k)/sqrt(d(i));
+            }
+            for(i = 0; i <= nvars-1; i++)
+            {
+                v = ap::vdotproduct(&z(i, 0), &tf(0), ap::vlen(0,nvars-1));
+                w(i,k) = v;
+            }
+        }
+    }
+    
+    //
+    // Post-processing:
+    // * normalization
+    // * converting to non-negative form, if possible
+    //
+    for(k = 0; k <= nvars-1; k++)
+    {
+        v = ap::vdotproduct(w.getcolumn(k, 0, nvars-1), w.getcolumn(k, 0, nvars-1));
+        v = 1/sqrt(v);
+        ap::vmul(w.getcolumn(k, 0, nvars-1), v);
+        v = 0;
+        for(i = 0; i <= nvars-1; i++)
+        {
+            v = v+w(i,k);
+        }
+        if( v<0 )
+        {
+            ap::vmul(w.getcolumn(k, 0, nvars-1), -1);
+        }
+    }
+}
+
+
+