svmlightcli 0.1.0

data/ext/svmlightcli/svm_common.h

@@ -0,0 +1,301 @@
+/************************************************************************/
+/*                                                                      */
+/*   svm_common.h                                                       */
+/*                                                                      */
+/*   Definitions and functions used in both svm_learn and svm_classify. */
+/*                                                                      */
+/*   Author: Thorsten Joachims                                          */
+/*   Date: 02.07.02                                                     */
+/*                                                                      */
+/*   Copyright (c) 2002  Thorsten Joachims - All rights reserved        */
+/*                                                                      */
+/*   This software is available for non-commercial use only. It must    */
+/*   not be modified and distributed without prior permission of the    */
+/*   author. The author is not responsible for implications from the    */
+/*   use of this software.                                              */
+/*                                                                      */
+/************************************************************************/
+
+#ifndef SVM_COMMON
+#define SVM_COMMON
+
+# include <stdio.h>
+# include <ctype.h>
+# include <math.h>
+# include <string.h>
+# include <stdlib.h>
+# include <time.h> 
+# include <float.h>
+
+# define VERSION       "V6.02"
+# define VERSION_DATE  "14.08.08"
+
+# define CFLOAT  float       /* the type of float to use for caching */
+                             /* kernel evaluations. Using float saves */
+                             /* us some memory, but you can use double, too */
+# define FNUM    long        /* the type used for storing feature ids */
+# define FVAL    float       /* the type used for storing feature values */
+# define MAXFEATNUM 99999999 /* maximum feature number (must be in
+			  	valid range of FNUM type and long int!) */
+
+# define LINEAR  0           /* linear kernel type */
+# define POLY    1           /* polynoial kernel type */
+# define RBF     2           /* rbf kernel type */
+# define SIGMOID 3           /* sigmoid kernel type */
+
+# define CLASSIFICATION 1    /* train classification model */
+# define REGRESSION     2    /* train regression model */
+# define RANKING        3    /* train ranking model */
+# define OPTIMIZATION   4    /* train on general set of constraints */
+
+# define MAXSHRINK     50000    /* maximum number of shrinking rounds */
+
+typedef struct word {
+  FNUM    wnum;	               /* word number */
+  FVAL    weight;              /* word weight */
+} WORD;
+
+typedef struct svector {
+  WORD    *words;              /* The features/values in the vector by
+				  increasing feature-number. Feature
+				  numbers that are skipped are
+				  interpreted as having value zero. */
+  double  twonorm_sq;          /* The squared euclidian length of the
+                                  vector. Used to speed up the RBF kernel. */
+  char    *userdefined;        /* You can put additional information
+				  here. This can be useful, if you are
+				  implementing your own kernel that
+				  does not work with feature/values
+				  representations (for example a
+				  string kernel). By default,
+				  svm-light will put here the string
+				  after the # sign from each line of
+				  the input file. */
+  long    kernel_id;           /* Feature vectors with different
+				  kernel_id's are orthogonal (ie. the
+				  feature number do not match). This
+				  is used for computing component
+				  kernels for linear constraints which
+				  are a sum of several different
+				  weight vectors. (currently not
+				  implemented). */
+  struct svector *next;        /* Let's you set up a list of SVECTOR's
+				  for linear constraints which are a
+				  sum of multiple feature
+				  vectors. List is terminated by
+				  NULL. */
+  double  factor;              /* Factor by which this feature vector
+				  is multiplied in the sum. */
+} SVECTOR;
+
+typedef struct doc {
+  long    docnum;              /* Document ID. This has to be the position of 
+                                  the document in the training set array. */
+  long    queryid;             /* for learning rankings, constraints are 
+				  generated for documents with the same 
+				  queryID. */
+  double  costfactor;          /* Scales the cost of misclassifying this
+				  document by this factor. The effect of this
+				  value is, that the upper bound on the alpha
+				  for this example is scaled by this factor.
+				  The factors are set by the feature 
+				  'cost:<val>' in the training data. */
+  long    slackid;             /* Index of the slack variable
+				  corresponding to this
+				  constraint. All constraints with the
+				  same slackid share the same slack
+				  variable. This can only be used for
+				  svm_learn_optimization. */
+  SVECTOR *fvec;               /* Feature vector of the example. The
+				  feature vector can actually be a
+				  list of feature vectors. For
+				  example, the list will have two
+				  elements, if this DOC is a
+				  preference constraint. The one
+				  vector that is supposed to be ranked
+				  higher, will have a factor of +1,
+				  the lower ranked one should have a
+				  factor of -1. */
+} DOC;
+
+typedef struct learn_parm {
+  long   type;                 /* selects between regression and
+				  classification */
+  double svm_c;                /* upper bound C on alphas */
+  double eps;                  /* regression epsilon (eps=1.0 for
+				  classification */
+  double svm_costratio;        /* factor to multiply C for positive examples */
+  double transduction_posratio;/* fraction of unlabeled examples to be */
+                               /* classified as positives */
+  long   biased_hyperplane;    /* if nonzero, use hyperplane w*x+b=0 
+				  otherwise w*x=0 */
+  long   sharedslack;          /* if nonzero, it will use the shared
+                                  slack variable mode in
+                                  svm_learn_optimization. It requires
+                                  that the slackid is set for every
+                                  training example */
+  long   svm_maxqpsize;        /* size q of working set */
+  long   svm_newvarsinqp;      /* new variables to enter the working set 
+				  in each iteration */
+  long   kernel_cache_size;    /* size of kernel cache in megabytes */
+  double epsilon_crit;         /* tolerable error for distances used 
+				  in stopping criterion */
+  double epsilon_shrink;       /* how much a multiplier should be above 
+				  zero for shrinking */
+  long   svm_iter_to_shrink;   /* iterations h after which an example can
+				  be removed by shrinking */
+  long   maxiter;              /* number of iterations after which the
+				  optimizer terminates, if there was
+				  no progress in maxdiff */
+  long   remove_inconsistent;  /* exclude examples with alpha at C and 
+				  retrain */
+  long   skip_final_opt_check; /* do not check KT-Conditions at the end of
+				  optimization for examples removed by 
+				  shrinking. WARNING: This might lead to 
+				  sub-optimal solutions! */
+  long   compute_loo;          /* if nonzero, computes leave-one-out
+				  estimates */
+  double rho;                  /* parameter in xi/alpha-estimates and for
+				  pruning leave-one-out range [1..2] */
+  long   xa_depth;             /* parameter in xi/alpha-estimates upper
+				  bounding the number of SV the current
+				  alpha_t is distributed over */
+  char predfile[200];          /* file for predicitions on unlabeled examples
+				  in transduction */
+  char alphafile[200];         /* file to store optimal alphas in. use  
+				  empty string if alphas should not be 
+				  output */
+
+  /* you probably do not want to touch the following */
+  double epsilon_const;        /* tolerable error on eq-constraint */
+  double epsilon_a;            /* tolerable error on alphas at bounds */
+  double opt_precision;        /* precision of solver, set to e.g. 1e-21 
+				  if you get convergence problems */
+
+  /* the following are only for internal use */
+  long   svm_c_steps;          /* do so many steps for finding optimal C */
+  double svm_c_factor;         /* increase C by this factor every step */
+  double svm_costratio_unlab;
+  double svm_unlabbound;
+  double *svm_cost;            /* individual upper bounds for each var */
+  long   totwords;             /* number of features */
+} LEARN_PARM;
+
+typedef struct kernel_parm {
+  long    kernel_type;   /* 0=linear, 1=poly, 2=rbf, 3=sigmoid, 4=custom */
+  long    poly_degree;
+  double  rbf_gamma;
+  double  coef_lin;
+  double  coef_const;
+  char    custom[50];    /* for user supplied kernel */
+} KERNEL_PARM;
+
+typedef struct model {
+  long    sv_num;	
+  long    at_upper_bound;
+  double  b;
+  DOC     **supvec;
+  double  *alpha;
+  long    *index;       /* index from docnum to position in model */
+  long    totwords;     /* number of features */
+  long    totdoc;       /* number of training documents */
+  KERNEL_PARM kernel_parm; /* kernel */
+
+  /* the following values are not written to file */
+  double  loo_error,loo_recall,loo_precision; /* leave-one-out estimates */
+  double  xa_error,xa_recall,xa_precision;    /* xi/alpha estimates */
+  double  *lin_weights;                       /* weights for linear case using
+						 folding */
+  double  maxdiff;                            /* precision, up to which this 
+						 model is accurate */
+} MODEL;
+
+typedef struct quadratic_program {
+  long   opt_n;            /* number of variables */
+  long   opt_m;            /* number of linear equality constraints */
+  double *opt_ce,*opt_ce0; /* linear equality constraints */
+  double *opt_g;           /* hessian of objective */
+  double *opt_g0;          /* linear part of objective */
+  double *opt_xinit;       /* initial value for variables */
+  double *opt_low,*opt_up; /* box constraints */
+} QP;
+
+typedef struct kernel_cache {
+  long   *index;  /* cache some kernel evalutations */
+  CFLOAT *buffer; /* to improve speed */
+  long   *invindex;
+  long   *active2totdoc;
+  long   *totdoc2active;
+  long   *lru;
+  long   *occu;
+  long   elems;
+  long   max_elems;
+  long   time;
+  long   activenum;
+  long   buffsize;
+} KERNEL_CACHE;
+
+
+typedef struct timing_profile {
+  long   time_kernel;
+  long   time_opti;
+  long   time_shrink;
+  long   time_update;
+  long   time_model;
+  long   time_check;
+  long   time_select;
+} TIMING;
+
+typedef struct shrink_state {
+  long   *active;
+  long   *inactive_since;
+  long   deactnum;
+  double **a_history;  /* for shrinking with non-linear kernel */
+  long   maxhistory;
+  double *last_a;      /* for shrinking with linear kernel */
+  double *last_lin;    /* for shrinking with linear kernel */
+} SHRINK_STATE;
+
+double classify_example(MODEL *, DOC *);
+double classify_example_linear(MODEL *, DOC *);
+double kernel(KERNEL_PARM *, DOC *, DOC *); 
+double single_kernel(KERNEL_PARM *, SVECTOR *, SVECTOR *); 
+double custom_kernel(KERNEL_PARM *, SVECTOR *, SVECTOR *); 
+SVECTOR *create_svector(WORD *, char *, double);
+SVECTOR *copy_svector(SVECTOR *);
+void   free_svector(SVECTOR *);
+double    sprod_ss(SVECTOR *, SVECTOR *);
+SVECTOR*  sub_ss(SVECTOR *, SVECTOR *); 
+SVECTOR*  add_ss(SVECTOR *, SVECTOR *); 
+SVECTOR*  add_list_ss(SVECTOR *); 
+void      append_svector_list(SVECTOR *a, SVECTOR *b);
+SVECTOR*  smult_s(SVECTOR *, double);
+int       featvec_eq(SVECTOR *, SVECTOR *); 
+double model_length_s(MODEL *, KERNEL_PARM *);
+void   clear_vector_n(double *, long);
+void   add_vector_ns(double *, SVECTOR *, double);
+double sprod_ns(double *, SVECTOR *);
+void   add_weight_vector_to_linear_model(MODEL *);
+DOC    *create_example(long, long, long, double, SVECTOR *);
+void   free_example(DOC *, long);
+MODEL  *read_model(char *);
+MODEL  *copy_model(MODEL *);
+void   free_model(MODEL *, int);
+void   read_documents(char *, DOC ***, double **, long *, long *);
+int    parse_document(char *, WORD *, double *, long *, long *, double *, long *, long, char **);
+double *read_alphas(char *,long);
+void   nol_ll(char *, long *, long *, long *);
+long   minl(long, long);
+long   maxl(long, long);
+long   get_runtime(void);
+int    space_or_null(int);
+void   *my_malloc(size_t); 
+void   copyright_notice(void);
+# ifdef _MSC_VER
+   int isnan(double);
+# endif
+
+extern long   verbosity;              /* verbosity level (0-4) */
+extern long   kernel_cache_statistic;
+
+#endif

data/ext/svmlightcli/svm_hideo.c

@@ -0,0 +1,1062 @@
+/***********************************************************************/
+/*                                                                     */
+/*   svm_hideo.c                                                       */
+/*                                                                     */
+/*   The Hildreth and D'Espo solver specialized for SVMs.              */
+/*                                                                     */
+/*   Author: Thorsten Joachims                                         */
+/*   Date: 02.07.02                                                    */
+/*                                                                     */
+/*   Copyright (c) 2002  Thorsten Joachims - All rights reserved       */
+/*                                                                     */
+/*   This software is available for non-commercial use only. It must   */
+/*   not be modified and distributed without prior permission of the   */
+/*   author. The author is not responsible for implications from the   */
+/*   use of this software.                                             */
+/*                                                                     */
+/***********************************************************************/
+
+# include <math.h>
+# include "svm_common.h"
+
+/* 
+  solve the quadratic programming problem
+ 
+  minimize   g0 * x + 1/2 x' * G * x
+  subject to ce*x = ce0
+             l <= x <= u
+ 
+  The linear constraint vector ce can only have -1/+1 as entries 
+*/
+
+/* Common Block Declarations */
+
+long verbosity;
+
+# define PRIMAL_OPTIMAL      1
+# define DUAL_OPTIMAL        2
+# define MAXITER_EXCEEDED    3
+# define NAN_SOLUTION        4
+# define ONLY_ONE_VARIABLE   5
+
+# define LARGEROUND          0
+# define SMALLROUND          1
+
+/* /////////////////////////////////////////////////////////////// */
+
+# define DEF_PRECISION          1E-5
+# define DEF_MAX_ITERATIONS     200
+# define DEF_LINDEP_SENSITIVITY 1E-8
+# define EPSILON_HIDEO          1E-20
+# define EPSILON_EQ             1E-5
+
+double *optimize_qp(QP *, double *, long, double *, LEARN_PARM *);
+double *primal=0,*dual=0;
+long   precision_violations=0;
+double opt_precision=DEF_PRECISION;
+long   maxiter=DEF_MAX_ITERATIONS;
+double lindep_sensitivity=DEF_LINDEP_SENSITIVITY;
+double *buffer;
+long   *nonoptimal;
+
+long  smallroundcount=0;
+long  roundnumber=0;
+
+/* /////////////////////////////////////////////////////////////// */
+
+void *my_malloc();
+
+int optimize_hildreth_despo(long,long,double,double,double,long,long,long,double,double *,
+			    double *,double *,double *,double *,double *,
+			    double *,double *,double *,long *,double *,double *);
+int solve_dual(long,long,double,double,long,double *,double *,double *,
+	       double *,double *,double *,double *,double *,double *,
+	       double *,double *,double *,double *,long);
+
+void linvert_matrix(double *, long, double *, double, long *);
+void lprint_matrix(double *, long);
+void ladd_matrix(double *, long, double);
+void lcopy_matrix(double *, long, double *);
+void lswitch_rows_matrix(double *, long, long, long);
+void lswitchrk_matrix(double *, long, long, long);
+
+double calculate_qp_objective(long, double *, double *, double *);
+
+
+
+double *optimize_qp(qp,epsilon_crit,nx,threshold,learn_parm)
+QP *qp;
+double *epsilon_crit;
+long nx; /* Maximum number of variables in QP */
+double *threshold; 
+LEARN_PARM *learn_parm;
+/* start the optimizer and return the optimal values */
+/* The HIDEO optimizer does not necessarily fully solve the problem. */
+/* Since it requires a strictly positive definite hessian, the solution */
+/* is restricted to a linear independent subset in case the matrix is */
+/* only semi-definite. */
+{
+  long i,j;
+  int result;
+  double eq,progress;
+
+  roundnumber++;
+
+  if(!primal) { /* allocate memory at first call */
+    primal=(double *)my_malloc(sizeof(double)*nx);
+    dual=(double *)my_malloc(sizeof(double)*((nx+1)*2));
+    nonoptimal=(long *)my_malloc(sizeof(long)*(nx));
+    buffer=(double *)my_malloc(sizeof(double)*((nx+1)*2*(nx+1)*2+
+					       nx*nx+2*(nx+1)*2+2*nx+1+2*nx+
+					       nx+nx+nx*nx));
+    (*threshold)=0;
+    for(i=0;i<nx;i++) {
+      primal[i]=0;
+    }
+  }
+
+  if(verbosity>=4) { /* really verbose */
+    printf("\n\n");
+    eq=qp->opt_ce0[0];
+    for(i=0;i<qp->opt_n;i++) {
+      eq+=qp->opt_xinit[i]*qp->opt_ce[i];
+      printf("%f: ",qp->opt_g0[i]);
+      for(j=0;j<qp->opt_n;j++) {
+	printf("%f ",qp->opt_g[i*qp->opt_n+j]);
+      }
+      printf(": a=%.10f < %f",qp->opt_xinit[i],qp->opt_up[i]);
+      printf(": y=%f\n",qp->opt_ce[i]);
+    }
+    if(qp->opt_m) {
+      printf("EQ: %f*x0",qp->opt_ce[0]);
+      for(i=1;i<qp->opt_n;i++) {
+	printf(" + %f*x%ld",qp->opt_ce[i],i);
+      }
+      printf(" = %f\n\n",-qp->opt_ce0[0]);
+    }
+  }
+
+  result=optimize_hildreth_despo(qp->opt_n,qp->opt_m,
+				 opt_precision,(*epsilon_crit),
+				 learn_parm->epsilon_a,maxiter,
+				 /* (long)PRIMAL_OPTIMAL, */
+				 (long)0, (long)0,
+				 lindep_sensitivity,
+				 qp->opt_g,qp->opt_g0,qp->opt_ce,qp->opt_ce0,
+				 qp->opt_low,qp->opt_up,primal,qp->opt_xinit,
+				 dual,nonoptimal,buffer,&progress);
+  if(verbosity>=3) { 
+    printf("return(%d)...",result);
+  }
+
+  if(learn_parm->totwords < learn_parm->svm_maxqpsize) { 
+    /* larger working sets will be linear dependent anyway */
+    learn_parm->svm_maxqpsize=maxl(learn_parm->totwords,(long)2);
+  }
+
+  if(result == NAN_SOLUTION) {
+    lindep_sensitivity*=2;  /* throw out linear dependent examples more */
+                            /* generously */
+    if(learn_parm->svm_maxqpsize>2) {
+      learn_parm->svm_maxqpsize--;  /* decrease size of qp-subproblems */
+    }
+    precision_violations++;
+  }
+
+  /* take one round of only two variable to get unstuck */
+  if((result != PRIMAL_OPTIMAL) || (!(roundnumber % 31)) || (progress <= 0)) {
+
+    smallroundcount++;
+
+    result=optimize_hildreth_despo(qp->opt_n,qp->opt_m,
+				   opt_precision,(*epsilon_crit),
+				   learn_parm->epsilon_a,(long)maxiter,
+				   (long)PRIMAL_OPTIMAL,(long)SMALLROUND,
+				   lindep_sensitivity,
+				   qp->opt_g,qp->opt_g0,qp->opt_ce,qp->opt_ce0,
+				   qp->opt_low,qp->opt_up,primal,qp->opt_xinit,
+				   dual,nonoptimal,buffer,&progress);
+    if(verbosity>=3) { 
+      printf("return_srd(%d)...",result);
+    }
+
+    if(result != PRIMAL_OPTIMAL) {
+      if(result != ONLY_ONE_VARIABLE) 
+	precision_violations++;
+      if(result == MAXITER_EXCEEDED) 
+	maxiter+=100;
+      if(result == NAN_SOLUTION) {
+	lindep_sensitivity*=2;  /* throw out linear dependent examples more */
+	                        /* generously */
+	/* results not valid, so return inital values */
+	for(i=0;i<qp->opt_n;i++) {
+	  primal[i]=qp->opt_xinit[i];
+	}
+      }
+    }
+  }
+
+
+  if(precision_violations > 50) {
+    precision_violations=0;
+    (*epsilon_crit)*=10.0; 
+    if(verbosity>=1) {
+      printf("\nWARNING: Relaxing epsilon on KT-Conditions (%f).\n",
+	     (*epsilon_crit));
+    }
+  }	  
+
+  if((qp->opt_m>0) && (result != NAN_SOLUTION) && (!isnan(dual[1]-dual[0])))
+    (*threshold)=dual[1]-dual[0];
+  else
+    (*threshold)=0;
+
+  if(verbosity>=4) { /* really verbose */
+    printf("\n\n");
+    eq=qp->opt_ce0[0];
+    for(i=0;i<qp->opt_n;i++) {
+      eq+=primal[i]*qp->opt_ce[i];
+      printf("%f: ",qp->opt_g0[i]);
+      for(j=0;j<qp->opt_n;j++) {
+	printf("%f ",qp->opt_g[i*qp->opt_n+j]);
+      }
+      printf(": a=%.30f",primal[i]);
+      printf(": nonopti=%ld",nonoptimal[i]);
+      printf(": y=%f\n",qp->opt_ce[i]);
+    }
+    printf("eq-constraint=%.30f\n",eq);
+    printf("b=%f\n",(*threshold));
+    printf(" smallroundcount=%ld ",smallroundcount);
+  }
+
+  return(primal);
+}
+
+
+
+int optimize_hildreth_despo(n,m,precision,epsilon_crit,epsilon_a,maxiter,goal,
+			    smallround,lindep_sensitivity,g,g0,ce,ce0,low,up,
+			    primal,init,dual,lin_dependent,buffer,progress)
+     long   n;            /* number of variables */
+     long   m;            /* number of linear equality constraints [0,1] */
+     double precision;    /* solve at least to this dual precision */
+     double epsilon_crit; /* stop, if KT-Conditions approx fulfilled */
+     double epsilon_a;    /* precision of alphas at bounds */
+     long   maxiter;      /* stop after this many iterations */
+     long   goal;         /* keep going until goal fulfilled */
+     long   smallround;   /* use only two variables of steepest descent */
+     double lindep_sensitivity; /* epsilon for detecting linear dependent ex */
+     double *g;           /* hessian of objective */
+     double *g0;          /* linear part of objective */
+     double *ce,*ce0;     /* linear equality constraints */
+     double *low,*up;     /* box constraints */
+     double *primal;      /* primal variables */
+     double *init;        /* initial values of primal */
+     double *dual;        /* dual variables */
+     long   *lin_dependent;
+     double *buffer;
+     double *progress;    /* delta in the objective function between
+                             before and after */
+{
+  long i,j,k,from,to,n_indep,changed;
+  double sum,bmin=0,bmax=0;
+  double *d,*d0,*ig,*dual_old,*temp,*start;       
+  double *g0_new,*g_new,*ce_new,*ce0_new,*low_new,*up_new;
+  double add,t;
+  int result;
+  double obj_before,obj_after; 
+  long b1,b2;
+  double g0_b1,g0_b2,ce0_b;
+
+  g0_new=&(buffer[0]);    /* claim regions of buffer */
+  d=&(buffer[n]);
+  d0=&(buffer[n+(n+m)*2*(n+m)*2]);
+  ce_new=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2]);
+  ce0_new=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n]);
+  ig=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m]);
+  dual_old=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n]);
+  low_new=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n+(n+m)*2]);
+  up_new=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n+(n+m)*2+n]);
+  start=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n+(n+m)*2+n+n]);
+  g_new=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n+(n+m)*2+n+n+n]);
+  temp=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n+(n+m)*2+n+n+n+n*n]);
+
+  b1=-1;
+  b2=-1;
+  for(i=0;i<n;i++) {   /* get variables with steepest feasible descent */
+    sum=g0[i];         
+    for(j=0;j<n;j++) 
+      sum+=init[j]*g[i*n+j];
+    sum=sum*ce[i];
+    if(((b1==-1) || (sum<bmin)) 
+       && (!((init[i]<=(low[i]+epsilon_a)) && (ce[i]<0.0)))
+       && (!((init[i]>=( up[i]-epsilon_a)) && (ce[i]>0.0)))
+       ) {
+      bmin=sum;
+      b1=i;
+    }
+    if(((b2==-1) || (sum>=bmax)) 
+       && (!((init[i]<=(low[i]+epsilon_a)) && (ce[i]>0.0)))
+       && (!((init[i]>=( up[i]-epsilon_a)) && (ce[i]<0.0)))
+       ) {
+      bmax=sum;
+      b2=i;
+    }
+  }
+  /* in case of unbiased hyperplane, the previous projection on */
+  /* equality constraint can lead to b1 or b2 being -1. */
+  if((b1 == -1) || (b2 == -1)) {
+    b1=maxl(b1,b2);
+    b2=maxl(b1,b2);
+  }
+
+  for(i=0;i<n;i++) {
+    start[i]=init[i];
+  }
+
+  /* in case both example vectors are linearly dependent */
+  /* WARNING: Assumes that ce[] in {-1,1} */
+  add=0;
+  changed=0;
+  if((b1 != b2) && (m==1)) {
+    for(i=0;i<n;i++) {  /* fix other vectors */
+      if(i==b1) 
+	g0_b1=g0[i];
+      if(i==b2) 
+	g0_b2=g0[i];
+    }
+    ce0_b=ce0[0];
+    for(i=0;i<n;i++) {  
+      if((i!=b1) && (i!=b2)) {
+	for(j=0;j<n;j++) {
+	  if(j==b1) 
+	    g0_b1+=start[i]*g[i*n+j];
+	  if(j==b2) 
+	    g0_b2+=start[i]*g[i*n+j];
+	}
+	ce0_b-=(start[i]*ce[i]);
+      }
+    }
+    if((g[b1*n+b2] == g[b1*n+b1]) && (g[b1*n+b2] == g[b2*n+b2])) {
+      /* printf("euqal\n"); */
+      if(ce[b1] == ce[b2]) { 
+	if(g0_b1 <= g0_b2) { /* set b1 to upper bound */
+	  /* printf("case +=<\n"); */
+	  changed=1;
+	  t=up[b1]-init[b1];
+	  if((init[b2]-low[b2]) < t) {
+	    t=init[b2]-low[b2];
+	  }
+	  start[b1]=init[b1]+t;
+	  start[b2]=init[b2]-t;
+	}
+	else if(g0_b1 > g0_b2) { /* set b2 to upper bound */
+	  /* printf("case +=>\n"); */
+	  changed=1;
+	  t=up[b2]-init[b2];
+	  if((init[b1]-low[b1]) < t) {
+	    t=init[b1]-low[b1];
+	  }
+	  start[b1]=init[b1]-t;
+	  start[b2]=init[b2]+t;
+	}
+      }
+      else if(((g[b1*n+b1]>0) || (g[b2*n+b2]>0))) { /* (ce[b1] != ce[b2]) */ 
+	/* printf("case +!\n"); */
+	t=((ce[b2]/ce[b1])*g0[b1]-g0[b2]+ce0[0]*(g[b1*n+b1]*ce[b2]/ce[b1]-g[b1*n+b2]/ce[b1]))/((ce[b2]*ce[b2]/(ce[b1]*ce[b1]))*g[b1*n+b1]+g[b2*n+b2]-2*(g[b1*n+b2]*ce[b2]/ce[b1]))-init[b2];
+	changed=1;
+	if((up[b2]-init[b2]) < t) {
+	  t=up[b2]-init[b2];
+	}
+	if((init[b2]-low[b2]) < -t) {
+	  t=-(init[b2]-low[b2]);
+	}
+	if((up[b1]-init[b1]) < t) {
+	  t=(up[b1]-init[b1]);
+	}
+	if((init[b1]-low[b1]) < -t) {
+	  t=-(init[b1]-low[b1]);
+	}
+	start[b1]=init[b1]+t;
+	start[b2]=init[b2]+t;
+      }
+    }
+    if((-g[b1*n+b2] == g[b1*n+b1]) && (-g[b1*n+b2] == g[b2*n+b2])) {
+      /* printf("diffeuqal\n"); */
+      if(ce[b1] != ce[b2]) {
+	if((g0_b1+g0_b2) < 0) { /* set b1 and b2 to upper bound */
+	  /* printf("case -!<\n"); */
+	  changed=1;
+	  t=up[b1]-init[b1];
+	  if((up[b2]-init[b2]) < t) {
+	    t=up[b2]-init[b2];
+	  }
+	  start[b1]=init[b1]+t;
+	  start[b2]=init[b2]+t;
+	}     
+	else if((g0_b1+g0_b2) >= 0) { /* set b1 and b2 to lower bound */
+	  /* printf("case -!>\n"); */
+	  changed=1;
+	  t=init[b1]-low[b1];
+	  if((init[b2]-low[b2]) < t) {
+	    t=init[b2]-low[b2];
+	  }
+	  start[b1]=init[b1]-t;
+	  start[b2]=init[b2]-t;
+	}
+      }
+      else if(((g[b1*n+b1]>0) || (g[b2*n+b2]>0))) { /* (ce[b1]==ce[b2]) */
+	/*  printf("case -=\n"); */
+	t=((ce[b2]/ce[b1])*g0[b1]-g0[b2]+ce0[0]*(g[b1*n+b1]*ce[b2]/ce[b1]-g[b1*n+b2]/ce[b1]))/((ce[b2]*ce[b2]/(ce[b1]*ce[b1]))*g[b1*n+b1]+g[b2*n+b2]-2*(g[b1*n+b2]*ce[b2]/ce[b1]))-init[b2];
+	changed=1;
+	if((up[b2]-init[b2]) < t) {
+	  t=up[b2]-init[b2];
+	}
+	if((init[b2]-low[b2]) < -t) {
+	  t=-(init[b2]-low[b2]);
+	}
+	if((up[b1]-init[b1]) < -t) {
+	  t=-(up[b1]-init[b1]);
+	}
+	if((init[b1]-low[b1]) < t) {
+	  t=init[b1]-low[b1];
+	}
+	start[b1]=init[b1]-t;
+	start[b2]=init[b2]+t;
+      }	
+    }
+  }
+  /* if we have a biased hyperplane, then adding a constant to the */
+  /* hessian does not change the solution. So that is done for examples */
+  /* with zero diagonal entry, since HIDEO cannot handle them. */
+  if((m>0) 
+     && ((fabs(g[b1*n+b1]) < lindep_sensitivity) 
+	 || (fabs(g[b2*n+b2]) < lindep_sensitivity))) {
+    /* printf("Case 0\n"); */
+    add+=0.093274;
+  }    
+  /* in case both examples are linear dependent */
+  else if((m>0) 
+	  && (g[b1*n+b2] != 0 && g[b2*n+b2] != 0)
+	  && (fabs(g[b1*n+b1]/g[b1*n+b2] - g[b1*n+b2]/g[b2*n+b2])
+	      < lindep_sensitivity)) { 
+    /* printf("Case lindep\n"); */
+    add+=0.078274;
+  }
+
+  /* special case for zero diagonal entry on unbiased hyperplane */
+  if((m==0) && (b1>=0))  {
+    if(fabs(g[b1*n+b1]) < lindep_sensitivity) { 
+      /* printf("Case 0b1\n"); */
+      for(i=0;i<n;i++) {  /* fix other vectors */
+	if(i==b1) 
+	  g0_b1=g0[i];
+      }
+      for(i=0;i<n;i++) {  
+	if(i!=b1) {
+	  for(j=0;j<n;j++) {
+	    if(j==b1) 
+	      g0_b1+=start[i]*g[i*n+j];
+	  }
+	}
+      }
+      if(g0_b1<0)
+	start[b1]=up[b1];
+      if(g0_b1>=0)
+	start[b1]=low[b1];
+    }
+  }
+  if((m==0) && (b2>=0))  {
+    if(fabs(g[b2*n+b2]) < lindep_sensitivity) { 
+      /* printf("Case 0b2\n"); */
+      for(i=0;i<n;i++) {  /* fix other vectors */
+	if(i==b2) 
+	  g0_b2=g0[i];
+      }
+      for(i=0;i<n;i++) {  
+	if(i!=b2) {
+	  for(j=0;j<n;j++) {
+	    if(j==b2) 
+	      g0_b2+=start[i]*g[i*n+j];
+	  }
+	}
+      }
+      if(g0_b2<0)
+	start[b2]=up[b2];
+      if(g0_b2>=0)
+	start[b2]=low[b2];
+    }
+  }
+
+  /* printf("b1=%ld,b2=%ld\n",b1,b2); */
+
+  lcopy_matrix(g,n,d);
+  if((m==1) && (add>0.0)) {
+    for(j=0;j<n;j++) {
+      for(k=0;k<n;k++) {
+	d[j*n+k]+=add*ce[j]*ce[k];
+      }
+    }
+  }
+  else {
+    add=0.0;
+  }
+
+  if(n>2) {                    /* switch, so that variables are better mixed */
+    lswitchrk_matrix(d,n,b1,(long)0); 
+    if(b2 == 0) 
+      lswitchrk_matrix(d,n,b1,(long)1); 
+    else
+      lswitchrk_matrix(d,n,b2,(long)1); 
+  }
+  if(smallround == SMALLROUND) {
+    for(i=2;i<n;i++) {
+      lin_dependent[i]=1;
+    }
+    if(m>0) { /* for biased hyperplane, pick two variables */
+      lin_dependent[0]=0;
+      lin_dependent[1]=0;
+    }
+    else {    /* for unbiased hyperplane, pick only one variable */
+      lin_dependent[0]=smallroundcount % 2;
+      lin_dependent[1]=(smallroundcount+1) % 2;
+    }
+  }
+  else {
+    for(i=0;i<n;i++) {
+      lin_dependent[i]=0;
+    }
+  }
+  linvert_matrix(d,n,ig,lindep_sensitivity,lin_dependent);
+  if(n>2) {                    /* now switch back */
+    if(b2 == 0) {
+      lswitchrk_matrix(ig,n,b1,(long)1); 
+      i=lin_dependent[1];  
+      lin_dependent[1]=lin_dependent[b1];
+      lin_dependent[b1]=i;
+    }
+    else {
+      lswitchrk_matrix(ig,n,b2,(long)1); 
+      i=lin_dependent[1];  
+      lin_dependent[1]=lin_dependent[b2];
+      lin_dependent[b2]=i;
+    }
+    lswitchrk_matrix(ig,n,b1,(long)0); 
+    i=lin_dependent[0];  
+    lin_dependent[0]=lin_dependent[b1];
+    lin_dependent[b1]=i;
+  }
+  /* lprint_matrix(d,n); */
+  /* lprint_matrix(ig,n); */
+
+  lcopy_matrix(g,n,g_new);   /* restore g_new matrix */
+  if(add>0)
+    for(j=0;j<n;j++) {
+      for(k=0;k<n;k++) {
+	g_new[j*n+k]+=add*ce[j]*ce[k];
+      }
+    }
+
+  for(i=0;i<n;i++) {  /* fix linear dependent vectors */
+    g0_new[i]=g0[i]+add*ce0[0]*ce[i];
+  }
+  if(m>0) ce0_new[0]=-ce0[0];
+  for(i=0;i<n;i++) {  /* fix linear dependent vectors */
+    if(lin_dependent[i]) {
+      for(j=0;j<n;j++) {
+	if(!lin_dependent[j]) {
+	  g0_new[j]+=start[i]*g_new[i*n+j];
+	}
+      }
+      if(m>0) ce0_new[0]-=(start[i]*ce[i]);
+    }
+  }
+  from=0;   /* remove linear dependent vectors */
+  to=0;
+  n_indep=0;
+  for(i=0;i<n;i++) {
+    if(!lin_dependent[i]) {
+      g0_new[n_indep]=g0_new[i];
+      ce_new[n_indep]=ce[i]; 
+      low_new[n_indep]=low[i];
+      up_new[n_indep]=up[i];
+      primal[n_indep]=start[i];
+      n_indep++;
+    }
+    for(j=0;j<n;j++) {
+      if((!lin_dependent[i]) && (!lin_dependent[j])) {
+        ig[to]=ig[from];
+        g_new[to]=g_new[from];
+	to++;
+      }
+      from++;
+    }
+  }
+
+  if(verbosity>=3) {
+    printf("real_qp_size(%ld)...",n_indep);
+  }
+  
+  /* cannot optimize with only one variable */
+  if((n_indep<=1) && (m>0) && (!changed)) { 
+    for(i=n-1;i>=0;i--) {
+      primal[i]=init[i];
+    }
+    return((int)ONLY_ONE_VARIABLE);
+  }
+
+  if((!changed) || (n_indep>1)) { 
+    result=solve_dual(n_indep,m,precision,epsilon_crit,maxiter,g_new,g0_new,
+		      ce_new,ce0_new,low_new,up_new,primal,d,d0,ig,
+		      dual,dual_old,temp,goal);
+  }
+  else {
+    result=PRIMAL_OPTIMAL;
+  }
+  
+  j=n_indep;
+  for(i=n-1;i>=0;i--) {
+    if(!lin_dependent[i]) {
+      j--;
+      primal[i]=primal[j];
+    }
+    else {
+      primal[i]=start[i];  /* leave as is */
+    }
+    temp[i]=primal[i];
+  }
+   
+  obj_before=calculate_qp_objective(n,g,g0,init);
+  obj_after=calculate_qp_objective(n,g,g0,primal);
+  (*progress)=obj_before-obj_after;
+  if(verbosity>=3) {
+    printf("before(%.30f)...after(%.30f)...result_sd(%d)...",
+	   obj_before,obj_after,result); 
+  }
+
+  return((int)result);
+}
+
+
+int solve_dual(n,m,precision,epsilon_crit,maxiter,g,g0,ce,ce0,low,up,primal,
+	       d,d0,ig,dual,dual_old,temp,goal)
+     /* Solves the dual using the method of Hildreth and D'Espo. */
+     /* Can only handle problems with zero or exactly one */
+     /* equality constraints. */
+
+     long   n;            /* number of variables */
+     long   m;            /* number of linear equality constraints */
+     double precision;    /* solve at least to this dual precision */
+     double epsilon_crit; /* stop, if KT-Conditions approx fulfilled */
+     long   maxiter;      /* stop after that many iterations */
+     double *g;
+     double *g0;          /* linear part of objective */
+     double *ce,*ce0;     /* linear equality constraints */
+     double *low,*up;     /* box constraints */
+     double *primal;      /* variables (with initial values) */
+     double *d,*d0,*ig,*dual,*dual_old,*temp;       /* buffer  */
+     long goal;
+{
+  long i,j,k,iter;
+  double sum,w,maxviol,viol,temp1,temp2,isnantest;
+  double model_b,dist;
+  long retrain,maxfaktor,primal_optimal=0,at_bound,scalemaxiter;
+  double epsilon_a=1E-15,epsilon_hideo;
+  double eq; 
+
+  if((m<0) || (m>1)) 
+    perror("SOLVE DUAL: inappropriate number of eq-constrains!");
+
+  /*  
+  printf("\n");
+  for(i=0;i<n;i++) {
+    printf("%f: ",g0[i]);
+    for(j=0;j<n;j++) {
+      printf("%f ",g[i*n+j]);
+    }
+    printf(": a=%.30f",primal[i]);
+    printf(": y=%f\n",ce[i]);
+  }
+  */
+
+  for(i=0;i<2*(n+m);i++) {
+    dual[i]=0;
+    dual_old[i]=0;
+  }
+  for(i=0;i<n;i++) {   
+    for(j=0;j<n;j++) {   /* dual hessian for box constraints */
+      d[i*2*(n+m)+j]=ig[i*n+j];
+      d[(i+n)*2*(n+m)+j]=-ig[i*n+j];
+      d[i*2*(n+m)+j+n]=-ig[i*n+j];
+      d[(i+n)*2*(n+m)+j+n]=ig[i*n+j];
+    }
+    if(m>0) {
+      sum=0;              /* dual hessian for eq constraints */
+      for(j=0;j<n;j++) {
+	sum+=(ce[j]*ig[i*n+j]);
+      }
+      d[i*2*(n+m)+2*n]=sum;
+      d[i*2*(n+m)+2*n+1]=-sum;
+      d[(n+i)*2*(n+m)+2*n]=-sum;
+      d[(n+i)*2*(n+m)+2*n+1]=sum;
+      d[(n+n)*2*(n+m)+i]=sum;
+      d[(n+n+1)*2*(n+m)+i]=-sum;
+      d[(n+n)*2*(n+m)+(n+i)]=-sum;
+      d[(n+n+1)*2*(n+m)+(n+i)]=sum;
+      
+      sum=0;
+      for(j=0;j<n;j++) {
+	for(k=0;k<n;k++) {
+	  sum+=(ce[k]*ce[j]*ig[j*n+k]);
+	}
+      }
+      d[(n+n)*2*(n+m)+2*n]=sum;
+      d[(n+n)*2*(n+m)+2*n+1]=-sum;
+      d[(n+n+1)*2*(n+m)+2*n]=-sum;
+      d[(n+n+1)*2*(n+m)+2*n+1]=sum;
+    } 
+  }
+
+  for(i=0;i<n;i++) {   /* dual linear component for the box constraints */
+    w=0;
+    for(j=0;j<n;j++) {
+      w+=(ig[i*n+j]*g0[j]); 
+    }
+    d0[i]=up[i]+w;
+    d0[i+n]=-low[i]-w;
+  }
+
+  if(m>0) {  
+    sum=0;             /* dual linear component for eq constraints */
+    for(j=0;j<n;j++) {
+      for(k=0;k<n;k++) {
+	sum+=(ce[k]*ig[k*n+j]*g0[j]); 
+      }
+    }
+    d0[2*n]=ce0[0]+sum;
+    d0[2*n+1]=-ce0[0]-sum;
+  }
+
+  maxviol=999999;
+  iter=0;
+  retrain=1;
+  maxfaktor=1;
+  scalemaxiter=maxiter/5;
+  while((retrain) && (maxviol > 0) && (iter < (scalemaxiter*maxfaktor))) {
+    iter++;
+    
+    while((maxviol > precision) && (iter < (scalemaxiter*maxfaktor))) {
+      iter++;
+      maxviol=0;
+      for(i=0;i<2*(n+m);i++) {
+	sum=d0[i];
+	for(j=0;j<2*(n+m);j++) {
+	  sum+=d[i*2*(n+m)+j]*dual_old[j];
+	}
+	sum-=d[i*2*(n+m)+i]*dual_old[i];
+	dual[i]=-sum/d[i*2*(n+m)+i];
+	if(dual[i]<0) dual[i]=0;
+	
+	viol=fabs(dual[i]-dual_old[i]);
+	if(viol>maxviol) 
+	  maxviol=viol;
+	dual_old[i]=dual[i];
+      }
+      /*
+      printf("%d) maxviol=%20f precision=%f\n",iter,maxviol,precision); 
+      */
+    }
+  
+    if(m>0) {
+      for(i=0;i<n;i++) {
+	temp[i]=dual[i]-dual[i+n]+ce[i]*(dual[n+n]-dual[n+n+1])+g0[i];
+      }
+    } 
+    else {
+      for(i=0;i<n;i++) {
+	temp[i]=dual[i]-dual[i+n]+g0[i];
+      }
+    }
+    for(i=0;i<n;i++) {
+      primal[i]=0;             /* calc value of primal variables */
+      for(j=0;j<n;j++) {
+	primal[i]+=ig[i*n+j]*temp[j];
+      }
+      primal[i]*=-1.0;
+      if(primal[i]<=(low[i])) {  /* clip conservatively */
+	primal[i]=low[i];
+      }
+      else if(primal[i]>=(up[i])) {
+	primal[i]=up[i];
+      }
+    }
+
+    if(m>0) 
+      model_b=dual[n+n+1]-dual[n+n];
+    else
+      model_b=0;
+
+    epsilon_hideo=EPSILON_HIDEO;
+    for(i=0;i<n;i++) {           /* check precision of alphas */
+      dist=-model_b*ce[i]; 
+      dist+=(g0[i]+1.0);
+      for(j=0;j<i;j++) {
+	dist+=(primal[j]*g[j*n+i]);
+      }
+      for(j=i;j<n;j++) {
+	dist+=(primal[j]*g[i*n+j]);
+      }
+      if((primal[i]<(up[i]-epsilon_hideo)) && (dist < (1.0-epsilon_crit))) {
+	epsilon_hideo=(up[i]-primal[i])*2.0;
+      }
+      else if((primal[i]>(low[i]+epsilon_hideo)) &&(dist>(1.0+epsilon_crit))) {
+	epsilon_hideo=(primal[i]-low[i])*2.0;
+      }
+    }
+    /* printf("\nEPSILON_HIDEO=%.30f\n",epsilon_hideo); */
+
+    for(i=0;i<n;i++) {           /* clip alphas to bounds */
+      if(primal[i]<=(low[i]+epsilon_hideo)) {
+	primal[i]=low[i];
+      }
+      else if(primal[i]>=(up[i]-epsilon_hideo)) {
+	primal[i]=up[i];
+      }
+    }
+
+    retrain=0;
+    primal_optimal=1;
+    at_bound=0;
+    for(i=0;(i<n);i++) {  /* check primal KT-Conditions */
+      dist=-model_b*ce[i]; 
+      dist+=(g0[i]+1.0);
+      for(j=0;j<i;j++) {
+	dist+=(primal[j]*g[j*n+i]);
+      }
+      for(j=i;j<n;j++) {
+	dist+=(primal[j]*g[i*n+j]);
+      }
+      if((primal[i]<(up[i]-epsilon_a)) && (dist < (1.0-epsilon_crit))) {
+	retrain=1;
+	primal_optimal=0;
+      }
+      else if((primal[i]>(low[i]+epsilon_a)) && (dist > (1.0+epsilon_crit))) {
+	retrain=1;
+	primal_optimal=0;
+      }
+      if((primal[i]<=(low[i]+epsilon_a)) || (primal[i]>=(up[i]-epsilon_a))) {
+	at_bound++;
+      }
+      /*    printf("HIDEOtemp: a[%ld]=%.30f, dist=%.6f, b=%f, at_bound=%ld\n",i,primal[i],dist,model_b,at_bound);  */
+    }
+    if(m>0) {
+      eq=-ce0[0];               /* check precision of eq-constraint */
+      for(i=0;i<n;i++) { 
+	eq+=(ce[i]*primal[i]);
+      }
+      if((EPSILON_EQ < fabs(eq)) 
+	 /*
+	 && !((goal==PRIMAL_OPTIMAL) 
+	       && (at_bound==n)) */
+	 ) {
+	retrain=1;
+	primal_optimal=0;
+      }
+      /* printf("\n eq=%.30f ce0=%f at-bound=%ld\n",eq,ce0[0],at_bound);  */
+    }
+
+    if(retrain) {
+      precision/=10;
+      if(((goal == PRIMAL_OPTIMAL) && (maxfaktor < 50000))
+	 || (maxfaktor < 5)) {
+	maxfaktor++;
+      }
+    }
+  }
+
+  if(!primal_optimal) {
+    for(i=0;i<n;i++) {
+      primal[i]=0;             /* calc value of primal variables */
+      for(j=0;j<n;j++) {
+	primal[i]+=ig[i*n+j]*temp[j];
+      }
+      primal[i]*=-1.0;
+      if(primal[i]<=(low[i]+epsilon_a)) {  /* clip conservatively */
+	primal[i]=low[i];
+      }
+      else if(primal[i]>=(up[i]-epsilon_a)) {
+	primal[i]=up[i];
+      }
+    }
+  }
+
+  isnantest=0;
+  for(i=0;i<n;i++) {           /* check for isnan */
+    isnantest+=primal[i];
+  }
+
+  if(m>0) {
+    temp1=dual[n+n+1];   /* copy the dual variables for the eq */
+    temp2=dual[n+n];     /* constraints to a handier location */
+    for(i=n+n+1;i>=2;i--) {
+      dual[i]=dual[i-2];
+    }
+    dual[0]=temp2;
+    dual[1]=temp1;
+    isnantest+=temp1+temp2;
+  }
+
+  if(isnan(isnantest)) {
+    return((int)NAN_SOLUTION);
+  }
+  else if(primal_optimal) {
+    return((int)PRIMAL_OPTIMAL);
+  }
+  else if(maxviol == 0.0) {
+    return((int)DUAL_OPTIMAL);
+  }
+  else {
+    return((int)MAXITER_EXCEEDED);
+  }
+}
+
+
+void linvert_matrix(matrix,depth,inverse,lindep_sensitivity,lin_dependent)
+double *matrix;
+long depth;
+double *inverse,lindep_sensitivity;
+long *lin_dependent;  /* indicates the active parts of matrix on 
+			 input and output*/
+{
+  long i,j,k;
+  double factor;
+
+  for(i=0;i<depth;i++) {
+    /*    lin_dependent[i]=0; */
+    for(j=0;j<depth;j++) {
+      inverse[i*depth+j]=0.0;
+    }
+    inverse[i*depth+i]=1.0;
+  }
+  for(i=0;i<depth;i++) {
+    if(lin_dependent[i] || (fabs(matrix[i*depth+i])<lindep_sensitivity)) {
+      lin_dependent[i]=1;
+    }
+    else {
+      for(j=i+1;j<depth;j++) {
+	factor=matrix[j*depth+i]/matrix[i*depth+i];
+	for(k=i;k<depth;k++) {
+	  matrix[j*depth+k]-=(factor*matrix[i*depth+k]);
+	}
+	for(k=0;k<depth;k++) {
+	  inverse[j*depth+k]-=(factor*inverse[i*depth+k]);
+	}
+      }
+    }
+  }
+  for(i=depth-1;i>=0;i--) {
+    if(!lin_dependent[i]) {
+      factor=1/matrix[i*depth+i];
+      for(k=0;k<depth;k++) {
+	inverse[i*depth+k]*=factor;
+      }
+      matrix[i*depth+i]=1;
+      for(j=i-1;j>=0;j--) {
+	factor=matrix[j*depth+i];
+	matrix[j*depth+i]=0;
+	for(k=0;k<depth;k++) {
+	  inverse[j*depth+k]-=(factor*inverse[i*depth+k]);
+	}
+      }
+    }
+  }
+}
+
+void lprint_matrix(matrix,depth)
+double *matrix;
+long depth;
+{
+  long i,j;
+  for(i=0;i<depth;i++) {
+    for(j=0;j<depth;j++) {
+      printf("%5.2f ",(double)(matrix[i*depth+j]));
+    }
+    printf("\n");
+  }
+  printf("\n");
+}
+
+void ladd_matrix(matrix,depth,scalar)
+double *matrix;
+long depth;
+double scalar;
+{
+  long i,j;
+  for(i=0;i<depth;i++) {
+    for(j=0;j<depth;j++) {
+      matrix[i*depth+j]+=scalar;
+    }
+  }
+}
+
+void lcopy_matrix(matrix,depth,matrix2) 
+double *matrix;
+long depth;
+double *matrix2;
+{
+  long i;
+  
+  for(i=0;i<(depth)*(depth);i++) {
+    matrix2[i]=matrix[i];
+  }
+}
+
+void lswitch_rows_matrix(matrix,depth,r1,r2) 
+double *matrix;
+long depth,r1,r2;
+{
+  long i;
+  double temp;
+
+  for(i=0;i<depth;i++) {
+    temp=matrix[r1*depth+i];
+    matrix[r1*depth+i]=matrix[r2*depth+i];
+    matrix[r2*depth+i]=temp;
+  }
+}
+
+void lswitchrk_matrix(matrix,depth,rk1,rk2) 
+double *matrix;
+long depth,rk1,rk2;
+{
+  long i;
+  double temp;
+
+  for(i=0;i<depth;i++) {
+    temp=matrix[rk1*depth+i];
+    matrix[rk1*depth+i]=matrix[rk2*depth+i];
+    matrix[rk2*depth+i]=temp;
+  }
+  for(i=0;i<depth;i++) {
+    temp=matrix[i*depth+rk1];
+    matrix[i*depth+rk1]=matrix[i*depth+rk2];
+    matrix[i*depth+rk2]=temp;
+  }
+}
+
+double calculate_qp_objective(opt_n,opt_g,opt_g0,alpha)
+long opt_n;
+double *opt_g,*opt_g0,*alpha;
+{
+  double obj;
+  long i,j;
+  obj=0;  /* calculate objective  */
+  for(i=0;i<opt_n;i++) {
+    obj+=(opt_g0[i]*alpha[i]);
+    obj+=(0.5*alpha[i]*alpha[i]*opt_g[i*opt_n+i]);
+    for(j=0;j<i;j++) {
+      obj+=(alpha[j]*alpha[i]*opt_g[j*opt_n+i]);
+    }
+  }
+  return(obj);
+}