numo-liblinear 1.0.0 → 1.1.0

data/ext/numo/liblinear/liblinear/linear.h

@@ -1,7 +1,7 @@
 #ifndef _LIBLINEAR_H
 #define _LIBLINEAR_H
 
-#define LIBLINEAR_VERSION 230
+#define LIBLINEAR_VERSION 241
 
 #ifdef __cplusplus
 extern "C" {
@@ -23,7 +23,7 @@ struct problem
 	double bias;            /* < 0 if no bias term */
 };
 
-enum { L2R_LR, L2R_L2LOSS_SVC_DUAL, L2R_L2LOSS_SVC, L2R_L1LOSS_SVC_DUAL, MCSVM_CS, L1R_L2LOSS_SVC, L1R_LR, L2R_LR_DUAL, L2R_L2LOSS_SVR = 11, L2R_L2LOSS_SVR_DUAL, L2R_L1LOSS_SVR_DUAL }; /* solver_type */
+enum { L2R_LR, L2R_L2LOSS_SVC_DUAL, L2R_L2LOSS_SVC, L2R_L1LOSS_SVC_DUAL, MCSVM_CS, L1R_L2LOSS_SVC, L1R_LR, L2R_LR_DUAL, L2R_L2LOSS_SVR = 11, L2R_L2LOSS_SVR_DUAL, L2R_L1LOSS_SVR_DUAL, ONECLASS_SVM = 21 }; /* solver_type */
 
 struct parameter
 {
@@ -36,7 +36,9 @@ struct parameter
 	int *weight_label;
 	double* weight;
 	double p;
+	double nu;
 	double *init_sol;
+	int regularize_bias;
 };
 
 struct model
@@ -47,6 +49,7 @@ struct model
 	double *w;
 	int *label;		/* label of each class */
 	double bias;
+	double rho;		/* one-class SVM only */
 };
 
 struct model* train(const struct problem *prob, const struct parameter *param);
@@ -65,6 +68,7 @@ int get_nr_class(const struct model *model_);
 void get_labels(const struct model *model_, int* label);
 double get_decfun_coef(const struct model *model_, int feat_idx, int label_idx);
 double get_decfun_bias(const struct model *model_, int label_idx);
+double get_decfun_rho(const struct model *model_);
 
 void free_model_content(struct model *model_ptr);
 void free_and_destroy_model(struct model **model_ptr_ptr);
@@ -73,6 +77,7 @@ void destroy_param(struct parameter *param);
 const char *check_parameter(const struct problem *prob, const struct parameter *param);
 int check_probability_model(const struct model *model);
 int check_regression_model(const struct model *model);
+int check_oneclass_model(const struct model *model);
 void set_print_string_function(void (*print_func) (const char*));
 
 #ifdef __cplusplus

data/ext/numo/liblinear/liblinear/newton.cpp

@@ -0,0 +1,245 @@
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdarg.h>
+#include "newton.h"
+
+#ifndef min
+template <class T> static inline T min(T x,T y) { return (x<y)?x:y; }
+#endif
+
+#ifndef max
+template <class T> static inline T max(T x,T y) { return (x>y)?x:y; }
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern double dnrm2_(int *, double *, int *);
+extern double ddot_(int *, double *, int *, double *, int *);
+extern int daxpy_(int *, double *, double *, int *, double *, int *);
+extern int dscal_(int *, double *, double *, int *);
+
+#ifdef __cplusplus
+}
+#endif
+
+static void default_print(const char *buf)
+{
+	fputs(buf,stdout);
+	fflush(stdout);
+}
+
+// On entry *f must be the function value of w
+// On exit w is updated and *f is the new function value
+double function::linesearch_and_update(double *w, double *s, double *f, double *g, double alpha)
+{
+	double gTs = 0;
+	double eta = 0.01;
+	int n = get_nr_variable();
+	int max_num_linesearch = 20;
+	double *w_new = new double[n];
+	double fold = *f;
+
+	for (int i=0;i<n;i++)
+		gTs += s[i] * g[i];
+
+	int num_linesearch = 0;
+	for(num_linesearch=0; num_linesearch < max_num_linesearch; num_linesearch++)
+	{
+		for (int i=0;i<n;i++)
+			w_new[i] = w[i] + alpha*s[i];
+		*f = fun(w_new);
+		if (*f - fold <= eta * alpha * gTs)
+			break;
+		else
+			alpha *= 0.5;
+	}
+
+	if (num_linesearch >= max_num_linesearch)
+	{
+		*f = fold;
+		return 0;
+	}
+	else
+		memcpy(w, w_new, sizeof(double)*n);
+
+	delete [] w_new;
+	return alpha;
+}
+
+void NEWTON::info(const char *fmt,...)
+{
+	char buf[BUFSIZ];
+	va_list ap;
+	va_start(ap,fmt);
+	vsprintf(buf,fmt,ap);
+	va_end(ap);
+	(*newton_print_string)(buf);
+}
+
+NEWTON::NEWTON(const function *fun_obj, double eps, double eps_cg, int max_iter)
+{
+	this->fun_obj=const_cast<function *>(fun_obj);
+	this->eps=eps;
+	this->eps_cg=eps_cg;
+	this->max_iter=max_iter;
+	newton_print_string = default_print;
+}
+
+NEWTON::~NEWTON()
+{
+}
+
+void NEWTON::newton(double *w)
+{
+	int n = fun_obj->get_nr_variable();
+	int i, cg_iter;
+	double step_size;
+	double f, fold, actred;
+	double init_step_size = 1;
+	int search = 1, iter = 1, inc = 1;
+	double *s = new double[n];
+	double *r = new double[n];
+	double *g = new double[n];
+
+	const double alpha_pcg = 0.01;
+	double *M = new double[n];
+
+	// calculate gradient norm at w=0 for stopping condition.
+	double *w0 = new double[n];
+	for (i=0; i<n; i++)
+		w0[i] = 0;
+	fun_obj->fun(w0);
+	fun_obj->grad(w0, g);
+	double gnorm0 = dnrm2_(&n, g, &inc);
+	delete [] w0;
+
+	f = fun_obj->fun(w);
+	info("init f %5.3e\n", f);
+	fun_obj->grad(w, g);
+	double gnorm = dnrm2_(&n, g, &inc);
+
+	if (gnorm <= eps*gnorm0)
+		search = 0;
+
+	double *w_new = new double[n];
+	while (iter <= max_iter && search)
+	{
+		fun_obj->get_diag_preconditioner(M);
+		for(i=0; i<n; i++)
+			M[i] = (1-alpha_pcg) + alpha_pcg*M[i];
+		cg_iter = pcg(g, M, s, r);
+
+		fold = f;
+		step_size = fun_obj->linesearch_and_update(w, s, & f, g, init_step_size);
+
+		if (step_size == 0)
+		{
+			info("WARNING: line search fails\n");
+			break;
+		}
+
+		info("iter %2d f %5.3e |g| %5.3e CG %3d step_size %4.2e \n", iter, f, gnorm, cg_iter, step_size);
+
+		actred = fold - f;
+		iter++;
+
+		fun_obj->grad(w, g);
+
+		gnorm = dnrm2_(&n, g, &inc);
+		if (gnorm <= eps*gnorm0)
+			break;
+		if (f < -1.0e+32)
+		{
+			info("WARNING: f < -1.0e+32\n");
+			break;
+		}
+		if (fabs(actred) <= 1.0e-12*fabs(f))
+		{
+			info("WARNING: actred too small\n");
+			break;
+		}
+	}
+
+	delete[] g;
+	delete[] r;
+	delete[] w_new;
+	delete[] s;
+	delete[] M;
+}
+
+int NEWTON::pcg(double *g, double *M, double *s, double *r)
+{
+	int i, inc = 1;
+	int n = fun_obj->get_nr_variable();
+	double one = 1;
+	double *d = new double[n];
+	double *Hd = new double[n];
+	double zTr, znewTrnew, alpha, beta, cgtol;
+	double *z = new double[n];
+	double Q = 0, newQ, Qdiff;
+
+	for (i=0; i<n; i++)
+	{
+		s[i] = 0;
+		r[i] = -g[i];
+		z[i] = r[i] / M[i];
+		d[i] = z[i];
+	}
+
+	zTr = ddot_(&n, z, &inc, r, &inc);
+	double gMinv_norm = sqrt(zTr);
+	cgtol = min(eps_cg, sqrt(gMinv_norm));
+	int cg_iter = 0;
+	int max_cg_iter = max(n, 5);
+
+	while (cg_iter < max_cg_iter)
+	{
+		cg_iter++;
+		fun_obj->Hv(d, Hd);
+
+		alpha = zTr/ddot_(&n, d, &inc, Hd, &inc);
+		daxpy_(&n, &alpha, d, &inc, s, &inc);
+		alpha = -alpha;
+		daxpy_(&n, &alpha, Hd, &inc, r, &inc);
+
+		// Using quadratic approximation as CG stopping criterion
+		newQ = -0.5*(ddot_(&n, s, &inc, r, &inc) - ddot_(&n, s, &inc, g, &inc));
+		Qdiff = newQ - Q;
+		if (newQ <= 0 && Qdiff <= 0)
+		{
+			if (cg_iter * Qdiff >= cgtol * newQ)
+				break;
+		}
+		else
+		{
+			info("WARNING: quadratic approximation > 0 or increasing in CG\n");
+			break;
+		}
+		Q = newQ;
+
+		for (i=0; i<n; i++)
+			z[i] = r[i] / M[i];
+		znewTrnew = ddot_(&n, z, &inc, r, &inc);
+		beta = znewTrnew/zTr;
+		dscal_(&n, &beta, d, &inc);
+		daxpy_(&n, &one, z, &inc, d, &inc);
+		zTr = znewTrnew;
+	}
+
+	if (cg_iter == max_cg_iter)
+		info("WARNING: reaching maximal number of CG steps\n");
+
+	delete[] d;
+	delete[] Hd;
+	delete[] z;
+
+	return(cg_iter);
+}
+
+void NEWTON::set_print_string(void (*print_string) (const char *buf))
+{
+	newton_print_string = print_string;
+}

data/ext/numo/liblinear/liblinear/{tron.h → newton.h}

@@ -1,5 +1,5 @@
-#ifndef _TRON_H
-#define _TRON_H
+#ifndef _NEWTON_H
+#define _NEWTON_H
 
 class function
 {
@@ -7,30 +7,31 @@ public:
 	virtual double fun(double *w) = 0 ;
 	virtual void grad(double *w, double *g) = 0 ;
 	virtual void Hv(double *s, double *Hs) = 0 ;
-
 	virtual int get_nr_variable(void) = 0 ;
 	virtual void get_diag_preconditioner(double *M) = 0 ;
 	virtual ~function(void){}
+
+	// base implementation in newton.cpp
+	virtual double linesearch_and_update(double *w, double *s, double *f, double *g, double alpha);
 };
 
-class TRON
+class NEWTON
 {
 public:
-	TRON(const function *fun_obj, double eps = 0.1, double eps_cg = 0.1, int max_iter = 1000);
-	~TRON();
+	NEWTON(const function *fun_obj, double eps = 0.1, double eps_cg = 0.5, int max_iter = 1000);
+	~NEWTON();
 
-	void tron(double *w);
+	void newton(double *w);
 	void set_print_string(void (*i_print) (const char *buf));
 
 private:
-	int trpcg(double delta, double *g, double *M, double *s, double *r, bool *reach_boundary);
-	double norm_inf(int n, double *x);
+	int pcg(double *g, double *M, double *s, double *r);
 
 	double eps;
 	double eps_cg;
 	int max_iter;
 	function *fun_obj;
 	void info(const char *fmt,...);
-	void (*tron_print_string)(const char *buf);
+	void (*newton_print_string)(const char *buf);
 };
 #endif

data/ext/numo/liblinear/model.c

@@ -14,6 +14,8 @@ struct model* rb_hash_to_model(VALUE model_hash)
   model->label = nary_to_int_vec(el);
   el = rb_hash_aref(model_hash, ID2SYM(rb_intern("bias")));
   model->bias = NUM2DBL(el);
+  el = rb_hash_aref(model_hash, ID2SYM(rb_intern("rho")));
+  model->rho = NUM2DBL(el);
   return model;
 }
 
@@ -29,6 +31,7 @@ VALUE model_to_rb_hash(struct model* const model)
   rb_hash_aset(model_hash, ID2SYM(rb_intern("label")),
       model->label ? int_vec_to_nary(model->label, model->nr_class) : Qnil);
   rb_hash_aset(model_hash, ID2SYM(rb_intern("bias")), DBL2NUM(model->bias));
+  rb_hash_aset(model_hash, ID2SYM(rb_intern("rho")), DBL2NUM(model->rho));
   return model_hash;
 }
 

data/ext/numo/liblinear/parameter.c

@@ -5,36 +5,39 @@ struct parameter* rb_hash_to_parameter(VALUE param_hash)
   VALUE el;
   struct parameter* param = ALLOC(struct parameter);
   el = rb_hash_aref(param_hash, ID2SYM(rb_intern("solver_type")));
-	param->solver_type = !NIL_P(el) ? NUM2INT(el) : L2R_L2LOSS_SVC_DUAL;
+  param->solver_type = !NIL_P(el) ? NUM2INT(el) : L2R_L2LOSS_SVC_DUAL;
   el = rb_hash_aref(param_hash, ID2SYM(rb_intern("eps")));
-	if (!NIL_P(el)) {
+  if (!NIL_P(el)) {
     param->eps = NUM2DBL(el);
-	} else {
-		switch(param->solver_type)
-		{
-			case L2R_LR:
-			case L2R_L2LOSS_SVC:
-				param->eps = 0.01;
-				break;
-			case L2R_L2LOSS_SVR:
-				param->eps = 0.0001;
-				break;
-			case L2R_L2LOSS_SVC_DUAL:
-			case L2R_L1LOSS_SVC_DUAL:
-			case MCSVM_CS:
-			case L2R_LR_DUAL:
-				param->eps = 0.1;
-				break;
-			case L1R_L2LOSS_SVC:
-			case L1R_LR:
-				param->eps = 0.01;
-				break;
-			case L2R_L1LOSS_SVR_DUAL:
-			case L2R_L2LOSS_SVR_DUAL:
-				param->eps = 0.1;
-				break;
+  } else {
+    switch(param->solver_type)
+    {
+      case L2R_LR:
+      case L2R_L2LOSS_SVC:
+        param->eps = 0.01;
+        break;
+      case L2R_L2LOSS_SVR:
+        param->eps = 0.0001;
+        break;
+      case L2R_L2LOSS_SVC_DUAL:
+      case L2R_L1LOSS_SVC_DUAL:
+      case MCSVM_CS:
+      case L2R_LR_DUAL:
+        param->eps = 0.1;
+        break;
+      case L1R_L2LOSS_SVC:
+      case L1R_LR:
+        param->eps = 0.01;
+        break;
+      case L2R_L1LOSS_SVR_DUAL:
+      case L2R_L2LOSS_SVR_DUAL:
+        param->eps = 0.1;
+        break;
+      case ONECLASS_SVM:
+        param->eps = 0.01;
+        break;
     }
-	}
+  }
   el = rb_hash_aref(param_hash, ID2SYM(rb_intern("C")));
   param->C = !NIL_P(el) ? NUM2DBL(el) : 1;
   el = rb_hash_aref(param_hash, ID2SYM(rb_intern("nr_weight")));
@@ -53,11 +56,14 @@ struct parameter* rb_hash_to_parameter(VALUE param_hash)
   }
   el = rb_hash_aref(param_hash, ID2SYM(rb_intern("p")));
   param->p = !NIL_P(el) ? NUM2DBL(el) : 0.1;
+  el = rb_hash_aref(param_hash, ID2SYM(rb_intern("nu")));
+  param->nu = !NIL_P(el) ? NUM2DBL(el) : 0.5;
   el = rb_hash_aref(param_hash, ID2SYM(rb_intern("init_sol")));
   param->init_sol = NULL;
   if (!NIL_P(el)) {
     param->init_sol = nary_to_dbl_vec(el);
   }
+  param->regularize_bias = 1;
   return param;
 }
 
@@ -73,6 +79,7 @@ VALUE parameter_to_rb_hash(struct parameter* const param)
   rb_hash_aset(param_hash, ID2SYM(rb_intern("weight")),
     param->weight ? dbl_vec_to_nary(param->weight, param->nr_weight) : Qnil);
   rb_hash_aset(param_hash, ID2SYM(rb_intern("p")), DBL2NUM(param->p));
+  rb_hash_aset(param_hash, ID2SYM(rb_intern("nu")), DBL2NUM(param->nu));
   rb_hash_aset(param_hash, ID2SYM(rb_intern("init_sol")), Qnil);
   return param_hash;
 }

data/ext/numo/liblinear/solver_type.c

@@ -17,18 +17,20 @@ void rb_init_solver_type_module()
   rb_define_const(mSolverType, "L2R_L2LOSS_SVC", INT2NUM(L2R_L2LOSS_SVC));
   /* L2-regularized L1-loss support vector classification (dual) */
   rb_define_const(mSolverType, "L2R_L1LOSS_SVC_DUAL", INT2NUM(L2R_L1LOSS_SVC_DUAL));
-	/* support vector classification by Crammer and Singer */
+  /* support vector classification by Crammer and Singer */
   rb_define_const(mSolverType, "MCSVM_CS", INT2NUM(MCSVM_CS));
-	/* L1-regularized L2-loss support vector classification */
+  /* L1-regularized L2-loss support vector classification */
   rb_define_const(mSolverType, "L1R_L2LOSS_SVC", INT2NUM(L1R_L2LOSS_SVC));
-	/* L1-regularized logistic regression */
+  /* L1-regularized logistic regression */
   rb_define_const(mSolverType, "L1R_LR", INT2NUM(L1R_LR));
   /* L2-regularized logistic regression (dual) */
   rb_define_const(mSolverType, "L2R_LR_DUAL", INT2NUM(L2R_LR_DUAL));
-	/* L2-regularized L2-loss support vector regression (primal) */
+  /* L2-regularized L2-loss support vector regression (primal) */
   rb_define_const(mSolverType, "L2R_L2LOSS_SVR", INT2NUM(L2R_L2LOSS_SVR));
-	/* L2-regularized L2-loss support vector regression (dual) */
+  /* L2-regularized L2-loss support vector regression (dual) */
   rb_define_const(mSolverType, "L2R_L2LOSS_SVR_DUAL", INT2NUM(L2R_L2LOSS_SVR_DUAL));
-	/* L2-regularized L1-loss support vector regression (dual) */
+  /* L2-regularized L1-loss support vector regression (dual) */
   rb_define_const(mSolverType, "L2R_L1LOSS_SVR_DUAL", INT2NUM(L2R_L1LOSS_SVR_DUAL));
+  /* one-class support vector machine (dual) */
+  rb_define_const(mSolverType, "ONECLASS_SVM", INT2NUM(ONECLASS_SVM));
 }