numo-liblinear 0.2.0 → 1.1.0

data/ext/numo/liblinear/liblinear/linear.h

@@ -0,0 +1,88 @@
+#ifndef _LIBLINEAR_H
+#define _LIBLINEAR_H
+
+#define LIBLINEAR_VERSION 241
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int liblinear_version;
+
+struct feature_node
+{
+	int index;
+	double value;
+};
+
+struct problem
+{
+	int l, n;
+	double *y;
+	struct feature_node **x;
+	double bias;            /* < 0 if no bias term */
+};
+
+enum { L2R_LR, L2R_L2LOSS_SVC_DUAL, L2R_L2LOSS_SVC, L2R_L1LOSS_SVC_DUAL, MCSVM_CS, L1R_L2LOSS_SVC, L1R_LR, L2R_LR_DUAL, L2R_L2LOSS_SVR = 11, L2R_L2LOSS_SVR_DUAL, L2R_L1LOSS_SVR_DUAL, ONECLASS_SVM = 21 }; /* solver_type */
+
+struct parameter
+{
+	int solver_type;
+
+	/* these are for training only */
+	double eps;	        /* stopping criteria */
+	double C;
+	int nr_weight;
+	int *weight_label;
+	double* weight;
+	double p;
+	double nu;
+	double *init_sol;
+	int regularize_bias;
+};
+
+struct model
+{
+	struct parameter param;
+	int nr_class;		/* number of classes */
+	int nr_feature;
+	double *w;
+	int *label;		/* label of each class */
+	double bias;
+	double rho;		/* one-class SVM only */
+};
+
+struct model* train(const struct problem *prob, const struct parameter *param);
+void cross_validation(const struct problem *prob, const struct parameter *param, int nr_fold, double *target);
+void find_parameters(const struct problem *prob, const struct parameter *param, int nr_fold, double start_C, double start_p, double *best_C, double *best_p, double *best_score);
+
+double predict_values(const struct model *model_, const struct feature_node *x, double* dec_values);
+double predict(const struct model *model_, const struct feature_node *x);
+double predict_probability(const struct model *model_, const struct feature_node *x, double* prob_estimates);
+
+int save_model(const char *model_file_name, const struct model *model_);
+struct model *load_model(const char *model_file_name);
+
+int get_nr_feature(const struct model *model_);
+int get_nr_class(const struct model *model_);
+void get_labels(const struct model *model_, int* label);
+double get_decfun_coef(const struct model *model_, int feat_idx, int label_idx);
+double get_decfun_bias(const struct model *model_, int label_idx);
+double get_decfun_rho(const struct model *model_);
+
+void free_model_content(struct model *model_ptr);
+void free_and_destroy_model(struct model **model_ptr_ptr);
+void destroy_param(struct parameter *param);
+
+const char *check_parameter(const struct problem *prob, const struct parameter *param);
+int check_probability_model(const struct model *model);
+int check_regression_model(const struct model *model);
+int check_oneclass_model(const struct model *model);
+void set_print_string_function(void (*print_func) (const char*));
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _LIBLINEAR_H */
+

data/ext/numo/liblinear/liblinear/newton.cpp

@@ -0,0 +1,245 @@
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdarg.h>
+#include "newton.h"
+
+#ifndef min
+template <class T> static inline T min(T x,T y) { return (x<y)?x:y; }
+#endif
+
+#ifndef max
+template <class T> static inline T max(T x,T y) { return (x>y)?x:y; }
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern double dnrm2_(int *, double *, int *);
+extern double ddot_(int *, double *, int *, double *, int *);
+extern int daxpy_(int *, double *, double *, int *, double *, int *);
+extern int dscal_(int *, double *, double *, int *);
+
+#ifdef __cplusplus
+}
+#endif
+
+static void default_print(const char *buf)
+{
+	fputs(buf,stdout);
+	fflush(stdout);
+}
+
+// On entry *f must be the function value of w
+// On exit w is updated and *f is the new function value
+double function::linesearch_and_update(double *w, double *s, double *f, double *g, double alpha)
+{
+	double gTs = 0;
+	double eta = 0.01;
+	int n = get_nr_variable();
+	int max_num_linesearch = 20;
+	double *w_new = new double[n];
+	double fold = *f;
+
+	for (int i=0;i<n;i++)
+		gTs += s[i] * g[i];
+
+	int num_linesearch = 0;
+	for(num_linesearch=0; num_linesearch < max_num_linesearch; num_linesearch++)
+	{
+		for (int i=0;i<n;i++)
+			w_new[i] = w[i] + alpha*s[i];
+		*f = fun(w_new);
+		if (*f - fold <= eta * alpha * gTs)
+			break;
+		else
+			alpha *= 0.5;
+	}
+
+	if (num_linesearch >= max_num_linesearch)
+	{
+		*f = fold;
+		return 0;
+	}
+	else
+		memcpy(w, w_new, sizeof(double)*n);
+
+	delete [] w_new;
+	return alpha;
+}
+
+void NEWTON::info(const char *fmt,...)
+{
+	char buf[BUFSIZ];
+	va_list ap;
+	va_start(ap,fmt);
+	vsprintf(buf,fmt,ap);
+	va_end(ap);
+	(*newton_print_string)(buf);
+}
+
+NEWTON::NEWTON(const function *fun_obj, double eps, double eps_cg, int max_iter)
+{
+	this->fun_obj=const_cast<function *>(fun_obj);
+	this->eps=eps;
+	this->eps_cg=eps_cg;
+	this->max_iter=max_iter;
+	newton_print_string = default_print;
+}
+
+NEWTON::~NEWTON()
+{
+}
+
+void NEWTON::newton(double *w)
+{
+	int n = fun_obj->get_nr_variable();
+	int i, cg_iter;
+	double step_size;
+	double f, fold, actred;
+	double init_step_size = 1;
+	int search = 1, iter = 1, inc = 1;
+	double *s = new double[n];
+	double *r = new double[n];
+	double *g = new double[n];
+
+	const double alpha_pcg = 0.01;
+	double *M = new double[n];
+
+	// calculate gradient norm at w=0 for stopping condition.
+	double *w0 = new double[n];
+	for (i=0; i<n; i++)
+		w0[i] = 0;
+	fun_obj->fun(w0);
+	fun_obj->grad(w0, g);
+	double gnorm0 = dnrm2_(&n, g, &inc);
+	delete [] w0;
+
+	f = fun_obj->fun(w);
+	info("init f %5.3e\n", f);
+	fun_obj->grad(w, g);
+	double gnorm = dnrm2_(&n, g, &inc);
+
+	if (gnorm <= eps*gnorm0)
+		search = 0;
+
+	double *w_new = new double[n];
+	while (iter <= max_iter && search)
+	{
+		fun_obj->get_diag_preconditioner(M);
+		for(i=0; i<n; i++)
+			M[i] = (1-alpha_pcg) + alpha_pcg*M[i];
+		cg_iter = pcg(g, M, s, r);
+
+		fold = f;
+		step_size = fun_obj->linesearch_and_update(w, s, & f, g, init_step_size);
+
+		if (step_size == 0)
+		{
+			info("WARNING: line search fails\n");
+			break;
+		}
+
+		info("iter %2d f %5.3e |g| %5.3e CG %3d step_size %4.2e \n", iter, f, gnorm, cg_iter, step_size);
+
+		actred = fold - f;
+		iter++;
+
+		fun_obj->grad(w, g);
+
+		gnorm = dnrm2_(&n, g, &inc);
+		if (gnorm <= eps*gnorm0)
+			break;
+		if (f < -1.0e+32)
+		{
+			info("WARNING: f < -1.0e+32\n");
+			break;
+		}
+		if (fabs(actred) <= 1.0e-12*fabs(f))
+		{
+			info("WARNING: actred too small\n");
+			break;
+		}
+	}
+
+	delete[] g;
+	delete[] r;
+	delete[] w_new;
+	delete[] s;
+	delete[] M;
+}
+
+int NEWTON::pcg(double *g, double *M, double *s, double *r)
+{
+	int i, inc = 1;
+	int n = fun_obj->get_nr_variable();
+	double one = 1;
+	double *d = new double[n];
+	double *Hd = new double[n];
+	double zTr, znewTrnew, alpha, beta, cgtol;
+	double *z = new double[n];
+	double Q = 0, newQ, Qdiff;
+
+	for (i=0; i<n; i++)
+	{
+		s[i] = 0;
+		r[i] = -g[i];
+		z[i] = r[i] / M[i];
+		d[i] = z[i];
+	}
+
+	zTr = ddot_(&n, z, &inc, r, &inc);
+	double gMinv_norm = sqrt(zTr);
+	cgtol = min(eps_cg, sqrt(gMinv_norm));
+	int cg_iter = 0;
+	int max_cg_iter = max(n, 5);
+
+	while (cg_iter < max_cg_iter)
+	{
+		cg_iter++;
+		fun_obj->Hv(d, Hd);
+
+		alpha = zTr/ddot_(&n, d, &inc, Hd, &inc);
+		daxpy_(&n, &alpha, d, &inc, s, &inc);
+		alpha = -alpha;
+		daxpy_(&n, &alpha, Hd, &inc, r, &inc);
+
+		// Using quadratic approximation as CG stopping criterion
+		newQ = -0.5*(ddot_(&n, s, &inc, r, &inc) - ddot_(&n, s, &inc, g, &inc));
+		Qdiff = newQ - Q;
+		if (newQ <= 0 && Qdiff <= 0)
+		{
+			if (cg_iter * Qdiff >= cgtol * newQ)
+				break;
+		}
+		else
+		{
+			info("WARNING: quadratic approximation > 0 or increasing in CG\n");
+			break;
+		}
+		Q = newQ;
+
+		for (i=0; i<n; i++)
+			z[i] = r[i] / M[i];
+		znewTrnew = ddot_(&n, z, &inc, r, &inc);
+		beta = znewTrnew/zTr;
+		dscal_(&n, &beta, d, &inc);
+		daxpy_(&n, &one, z, &inc, d, &inc);
+		zTr = znewTrnew;
+	}
+
+	if (cg_iter == max_cg_iter)
+		info("WARNING: reaching maximal number of CG steps\n");
+
+	delete[] d;
+	delete[] Hd;
+	delete[] z;
+
+	return(cg_iter);
+}
+
+void NEWTON::set_print_string(void (*print_string) (const char *buf))
+{
+	newton_print_string = print_string;
+}

data/ext/numo/liblinear/liblinear/newton.h

@@ -0,0 +1,37 @@
+#ifndef _NEWTON_H
+#define _NEWTON_H
+
+class function
+{
+public:
+	virtual double fun(double *w) = 0 ;
+	virtual void grad(double *w, double *g) = 0 ;
+	virtual void Hv(double *s, double *Hs) = 0 ;
+	virtual int get_nr_variable(void) = 0 ;
+	virtual void get_diag_preconditioner(double *M) = 0 ;
+	virtual ~function(void){}
+
+	// base implementation in newton.cpp
+	virtual double linesearch_and_update(double *w, double *s, double *f, double *g, double alpha);
+};
+
+class NEWTON
+{
+public:
+	NEWTON(const function *fun_obj, double eps = 0.1, double eps_cg = 0.5, int max_iter = 1000);
+	~NEWTON();
+
+	void newton(double *w);
+	void set_print_string(void (*i_print) (const char *buf));
+
+private:
+	int pcg(double *g, double *M, double *s, double *r);
+
+	double eps;
+	double eps_cg;
+	int max_iter;
+	function *fun_obj;
+	void info(const char *fmt,...);
+	void (*newton_print_string)(const char *buf);
+};
+#endif

data/ext/numo/liblinear/liblinearext.c

@@ -16,6 +16,28 @@ void print_null(const char *s) {}
  *   @param y [Numo::DFloat] (shape: [n_samples]) The labels or target values for samples.
  *   @param param [Hash] The parameters of a model.
  *
+ * @example
+ *   require 'numo/liblinear'
+ *
+ *   # Prepare training dataset.
+ *   x = Numo::DFloat[[-0.8, 1.0], [-0.5, 0.8], [0.9, -0.8], [0.8, -0.7]]
+ *   y = Numo::Int32[-1, -1, 1, 1]
+ *
+ *   # Train L2-regularized L2-loss support vector classifier.
+ *   param = {
+ *     solver_type: Numo::Liblinear::SolverType::L2R_L2LOSS_SVC_DUAL,
+ *     C: 0.1,
+ *     random_seed: 1
+ *   }
+ *   model = Numo::Liblinear.train(x, y, param)
+ *
+ *   # Predict labels of test data.
+ *   x_test = Numo::DFloat[[-0.7, 0.9], [0.5, -0.4]]
+ *   result = Numo::Liblinear.predict(x_test, param, model)
+ *   p result
+ *   # Numo::DFloat#shape=[2]
+ *   # [-1, 1]
+ *
  * @raise [ArgumentError] If the sample array is not 2-dimensional, the label array is not 1-dimensional,
  *   the sample array and label array do not have the same number of samples, or
  *   the hyperparameter has an invalid value, this error is raised.
@@ -30,6 +52,8 @@ VALUE numo_liblinear_train(VALUE self, VALUE x_val, VALUE y_val, VALUE param_has
   narray_t* x_nary;
   narray_t* y_nary;
   char* err_msg;
+  VALUE random_seed;
+  VALUE verbose;
   VALUE model_hash;
 
   if (CLASS_OF(x_val) != numo_cDFloat) {
@@ -60,6 +84,11 @@ VALUE numo_liblinear_train(VALUE self, VALUE x_val, VALUE y_val, VALUE param_has
     return Qnil;
   }
 
+  random_seed = rb_hash_aref(param_hash, ID2SYM(rb_intern("random_seed")));
+  if (!NIL_P(random_seed)) {
+    srand(NUM2UINT(random_seed));
+  }
+
   param = rb_hash_to_parameter(param_hash);
   problem = dataset_to_problem(x_val, y_val);
 
@@ -71,7 +100,11 @@ VALUE numo_liblinear_train(VALUE self, VALUE x_val, VALUE y_val, VALUE param_has
     return Qnil;
   }
 
-  set_print_string_function(print_null);
+  verbose = rb_hash_aref(param_hash, ID2SYM(rb_intern("verbose")));
+  if (verbose != Qtrue) {
+    set_print_string_function(print_null);
+  }
+
   model = train(problem, param);
   model_hash = model_to_rb_hash(model);
   free_and_destroy_model(&model);
@@ -92,6 +125,28 @@ VALUE numo_liblinear_train(VALUE self, VALUE x_val, VALUE y_val, VALUE param_has
  *   @param param [Hash] The parameters of a model.
  *   @param n_folds [Integer] The number of folds.
  *
+ * @example
+ *   require 'numo/liblinear'
+ *
+ *   # x: samples
+ *   # y: labels
+ *
+ *   # Define parameters of L2-regularized L2-loss support vector classification.
+ *   param = {
+ *     solver_type: Numo::Liblinear::SolverType::L2R_L2LOSS_SVC_DUAL,
+ *     C: 1,
+ *     random_seed: 1,
+ *     verbose: true
+ *   }
+ *
+ *   # Perform 5-cross validation.
+ *   n_folds = 5
+ *   res = Numo::Liblinear::cv(x, y, param, n_folds)
+ *
+ *   # Print mean accuracy.
+ *   mean_accuracy = y.eq(res).count.fdiv(y.size)
+ *   puts "Accuracy: %.1f %%" % (100 * mean_accuracy)
+ *
  * @raise [ArgumentError] If the sample array is not 2-dimensional, the label array is not 1-dimensional,
  *   the sample array and label array do not have the same number of samples, or
  *   the hyperparameter has an invalid value, this error is raised.
@@ -107,6 +162,8 @@ VALUE numo_liblinear_cross_validation(VALUE self, VALUE x_val, VALUE y_val, VALU
   narray_t* x_nary;
   narray_t* y_nary;
   char* err_msg;
+  VALUE random_seed;
+  VALUE verbose;
   struct problem* problem;
   struct parameter* param;
 
@@ -138,6 +195,11 @@ VALUE numo_liblinear_cross_validation(VALUE self, VALUE x_val, VALUE y_val, VALU
     return Qnil;
   }
 
+  random_seed = rb_hash_aref(param_hash, ID2SYM(rb_intern("random_seed")));
+  if (!NIL_P(random_seed)) {
+    srand(NUM2UINT(random_seed));
+  }
+
   param = rb_hash_to_parameter(param_hash);
   problem = dataset_to_problem(x_val, y_val);
 
@@ -153,7 +215,11 @@ VALUE numo_liblinear_cross_validation(VALUE self, VALUE x_val, VALUE y_val, VALU
   t_val = rb_narray_new(numo_cDFloat, 1, t_shape);
   t_pt = (double*)na_get_pointer_for_write(t_val);
 
-  set_print_string_function(print_null);
+  verbose = rb_hash_aref(param_hash, ID2SYM(rb_intern("verbose")));
+  if (verbose != Qtrue) {
+    set_print_string_function(print_null);
+  }
+
   cross_validation(problem, param, n_folds, t_pt);
 
   xfree_problem(problem);
@@ -216,18 +282,12 @@ VALUE numo_liblinear_predict(VALUE self, VALUE x_val, VALUE param_hash, VALUE mo
   x_pt = (double*)na_get_pointer_for_read(x_val);
 
   /* Predict values. */
-  x_nodes = ALLOC_N(struct feature_node, n_features + 1);
-  x_nodes[n_features].index = -1;
-  x_nodes[n_features].value = 0.0;
   for (i = 0; i < n_samples; i++) {
-    for (j = 0; j < n_features; j++) {
-      x_nodes[j].index = j + 1;
-      x_nodes[j].value = (double)x_pt[i * n_features + j];
-    }
+    x_nodes = dbl_vec_to_node(&x_pt[i * n_features], n_features);
     y_pt[i] = predict(model, x_nodes);
+    xfree(x_nodes);
   }
 
-  xfree(x_nodes);
   xfree_model(model);
   xfree_parameter(param);
 
@@ -299,34 +359,22 @@ VALUE numo_liblinear_decision_function(VALUE self, VALUE x_val, VALUE param_hash
 
   /* Predict values. */
   if (model->nr_class == 2 && model->param.solver_type != MCSVM_CS) {
-    x_nodes = ALLOC_N(struct feature_node, n_features + 1);
-    x_nodes[n_features].index = -1;
-    x_nodes[n_features].value = 0.0;
     for (i = 0; i < n_samples; i++) {
-      for (j = 0; j < n_features; j++) {
-        x_nodes[j].index = j + 1;
-        x_nodes[j].value = (double)x_pt[i * n_features + j];
-      }
+      x_nodes = dbl_vec_to_node(&x_pt[i * n_features], n_features);
       predict_values(model, x_nodes, &y_pt[i]);
+      xfree(x_nodes);
     }
-    xfree(x_nodes);
   } else {
     y_cols = (int)y_shape[1];
     dec_values = ALLOC_N(double, y_cols);
-    x_nodes = ALLOC_N(struct feature_node, n_features + 1);
-    x_nodes[n_features].index = -1;
-    x_nodes[n_features].value = 0.0;
     for (i = 0; i < n_samples; i++) {
-      for (j = 0; j < n_features; j++) {
-        x_nodes[j].index = j + 1;
-        x_nodes[j].value = (double)x_pt[i * n_features + j];
-      }
+      x_nodes = dbl_vec_to_node(&x_pt[i * n_features], n_features);
       predict_values(model, x_nodes, dec_values);
+      xfree(x_nodes);
       for (j = 0; j < y_cols; j++) {
         y_pt[i * y_cols + j] = dec_values[j];
       }
     }
-    xfree(x_nodes);
     xfree(dec_values);
   }
 
@@ -395,20 +443,14 @@ VALUE numo_liblinear_predict_proba(VALUE self, VALUE x_val, VALUE param_hash, VA
 
     /* Predict values. */
     probs = ALLOC_N(double, model->nr_class);
-    x_nodes = ALLOC_N(struct feature_node, n_features + 1);
-    x_nodes[n_features].index = -1;
-    x_nodes[n_features].value = 0.0;
     for (i = 0; i < n_samples; i++) {
-      for (j = 0; j < n_features; j++) {
-        x_nodes[j].index = j + 1;
-        x_nodes[j].value = (double)x_pt[i * n_features + j];
-      }
+      x_nodes = dbl_vec_to_node(&x_pt[i * n_features], n_features);
       predict_probability(model, x_nodes, probs);
+      xfree(x_nodes);
       for (j = 0; j < model->nr_class; j++) {
         y_pt[i * model->nr_class + j] = probs[j];
       }
     }
-    xfree(x_nodes);
     xfree(probs);
   }
 
@@ -503,6 +545,9 @@ void Init_liblinearext()
    */
   mLiblinear = rb_define_module_under(mNumo, "Liblinear");
 
+  /* The version of LIBLINEAR used in backgroud library. */
+  rb_define_const(mLiblinear, "LIBLINEAR_VERSION", INT2NUM(LIBLINEAR_VERSION));
+
   rb_define_module_function(mLiblinear, "train", numo_liblinear_train, 3);
   rb_define_module_function(mLiblinear, "cv", numo_liblinear_cross_validation, 4);
   rb_define_module_function(mLiblinear, "predict", numo_liblinear_predict, 3);