liblinear-ruby 0.0.6 → 0.0.7

data/{liblinear-1.95 → liblinear-2.1}/linear.def

@@ -19,3 +19,4 @@ EXPORTS
     get_decfun_coef @17
     get_decfun_bias @18
     check_regression_model  @19
+    find_parameter_C @20

data/{liblinear-1.95 → liblinear-2.1}/linear.h

@@ -32,6 +32,7 @@ struct parameter
 	int *weight_label;
 	double* weight;
 	double p;
+	double *init_sol;
 };
 
 struct model
@@ -46,6 +47,7 @@ struct model
 
 struct model* train(const struct problem *prob, const struct parameter *param);
 void cross_validation(const struct problem *prob, const struct parameter *param, int nr_fold, double *target);
+void find_parameter_C(const struct problem *prob, const struct parameter *param, int nr_fold, double start_C, double max_C, double *best_C, double *best_rate);
 
 double predict_values(const struct model *model_, const struct feature_node *x, double* dec_values);
 double predict(const struct model *model_, const struct feature_node *x);

data/{liblinear-1.95 → liblinear-2.1}/matlab/README

@@ -117,7 +117,7 @@ The 'train' function returns a model which can be used for future
 prediction.  It is a structure and is organized as [Parameters, nr_class,
 nr_feature, bias, Label, w]:
 
-        -Parameters: Parameters
+        -Parameters: Parameters (now only solver type is provided)
         -nr_class: number of classes; = 2 for regression
         -nr_feature: number of features in training data (without including the bias term)
         -bias: If >= 0, we assume one additional feature is added to the end
@@ -131,7 +131,12 @@ nr_feature, bias, Label, w]:
 
 If the '-v' option is specified, cross validation is conducted and the
 returned model is just a scalar: cross-validation accuracy for 
-classification and mean-squared error for regression.
+classification and mean-squared error for regression. If the '-C' option
+is specified, the best parameter C is found by cross validation. The 
+returned model is a two dimensional vector, where the first value is 
+the best C and the second value is the corresponding cross-validation 
+accuracy. The parameter selection utility is supported by only -s 0
+and -s 2.
 
 Result of Prediction
 ====================
@@ -184,6 +189,11 @@ For probability estimates, you need '-b 1' only in the testing phase:
 
 matlab> [predict_label, accuracy, prob_estimates] = predict(heart_scale_label, heart_scale_inst, model, '-b 1');
 
+Use the best parameter to train (only supported by -s 0 and -s 2):
+
+matlab> best = train(heart_scale_label, heart_scale_inst, '-C -s 0');
+matlab> model = train(heart_scale_label, heart_scale_inst, sprintf('-c %f -s 0', best(1))); % use the same solver: -s 0 
+
 Additional Information
 ======================
 

data/{liblinear-1.95 → liblinear-2.1}/matlab/libsvmwrite.c

@@ -72,7 +72,7 @@ void libsvmwrite(const char *filename, const mxArray *label_vec, const mxArray *
 
 		low = jc[i], high = jc[i+1];
 		for(k=low;k<high;k++)
-			fprintf(fp," %zu:%g", (size_t)ir[k]+1, samples[k]);		
+			fprintf(fp," %lu:%g", (size_t)ir[k]+1, samples[k]);		
 
 		fprintf(fp,"\n");
 	}

data/{liblinear-1.95 → liblinear-2.1}/matlab/linear_model_matlab.c

@@ -1,6 +1,6 @@
 #include <stdlib.h>
 #include <string.h>
-#include "../linear.h"
+#include "linear.h"
 
 #include "mex.h"
 

data/liblinear-2.1/matlab/make.m

@@ -0,0 +1,22 @@
+% This make.m is for MATLAB and OCTAVE under Windows, Mac, and Unix
+function make()
+try
+	% This part is for OCTAVE
+	if(exist('OCTAVE_VERSION', 'builtin'))
+		mex libsvmread.c
+		mex libsvmwrite.c
+		mex -I.. train.c linear_model_matlab.c ../linear.cpp ../tron.cpp ../blas/daxpy.c ../blas/ddot.c ../blas/dnrm2.c ../blas/dscal.c
+		mex -I.. predict.c linear_model_matlab.c ../linear.cpp ../tron.cpp ../blas/daxpy.c ../blas/ddot.c ../blas/dnrm2.c ../blas/dscal.c
+	% This part is for MATLAB
+	% Add -largeArrayDims on 64-bit machines of MATLAB
+	else
+		mex CFLAGS="\$CFLAGS -std=c99" -largeArrayDims libsvmread.c
+		mex CFLAGS="\$CFLAGS -std=c99" -largeArrayDims libsvmwrite.c
+		mex CFLAGS="\$CFLAGS -std=c99" -I.. -largeArrayDims train.c linear_model_matlab.c ../linear.cpp ../tron.cpp ../blas/daxpy.c ../blas/ddot.c ../blas/dnrm2.c ../blas/dscal.c
+		mex CFLAGS="\$CFLAGS -std=c99" -I.. -largeArrayDims predict.c linear_model_matlab.c ../linear.cpp ../tron.cpp ../blas/daxpy.c ../blas/ddot.c ../blas/dnrm2.c ../blas/dscal.c
+	end
+catch err
+	fprintf('Error: %s failed (line %d)\n', err.stack(1).file, err.stack(1).line);
+	disp(err.message);
+	fprintf('=> Please check README for detailed instructions.\n');
+end

data/{liblinear-1.95 → liblinear-2.1}/matlab/predict.c

@@ -1,7 +1,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include "../linear.h"
+#include "linear.h"
 
 #include "mex.h"
 #include "linear_model_matlab.h"

data/{liblinear-1.95 → liblinear-2.1}/matlab/train.c

@@ -1,9 +1,8 @@
-#include <stdio.h>
 #include <math.h>
 #include <stdlib.h>
 #include <string.h>
 #include <ctype.h>
-#include "../linear.h"
+#include "linear.h"
 
 #include "mex.h"
 #include "linear_model_matlab.h"
@@ -60,6 +59,7 @@ void exit_with_help()
 	"-B bias : if bias >= 0, instance x becomes [x; bias]; if < 0, no bias term added (default -1)\n"
 	"-wi weight: weights adjust the parameter C of different classes (see README for details)\n"
 	"-v n: n-fold cross validation mode\n"
+	"-C : find parameter C (only for -s 0 and 2)\n"
 	"-q : quiet mode (no outputs)\n"
 	"col:\n"
 	"	if 'col' is setted, training_instance_matrix is parsed in column format, otherwise is in row format\n"
@@ -71,11 +71,28 @@ struct parameter param;		// set by parse_command_line
 struct problem prob;		// set by read_problem
 struct model *model_;
 struct feature_node *x_space;
-int cross_validation_flag;
+int flag_cross_validation;
+int flag_find_C;
+int flag_C_specified;
+int flag_solver_specified;
 int col_format_flag;
 int nr_fold;
 double bias;
 
+
+void do_find_parameter_C(double *best_C, double *best_rate)
+{
+	double start_C;
+	double max_C = 1024;
+	if (flag_C_specified)
+		start_C = param.C;
+	else
+		start_C = -1.0;
+	find_parameter_C(&prob, &param, nr_fold, start_C, max_C, best_C, best_rate);
+	mexPrintf("Best C = %lf  CV accuracy = %g%%\n", *best_C, 100.0**best_rate);	
+}
+
+
 double do_cross_validation()
 {
 	int i;
@@ -101,8 +118,8 @@ double do_cross_validation()
                         sumyy += y*y;
                         sumvy += v*y;
                 }
-                printf("Cross Validation Mean squared error = %g\n",total_error/prob.l);
-                printf("Cross Validation Squared correlation coefficient = %g\n",
+                mexPrintf("Cross Validation Mean squared error = %g\n",total_error/prob.l);
+                mexPrintf("Cross Validation Squared correlation coefficient = %g\n",
                         ((prob.l*sumvy-sumv*sumy)*(prob.l*sumvy-sumv*sumy))/
                         ((prob.l*sumvv-sumv*sumv)*(prob.l*sumyy-sumy*sumy))
                         );
@@ -113,7 +130,7 @@ double do_cross_validation()
 		for(i=0;i<prob.l;i++)
 			if(target[i] == prob.y[i])
 				++total_correct;
-		printf("Cross Validation Accuracy = %g%%\n",100.0*total_correct/prob.l);
+		mexPrintf("Cross Validation Accuracy = %g%%\n",100.0*total_correct/prob.l);
 		retval = 100.0*total_correct/prob.l;
 	}
 
@@ -137,8 +154,12 @@ int parse_command_line(int nrhs, const mxArray *prhs[], char *model_file_name)
 	param.nr_weight = 0;
 	param.weight_label = NULL;
 	param.weight = NULL;
-	cross_validation_flag = 0;
+	param.init_sol = NULL;
+	flag_cross_validation = 0;
 	col_format_flag = 0;
+	flag_C_specified = 0;
+	flag_solver_specified = 0;
+	flag_find_C = 0;
 	bias = -1;
 
 
@@ -166,15 +187,17 @@ int parse_command_line(int nrhs, const mxArray *prhs[], char *model_file_name)
 	{
 		if(argv[i][0] != '-') break;
 		++i;
-		if(i>=argc && argv[i-1][1] != 'q') // since option -q has no parameter
+		if(i>=argc && argv[i-1][1] != 'q' && argv[i-1][1] != 'C') // since options -q and -C have no parameter
 			return 1;
 		switch(argv[i-1][1])
 		{
 			case 's':
 				param.solver_type = atoi(argv[i]);
+				flag_solver_specified = 1;
 				break;
 			case 'c':
 				param.C = atof(argv[i]);
+				flag_C_specified = 1;
 				break;
 			case 'p':
 				param.p = atof(argv[i]);
@@ -186,7 +209,7 @@ int parse_command_line(int nrhs, const mxArray *prhs[], char *model_file_name)
 				bias = atof(argv[i]);
 				break;
 			case 'v':
-				cross_validation_flag = 1;
+				flag_cross_validation = 1;
 				nr_fold = atoi(argv[i]);
 				if(nr_fold < 2)
 				{
@@ -205,6 +228,10 @@ int parse_command_line(int nrhs, const mxArray *prhs[], char *model_file_name)
 				print_func = &print_null;
 				i--;
 				break;
+			case 'C':
+				flag_find_C = 1;
+				i--;
+				break;
 			default:
 				mexPrintf("unknown option\n");
 				return 1;
@@ -213,6 +240,23 @@ int parse_command_line(int nrhs, const mxArray *prhs[], char *model_file_name)
 
 	set_print_string_function(print_func);
 
+	// default solver for parameter selection is L2R_L2LOSS_SVC
+	if(flag_find_C)
+	{
+		if(!flag_cross_validation)
+			nr_fold = 5;
+		if(!flag_solver_specified)
+		{
+			mexPrintf("Solver not specified. Using -s 2\n");
+			param.solver_type = L2R_L2LOSS_SVC;
+		}
+		else if(param.solver_type != L2R_LR && param.solver_type != L2R_L2LOSS_SVC)
+		{
+			mexPrintf("Warm-start parameter search only available for -s 0 and -s 2\n");
+			return 1;
+		}
+	}
+
 	if(param.eps == INF)
 	{
 		switch(param.solver_type)
@@ -406,7 +450,18 @@ void mexFunction( int nlhs, mxArray *plhs[],
 			return;
 		}
 
-		if(cross_validation_flag)
+		if (flag_find_C)
+		{
+			double best_C, best_rate, *ptr;
+			
+			do_find_parameter_C(&best_C, &best_rate);	
+			
+			plhs[0] = mxCreateDoubleMatrix(2, 1, mxREAL);
+			ptr = mxGetPr(plhs[0]);
+			ptr[0] = best_C;
+			ptr[1] = best_rate;
+		}
+		else if(flag_cross_validation)
 		{
 			double *ptr;
 			plhs[0] = mxCreateDoubleMatrix(1, 1, mxREAL);

data/{liblinear-1.95 → liblinear-2.1}/python/README

@@ -277,6 +277,11 @@ The above command loads
            structure. If '-v' is specified, cross validation is
            conducted and the returned model is just a scalar: cross-validation
            accuracy for classification and mean-squared error for regression.
+           If the '-C' option is specified, the best parameter C is found 
+	   by cross validation. The returned model is a tuple of the best C 
+	   and the corresponding cross-validation accuracy. The parameter 
+	   selection utility is supported by only -s 0 and -s 2.
+
 
     To train the same data many times with different
     parameters, the second and the third ways should be faster..
@@ -290,6 +295,8 @@ The above command loads
     >>> m = train(prob, '-w1 5 -c 5')
     >>> m = train(prob, param)
     >>> CV_ACC = train(y, x, '-v 3')
+    >>> best_C, best_rate = train(y, x, '-C -s 0')
+    >>> m = train(y, x, '-c {0} -s 0'.format(best_C)) # use the same solver: -s 0
 
 - Function: predict
 

data/{liblinear-1.95 → liblinear-2.1}/python/liblinear.py

@@ -16,7 +16,7 @@ try:
 	if sys.platform == 'win32':
 		liblinear = CDLL(path.join(dirname, r'..\windows\liblinear.dll'))
 	else:
-		liblinear = CDLL(path.join(dirname, '../liblinear.so.2'))
+		liblinear = CDLL(path.join(dirname, '../liblinear.so.3'))
 except:
 # For unix the prefix 'lib' is not considered.
 	if find_library('linear'):
@@ -127,8 +127,8 @@ class problem(Structure):
 
 
 class parameter(Structure):
-	_names = ["solver_type", "eps", "C", "nr_weight", "weight_label", "weight", "p"]
-	_types = [c_int, c_double, c_double, c_int, POINTER(c_int), POINTER(c_double), c_double]
+	_names = ["solver_type", "eps", "C", "nr_weight", "weight_label", "weight", "p", "init_sol"]
+	_types = [c_int, c_double, c_double, c_int, POINTER(c_int), POINTER(c_double), c_double, POINTER(c_double)]
 	_fields_ = genFields(_names, _types)
 
 	def __init__(self, options = None):
@@ -152,10 +152,14 @@ class parameter(Structure):
 		self.C = 1
 		self.p = 0.1
 		self.nr_weight = 0
-		self.weight_label = (c_int * 0)()
-		self.weight = (c_double * 0)()
+		self.weight_label = None
+		self.weight = None
+		self.init_sol = None
 		self.bias = -1
-		self.cross_validation = False
+		self.flag_cross_validation = False
+		self.flag_C_specified = False
+		self.flag_solver_specified = False
+		self.flag_find_C = False
 		self.nr_fold = 0
 		self.print_func = cast(None, PRINT_STRING_FUN)
 
@@ -176,9 +180,11 @@ class parameter(Structure):
 			if argv[i] == "-s":
 				i = i + 1
 				self.solver_type = int(argv[i])
+				self.flag_solver_specified = True
 			elif argv[i] == "-c":
 				i = i + 1
 				self.C = float(argv[i])
+				self.flag_C_specified = True
 			elif argv[i] == "-p":
 				i = i + 1
 				self.p = float(argv[i])
@@ -190,18 +196,20 @@ class parameter(Structure):
 				self.bias = float(argv[i])
 			elif argv[i] == "-v":
 				i = i + 1
-				self.cross_validation = 1
+				self.flag_cross_validation = 1
 				self.nr_fold = int(argv[i])
 				if self.nr_fold < 2 :
 					raise ValueError("n-fold cross validation: n must >= 2")
 			elif argv[i].startswith("-w"):
 				i = i + 1
 				self.nr_weight += 1
-				nr_weight = self.nr_weight
 				weight_label += [int(argv[i-1][2:])]
 				weight += [float(argv[i])]
 			elif argv[i] == "-q":
 				self.print_func = PRINT_STRING_FUN(print_null)
+			elif argv[i] == "-C":
+				self.flag_find_C = True
+
 			else :
 				raise ValueError("Wrong options")
 			i += 1
@@ -213,6 +221,16 @@ class parameter(Structure):
 			self.weight[i] = weight[i]
 			self.weight_label[i] = weight_label[i]
 
+		# default solver for parameter selection is L2R_L2LOSS_SVC
+		if self.flag_find_C:
+			if not self.flag_cross_validation:
+				self.nr_fold = 5
+			if not self.flag_solver_specified:
+				self.solver_type = L2R_L2LOSS_SVC
+				self.flag_solver_specified = True
+			elif self.solver_type not in [L2R_LR, L2R_L2LOSS_SVC]:
+				raise ValueError("Warm-start parameter search only available for -s 0 and -s 2")
+
 		if self.eps == float('inf'):
 			if self.solver_type in [L2R_LR, L2R_L2LOSS_SVC]:
 				self.eps = 0.01
@@ -280,6 +298,7 @@ def toPyModel(model_ptr):
 	return m
 
 fillprototype(liblinear.train, POINTER(model), [POINTER(problem), POINTER(parameter)])
+fillprototype(liblinear.find_parameter_C, None, [POINTER(problem), POINTER(parameter), c_int, c_double, c_double, POINTER(c_double), POINTER(c_double)])
 fillprototype(liblinear.cross_validation, None, [POINTER(problem), POINTER(parameter), c_int, POINTER(c_double)])
 
 fillprototype(liblinear.predict_values, c_double, [POINTER(model), POINTER(feature_node), POINTER(c_double)])

data/{liblinear-1.95 → liblinear-2.1}/python/liblinearutil.py

@@ -127,7 +127,7 @@ def train(arg1, arg2=None, arg3=None):
 		-B bias : if bias >= 0, instance x becomes [x; bias]; if < 0, no bias term added (default -1)
 		-wi weight: weights adjust the parameter C of different classes (see README for details)
 		-v n: n-fold cross validation mode
-	    -q : quiet mode (no outputs)
+		-q : quiet mode (no outputs)
 	"""
 	prob, param = None, None
 	if isinstance(arg1, (list, tuple)):
@@ -150,7 +150,21 @@ def train(arg1, arg2=None, arg3=None):
 	if err_msg :
 		raise ValueError('Error: %s' % err_msg)
 
-	if param.cross_validation:
+	if param.flag_find_C:
+		nr_fold = param.nr_fold
+		best_C = c_double()
+		best_rate = c_double()		
+		max_C = 1024
+		if param.flag_C_specified:
+			start_C = param.C
+		else:
+			start_C = -1.0
+		liblinear.find_parameter_C(prob, param, nr_fold, start_C, max_C, best_C, best_rate)
+		print("Best C = %lf  CV accuracy = %g%%\n"% (best_C.value, 100.0*best_rate.value))
+		return best_C.value,best_rate.value
+
+
+	elif param.flag_cross_validation:
 		l, nr_fold = prob.l, param.nr_fold
 		target = (c_double * l)()
 		liblinear.cross_validation(prob, param, nr_fold, target)

data/{liblinear-1.95 → liblinear-2.1}/train.c

@@ -49,6 +49,7 @@ void exit_with_help()
 	"-B bias : if bias >= 0, instance x becomes [x; bias]; if < 0, no bias term added (default -1)\n"
 	"-wi weight: weights adjust the parameter C of different classes (see README for details)\n"
 	"-v n: n-fold cross validation mode\n"
+	"-C : find parameter C (only for -s 0 and 2)\n"
 	"-q : quiet mode (no outputs)\n"
 	);
 	exit(1);
@@ -84,12 +85,16 @@ static char* readline(FILE *input)
 void parse_command_line(int argc, char **argv, char *input_file_name, char *model_file_name);
 void read_problem(const char *filename);
 void do_cross_validation();
+void do_find_parameter_C();
 
 struct feature_node *x_space;
 struct parameter param;
 struct problem prob;
 struct model* model_;
 int flag_cross_validation;
+int flag_find_C;
+int flag_C_specified;
+int flag_solver_specified;
 int nr_fold;
 double bias;
 
@@ -109,7 +114,11 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
-	if(flag_cross_validation)
+	if (flag_find_C)
+	{
+		do_find_parameter_C();
+	}
+	else if(flag_cross_validation)
 	{
 		do_cross_validation();
 	}
@@ -132,6 +141,19 @@ int main(int argc, char **argv)
 	return 0;
 }
 
+void do_find_parameter_C()
+{
+	double start_C, best_C, best_rate;
+	double max_C = 1024;
+	if (flag_C_specified)
+		start_C = param.C;
+	else
+		start_C = -1.0;
+	printf("Doing parameter search with %d-fold cross validation.\n", nr_fold);
+	find_parameter_C(&prob, &param, nr_fold, start_C, max_C, &best_C, &best_rate);
+	printf("Best C = %g  CV accuracy = %g%%\n", best_C, 100.0*best_rate);
+}
+
 void do_cross_validation()
 {
 	int i;
@@ -186,7 +208,11 @@ void parse_command_line(int argc, char **argv, char *input_file_name, char *mode
 	param.nr_weight = 0;
 	param.weight_label = NULL;
 	param.weight = NULL;
+	param.init_sol = NULL;
 	flag_cross_validation = 0;
+	flag_C_specified = 0;
+	flag_solver_specified = 0;
+	flag_find_C = 0;
 	bias = -1;
 
 	// parse options
@@ -199,10 +225,12 @@ void parse_command_line(int argc, char **argv, char *input_file_name, char *mode
 		{
 			case 's':
 				param.solver_type = atoi(argv[i]);
+				flag_solver_specified = 1;
 				break;
 
 			case 'c':
 				param.C = atof(argv[i]);
+				flag_C_specified = 1;
 				break;
 
 			case 'p':
@@ -240,6 +268,11 @@ void parse_command_line(int argc, char **argv, char *input_file_name, char *mode
 				i--;
 				break;
 
+			case 'C':
+				flag_find_C = 1;
+				i--;
+				break;
+
 			default:
 				fprintf(stderr,"unknown option: -%c\n", argv[i-1][1]);
 				exit_with_help();
@@ -267,6 +300,23 @@ void parse_command_line(int argc, char **argv, char *input_file_name, char *mode
 		sprintf(model_file_name,"%s.model",p);
 	}
 
+	// default solver for parameter selection is L2R_L2LOSS_SVC
+	if(flag_find_C)
+	{
+		if(!flag_cross_validation)
+			nr_fold = 5;
+		if(!flag_solver_specified)
+		{
+			fprintf(stderr, "Solver not specified. Using -s 2\n");
+			param.solver_type = L2R_L2LOSS_SVC;
+		}
+		else if(param.solver_type != L2R_LR && param.solver_type != L2R_L2LOSS_SVC)
+		{
+			fprintf(stderr, "Warm-start parameter search only available for -s 0 and -s 2\n");
+			exit_with_help();
+		}
+	}
+
 	if(param.eps == INF)
 	{
 		switch(param.solver_type)