numo-liblinear 0.2.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,88 @@
1
+ #ifndef _LIBLINEAR_H
2
+ #define _LIBLINEAR_H
3
+
4
+ #define LIBLINEAR_VERSION 241
5
+
6
+ #ifdef __cplusplus
7
+ extern "C" {
8
+ #endif
9
+
10
+ extern int liblinear_version;
11
+
12
+ struct feature_node
13
+ {
14
+ int index;
15
+ double value;
16
+ };
17
+
18
+ struct problem
19
+ {
20
+ int l, n;
21
+ double *y;
22
+ struct feature_node **x;
23
+ double bias; /* < 0 if no bias term */
24
+ };
25
+
26
+ enum { L2R_LR, L2R_L2LOSS_SVC_DUAL, L2R_L2LOSS_SVC, L2R_L1LOSS_SVC_DUAL, MCSVM_CS, L1R_L2LOSS_SVC, L1R_LR, L2R_LR_DUAL, L2R_L2LOSS_SVR = 11, L2R_L2LOSS_SVR_DUAL, L2R_L1LOSS_SVR_DUAL, ONECLASS_SVM = 21 }; /* solver_type */
27
+
28
+ struct parameter
29
+ {
30
+ int solver_type;
31
+
32
+ /* these are for training only */
33
+ double eps; /* stopping criteria */
34
+ double C;
35
+ int nr_weight;
36
+ int *weight_label;
37
+ double* weight;
38
+ double p;
39
+ double nu;
40
+ double *init_sol;
41
+ int regularize_bias;
42
+ };
43
+
44
+ struct model
45
+ {
46
+ struct parameter param;
47
+ int nr_class; /* number of classes */
48
+ int nr_feature;
49
+ double *w;
50
+ int *label; /* label of each class */
51
+ double bias;
52
+ double rho; /* one-class SVM only */
53
+ };
54
+
55
+ struct model* train(const struct problem *prob, const struct parameter *param);
56
+ void cross_validation(const struct problem *prob, const struct parameter *param, int nr_fold, double *target);
57
+ void find_parameters(const struct problem *prob, const struct parameter *param, int nr_fold, double start_C, double start_p, double *best_C, double *best_p, double *best_score);
58
+
59
+ double predict_values(const struct model *model_, const struct feature_node *x, double* dec_values);
60
+ double predict(const struct model *model_, const struct feature_node *x);
61
+ double predict_probability(const struct model *model_, const struct feature_node *x, double* prob_estimates);
62
+
63
+ int save_model(const char *model_file_name, const struct model *model_);
64
+ struct model *load_model(const char *model_file_name);
65
+
66
+ int get_nr_feature(const struct model *model_);
67
+ int get_nr_class(const struct model *model_);
68
+ void get_labels(const struct model *model_, int* label);
69
+ double get_decfun_coef(const struct model *model_, int feat_idx, int label_idx);
70
+ double get_decfun_bias(const struct model *model_, int label_idx);
71
+ double get_decfun_rho(const struct model *model_);
72
+
73
+ void free_model_content(struct model *model_ptr);
74
+ void free_and_destroy_model(struct model **model_ptr_ptr);
75
+ void destroy_param(struct parameter *param);
76
+
77
+ const char *check_parameter(const struct problem *prob, const struct parameter *param);
78
+ int check_probability_model(const struct model *model);
79
+ int check_regression_model(const struct model *model);
80
+ int check_oneclass_model(const struct model *model);
81
+ void set_print_string_function(void (*print_func) (const char*));
82
+
83
+ #ifdef __cplusplus
84
+ }
85
+ #endif
86
+
87
+ #endif /* _LIBLINEAR_H */
88
+
@@ -0,0 +1,245 @@
1
+ #include <math.h>
2
+ #include <stdio.h>
3
+ #include <string.h>
4
+ #include <stdarg.h>
5
+ #include "newton.h"
6
+
7
+ #ifndef min
8
+ template <class T> static inline T min(T x,T y) { return (x<y)?x:y; }
9
+ #endif
10
+
11
+ #ifndef max
12
+ template <class T> static inline T max(T x,T y) { return (x>y)?x:y; }
13
+ #endif
14
+
15
+ #ifdef __cplusplus
16
+ extern "C" {
17
+ #endif
18
+
19
+ extern double dnrm2_(int *, double *, int *);
20
+ extern double ddot_(int *, double *, int *, double *, int *);
21
+ extern int daxpy_(int *, double *, double *, int *, double *, int *);
22
+ extern int dscal_(int *, double *, double *, int *);
23
+
24
+ #ifdef __cplusplus
25
+ }
26
+ #endif
27
+
28
+ static void default_print(const char *buf)
29
+ {
30
+ fputs(buf,stdout);
31
+ fflush(stdout);
32
+ }
33
+
34
+ // On entry *f must be the function value of w
35
+ // On exit w is updated and *f is the new function value
36
+ double function::linesearch_and_update(double *w, double *s, double *f, double *g, double alpha)
37
+ {
38
+ double gTs = 0;
39
+ double eta = 0.01;
40
+ int n = get_nr_variable();
41
+ int max_num_linesearch = 20;
42
+ double *w_new = new double[n];
43
+ double fold = *f;
44
+
45
+ for (int i=0;i<n;i++)
46
+ gTs += s[i] * g[i];
47
+
48
+ int num_linesearch = 0;
49
+ for(num_linesearch=0; num_linesearch < max_num_linesearch; num_linesearch++)
50
+ {
51
+ for (int i=0;i<n;i++)
52
+ w_new[i] = w[i] + alpha*s[i];
53
+ *f = fun(w_new);
54
+ if (*f - fold <= eta * alpha * gTs)
55
+ break;
56
+ else
57
+ alpha *= 0.5;
58
+ }
59
+
60
+ if (num_linesearch >= max_num_linesearch)
61
+ {
62
+ *f = fold;
63
+ return 0;
64
+ }
65
+ else
66
+ memcpy(w, w_new, sizeof(double)*n);
67
+
68
+ delete [] w_new;
69
+ return alpha;
70
+ }
71
+
72
+ void NEWTON::info(const char *fmt,...)
73
+ {
74
+ char buf[BUFSIZ];
75
+ va_list ap;
76
+ va_start(ap,fmt);
77
+ vsprintf(buf,fmt,ap);
78
+ va_end(ap);
79
+ (*newton_print_string)(buf);
80
+ }
81
+
82
+ NEWTON::NEWTON(const function *fun_obj, double eps, double eps_cg, int max_iter)
83
+ {
84
+ this->fun_obj=const_cast<function *>(fun_obj);
85
+ this->eps=eps;
86
+ this->eps_cg=eps_cg;
87
+ this->max_iter=max_iter;
88
+ newton_print_string = default_print;
89
+ }
90
+
91
+ NEWTON::~NEWTON()
92
+ {
93
+ }
94
+
95
+ void NEWTON::newton(double *w)
96
+ {
97
+ int n = fun_obj->get_nr_variable();
98
+ int i, cg_iter;
99
+ double step_size;
100
+ double f, fold, actred;
101
+ double init_step_size = 1;
102
+ int search = 1, iter = 1, inc = 1;
103
+ double *s = new double[n];
104
+ double *r = new double[n];
105
+ double *g = new double[n];
106
+
107
+ const double alpha_pcg = 0.01;
108
+ double *M = new double[n];
109
+
110
+ // calculate gradient norm at w=0 for stopping condition.
111
+ double *w0 = new double[n];
112
+ for (i=0; i<n; i++)
113
+ w0[i] = 0;
114
+ fun_obj->fun(w0);
115
+ fun_obj->grad(w0, g);
116
+ double gnorm0 = dnrm2_(&n, g, &inc);
117
+ delete [] w0;
118
+
119
+ f = fun_obj->fun(w);
120
+ info("init f %5.3e\n", f);
121
+ fun_obj->grad(w, g);
122
+ double gnorm = dnrm2_(&n, g, &inc);
123
+
124
+ if (gnorm <= eps*gnorm0)
125
+ search = 0;
126
+
127
+ double *w_new = new double[n];
128
+ while (iter <= max_iter && search)
129
+ {
130
+ fun_obj->get_diag_preconditioner(M);
131
+ for(i=0; i<n; i++)
132
+ M[i] = (1-alpha_pcg) + alpha_pcg*M[i];
133
+ cg_iter = pcg(g, M, s, r);
134
+
135
+ fold = f;
136
+ step_size = fun_obj->linesearch_and_update(w, s, & f, g, init_step_size);
137
+
138
+ if (step_size == 0)
139
+ {
140
+ info("WARNING: line search fails\n");
141
+ break;
142
+ }
143
+
144
+ info("iter %2d f %5.3e |g| %5.3e CG %3d step_size %4.2e \n", iter, f, gnorm, cg_iter, step_size);
145
+
146
+ actred = fold - f;
147
+ iter++;
148
+
149
+ fun_obj->grad(w, g);
150
+
151
+ gnorm = dnrm2_(&n, g, &inc);
152
+ if (gnorm <= eps*gnorm0)
153
+ break;
154
+ if (f < -1.0e+32)
155
+ {
156
+ info("WARNING: f < -1.0e+32\n");
157
+ break;
158
+ }
159
+ if (fabs(actred) <= 1.0e-12*fabs(f))
160
+ {
161
+ info("WARNING: actred too small\n");
162
+ break;
163
+ }
164
+ }
165
+
166
+ delete[] g;
167
+ delete[] r;
168
+ delete[] w_new;
169
+ delete[] s;
170
+ delete[] M;
171
+ }
172
+
173
+ int NEWTON::pcg(double *g, double *M, double *s, double *r)
174
+ {
175
+ int i, inc = 1;
176
+ int n = fun_obj->get_nr_variable();
177
+ double one = 1;
178
+ double *d = new double[n];
179
+ double *Hd = new double[n];
180
+ double zTr, znewTrnew, alpha, beta, cgtol;
181
+ double *z = new double[n];
182
+ double Q = 0, newQ, Qdiff;
183
+
184
+ for (i=0; i<n; i++)
185
+ {
186
+ s[i] = 0;
187
+ r[i] = -g[i];
188
+ z[i] = r[i] / M[i];
189
+ d[i] = z[i];
190
+ }
191
+
192
+ zTr = ddot_(&n, z, &inc, r, &inc);
193
+ double gMinv_norm = sqrt(zTr);
194
+ cgtol = min(eps_cg, sqrt(gMinv_norm));
195
+ int cg_iter = 0;
196
+ int max_cg_iter = max(n, 5);
197
+
198
+ while (cg_iter < max_cg_iter)
199
+ {
200
+ cg_iter++;
201
+ fun_obj->Hv(d, Hd);
202
+
203
+ alpha = zTr/ddot_(&n, d, &inc, Hd, &inc);
204
+ daxpy_(&n, &alpha, d, &inc, s, &inc);
205
+ alpha = -alpha;
206
+ daxpy_(&n, &alpha, Hd, &inc, r, &inc);
207
+
208
+ // Using quadratic approximation as CG stopping criterion
209
+ newQ = -0.5*(ddot_(&n, s, &inc, r, &inc) - ddot_(&n, s, &inc, g, &inc));
210
+ Qdiff = newQ - Q;
211
+ if (newQ <= 0 && Qdiff <= 0)
212
+ {
213
+ if (cg_iter * Qdiff >= cgtol * newQ)
214
+ break;
215
+ }
216
+ else
217
+ {
218
+ info("WARNING: quadratic approximation > 0 or increasing in CG\n");
219
+ break;
220
+ }
221
+ Q = newQ;
222
+
223
+ for (i=0; i<n; i++)
224
+ z[i] = r[i] / M[i];
225
+ znewTrnew = ddot_(&n, z, &inc, r, &inc);
226
+ beta = znewTrnew/zTr;
227
+ dscal_(&n, &beta, d, &inc);
228
+ daxpy_(&n, &one, z, &inc, d, &inc);
229
+ zTr = znewTrnew;
230
+ }
231
+
232
+ if (cg_iter == max_cg_iter)
233
+ info("WARNING: reaching maximal number of CG steps\n");
234
+
235
+ delete[] d;
236
+ delete[] Hd;
237
+ delete[] z;
238
+
239
+ return(cg_iter);
240
+ }
241
+
242
+ void NEWTON::set_print_string(void (*print_string) (const char *buf))
243
+ {
244
+ newton_print_string = print_string;
245
+ }
@@ -0,0 +1,37 @@
1
+ #ifndef _NEWTON_H
2
+ #define _NEWTON_H
3
+
4
+ class function
5
+ {
6
+ public:
7
+ virtual double fun(double *w) = 0 ;
8
+ virtual void grad(double *w, double *g) = 0 ;
9
+ virtual void Hv(double *s, double *Hs) = 0 ;
10
+ virtual int get_nr_variable(void) = 0 ;
11
+ virtual void get_diag_preconditioner(double *M) = 0 ;
12
+ virtual ~function(void){}
13
+
14
+ // base implementation in newton.cpp
15
+ virtual double linesearch_and_update(double *w, double *s, double *f, double *g, double alpha);
16
+ };
17
+
18
+ class NEWTON
19
+ {
20
+ public:
21
+ NEWTON(const function *fun_obj, double eps = 0.1, double eps_cg = 0.5, int max_iter = 1000);
22
+ ~NEWTON();
23
+
24
+ void newton(double *w);
25
+ void set_print_string(void (*i_print) (const char *buf));
26
+
27
+ private:
28
+ int pcg(double *g, double *M, double *s, double *r);
29
+
30
+ double eps;
31
+ double eps_cg;
32
+ int max_iter;
33
+ function *fun_obj;
34
+ void info(const char *fmt,...);
35
+ void (*newton_print_string)(const char *buf);
36
+ };
37
+ #endif
@@ -16,6 +16,28 @@ void print_null(const char *s) {}
16
16
  * @param y [Numo::DFloat] (shape: [n_samples]) The labels or target values for samples.
17
17
  * @param param [Hash] The parameters of a model.
18
18
  *
19
+ * @example
20
+ * require 'numo/liblinear'
21
+ *
22
+ * # Prepare training dataset.
23
+ * x = Numo::DFloat[[-0.8, 1.0], [-0.5, 0.8], [0.9, -0.8], [0.8, -0.7]]
24
+ * y = Numo::Int32[-1, -1, 1, 1]
25
+ *
26
+ * # Train L2-regularized L2-loss support vector classifier.
27
+ * param = {
28
+ * solver_type: Numo::Liblinear::SolverType::L2R_L2LOSS_SVC_DUAL,
29
+ * C: 0.1,
30
+ * random_seed: 1
31
+ * }
32
+ * model = Numo::Liblinear.train(x, y, param)
33
+ *
34
+ * # Predict labels of test data.
35
+ * x_test = Numo::DFloat[[-0.7, 0.9], [0.5, -0.4]]
36
+ * result = Numo::Liblinear.predict(x_test, param, model)
37
+ * p result
38
+ * # Numo::DFloat#shape=[2]
39
+ * # [-1, 1]
40
+ *
19
41
  * @raise [ArgumentError] If the sample array is not 2-dimensional, the label array is not 1-dimensional,
20
42
  * the sample array and label array do not have the same number of samples, or
21
43
  * the hyperparameter has an invalid value, this error is raised.
@@ -30,6 +52,8 @@ VALUE numo_liblinear_train(VALUE self, VALUE x_val, VALUE y_val, VALUE param_has
30
52
  narray_t* x_nary;
31
53
  narray_t* y_nary;
32
54
  char* err_msg;
55
+ VALUE random_seed;
56
+ VALUE verbose;
33
57
  VALUE model_hash;
34
58
 
35
59
  if (CLASS_OF(x_val) != numo_cDFloat) {
@@ -60,6 +84,11 @@ VALUE numo_liblinear_train(VALUE self, VALUE x_val, VALUE y_val, VALUE param_has
60
84
  return Qnil;
61
85
  }
62
86
 
87
+ random_seed = rb_hash_aref(param_hash, ID2SYM(rb_intern("random_seed")));
88
+ if (!NIL_P(random_seed)) {
89
+ srand(NUM2UINT(random_seed));
90
+ }
91
+
63
92
  param = rb_hash_to_parameter(param_hash);
64
93
  problem = dataset_to_problem(x_val, y_val);
65
94
 
@@ -71,7 +100,11 @@ VALUE numo_liblinear_train(VALUE self, VALUE x_val, VALUE y_val, VALUE param_has
71
100
  return Qnil;
72
101
  }
73
102
 
74
- set_print_string_function(print_null);
103
+ verbose = rb_hash_aref(param_hash, ID2SYM(rb_intern("verbose")));
104
+ if (verbose != Qtrue) {
105
+ set_print_string_function(print_null);
106
+ }
107
+
75
108
  model = train(problem, param);
76
109
  model_hash = model_to_rb_hash(model);
77
110
  free_and_destroy_model(&model);
@@ -92,6 +125,28 @@ VALUE numo_liblinear_train(VALUE self, VALUE x_val, VALUE y_val, VALUE param_has
92
125
  * @param param [Hash] The parameters of a model.
93
126
  * @param n_folds [Integer] The number of folds.
94
127
  *
128
+ * @example
129
+ * require 'numo/liblinear'
130
+ *
131
+ * # x: samples
132
+ * # y: labels
133
+ *
134
+ * # Define parameters of L2-regularized L2-loss support vector classification.
135
+ * param = {
136
+ * solver_type: Numo::Liblinear::SolverType::L2R_L2LOSS_SVC_DUAL,
137
+ * C: 1,
138
+ * random_seed: 1,
139
+ * verbose: true
140
+ * }
141
+ *
142
+ * # Perform 5-cross validation.
143
+ * n_folds = 5
144
+ * res = Numo::Liblinear::cv(x, y, param, n_folds)
145
+ *
146
+ * # Print mean accuracy.
147
+ * mean_accuracy = y.eq(res).count.fdiv(y.size)
148
+ * puts "Accuracy: %.1f %%" % (100 * mean_accuracy)
149
+ *
95
150
  * @raise [ArgumentError] If the sample array is not 2-dimensional, the label array is not 1-dimensional,
96
151
  * the sample array and label array do not have the same number of samples, or
97
152
  * the hyperparameter has an invalid value, this error is raised.
@@ -107,6 +162,8 @@ VALUE numo_liblinear_cross_validation(VALUE self, VALUE x_val, VALUE y_val, VALU
107
162
  narray_t* x_nary;
108
163
  narray_t* y_nary;
109
164
  char* err_msg;
165
+ VALUE random_seed;
166
+ VALUE verbose;
110
167
  struct problem* problem;
111
168
  struct parameter* param;
112
169
 
@@ -138,6 +195,11 @@ VALUE numo_liblinear_cross_validation(VALUE self, VALUE x_val, VALUE y_val, VALU
138
195
  return Qnil;
139
196
  }
140
197
 
198
+ random_seed = rb_hash_aref(param_hash, ID2SYM(rb_intern("random_seed")));
199
+ if (!NIL_P(random_seed)) {
200
+ srand(NUM2UINT(random_seed));
201
+ }
202
+
141
203
  param = rb_hash_to_parameter(param_hash);
142
204
  problem = dataset_to_problem(x_val, y_val);
143
205
 
@@ -153,7 +215,11 @@ VALUE numo_liblinear_cross_validation(VALUE self, VALUE x_val, VALUE y_val, VALU
153
215
  t_val = rb_narray_new(numo_cDFloat, 1, t_shape);
154
216
  t_pt = (double*)na_get_pointer_for_write(t_val);
155
217
 
156
- set_print_string_function(print_null);
218
+ verbose = rb_hash_aref(param_hash, ID2SYM(rb_intern("verbose")));
219
+ if (verbose != Qtrue) {
220
+ set_print_string_function(print_null);
221
+ }
222
+
157
223
  cross_validation(problem, param, n_folds, t_pt);
158
224
 
159
225
  xfree_problem(problem);
@@ -216,18 +282,12 @@ VALUE numo_liblinear_predict(VALUE self, VALUE x_val, VALUE param_hash, VALUE mo
216
282
  x_pt = (double*)na_get_pointer_for_read(x_val);
217
283
 
218
284
  /* Predict values. */
219
- x_nodes = ALLOC_N(struct feature_node, n_features + 1);
220
- x_nodes[n_features].index = -1;
221
- x_nodes[n_features].value = 0.0;
222
285
  for (i = 0; i < n_samples; i++) {
223
- for (j = 0; j < n_features; j++) {
224
- x_nodes[j].index = j + 1;
225
- x_nodes[j].value = (double)x_pt[i * n_features + j];
226
- }
286
+ x_nodes = dbl_vec_to_node(&x_pt[i * n_features], n_features);
227
287
  y_pt[i] = predict(model, x_nodes);
288
+ xfree(x_nodes);
228
289
  }
229
290
 
230
- xfree(x_nodes);
231
291
  xfree_model(model);
232
292
  xfree_parameter(param);
233
293
 
@@ -299,34 +359,22 @@ VALUE numo_liblinear_decision_function(VALUE self, VALUE x_val, VALUE param_hash
299
359
 
300
360
  /* Predict values. */
301
361
  if (model->nr_class == 2 && model->param.solver_type != MCSVM_CS) {
302
- x_nodes = ALLOC_N(struct feature_node, n_features + 1);
303
- x_nodes[n_features].index = -1;
304
- x_nodes[n_features].value = 0.0;
305
362
  for (i = 0; i < n_samples; i++) {
306
- for (j = 0; j < n_features; j++) {
307
- x_nodes[j].index = j + 1;
308
- x_nodes[j].value = (double)x_pt[i * n_features + j];
309
- }
363
+ x_nodes = dbl_vec_to_node(&x_pt[i * n_features], n_features);
310
364
  predict_values(model, x_nodes, &y_pt[i]);
365
+ xfree(x_nodes);
311
366
  }
312
- xfree(x_nodes);
313
367
  } else {
314
368
  y_cols = (int)y_shape[1];
315
369
  dec_values = ALLOC_N(double, y_cols);
316
- x_nodes = ALLOC_N(struct feature_node, n_features + 1);
317
- x_nodes[n_features].index = -1;
318
- x_nodes[n_features].value = 0.0;
319
370
  for (i = 0; i < n_samples; i++) {
320
- for (j = 0; j < n_features; j++) {
321
- x_nodes[j].index = j + 1;
322
- x_nodes[j].value = (double)x_pt[i * n_features + j];
323
- }
371
+ x_nodes = dbl_vec_to_node(&x_pt[i * n_features], n_features);
324
372
  predict_values(model, x_nodes, dec_values);
373
+ xfree(x_nodes);
325
374
  for (j = 0; j < y_cols; j++) {
326
375
  y_pt[i * y_cols + j] = dec_values[j];
327
376
  }
328
377
  }
329
- xfree(x_nodes);
330
378
  xfree(dec_values);
331
379
  }
332
380
 
@@ -395,20 +443,14 @@ VALUE numo_liblinear_predict_proba(VALUE self, VALUE x_val, VALUE param_hash, VA
395
443
 
396
444
  /* Predict values. */
397
445
  probs = ALLOC_N(double, model->nr_class);
398
- x_nodes = ALLOC_N(struct feature_node, n_features + 1);
399
- x_nodes[n_features].index = -1;
400
- x_nodes[n_features].value = 0.0;
401
446
  for (i = 0; i < n_samples; i++) {
402
- for (j = 0; j < n_features; j++) {
403
- x_nodes[j].index = j + 1;
404
- x_nodes[j].value = (double)x_pt[i * n_features + j];
405
- }
447
+ x_nodes = dbl_vec_to_node(&x_pt[i * n_features], n_features);
406
448
  predict_probability(model, x_nodes, probs);
449
+ xfree(x_nodes);
407
450
  for (j = 0; j < model->nr_class; j++) {
408
451
  y_pt[i * model->nr_class + j] = probs[j];
409
452
  }
410
453
  }
411
- xfree(x_nodes);
412
454
  xfree(probs);
413
455
  }
414
456
 
@@ -503,6 +545,9 @@ void Init_liblinearext()
503
545
  */
504
546
  mLiblinear = rb_define_module_under(mNumo, "Liblinear");
505
547
 
548
+ /* The version of LIBLINEAR used in backgroud library. */
549
+ rb_define_const(mLiblinear, "LIBLINEAR_VERSION", INT2NUM(LIBLINEAR_VERSION));
550
+
506
551
  rb_define_module_function(mLiblinear, "train", numo_liblinear_train, 3);
507
552
  rb_define_module_function(mLiblinear, "cv", numo_liblinear_cross_validation, 4);
508
553
  rb_define_module_function(mLiblinear, "predict", numo_liblinear_predict, 3);