liblinear-ruby 0.0.1

data/ext/linear.h

@@ -0,0 +1,74 @@
+#ifndef _LIBLINEAR_H
+#define _LIBLINEAR_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct feature_node
+{
+	int index;
+	double value;
+};
+
+struct problem
+{
+	int l, n;
+	double *y;
+	struct feature_node **x;
+	double bias;            /* < 0 if no bias term */  
+};
+
+enum { L2R_LR, L2R_L2LOSS_SVC_DUAL, L2R_L2LOSS_SVC, L2R_L1LOSS_SVC_DUAL, MCSVM_CS, L1R_L2LOSS_SVC, L1R_LR, L2R_LR_DUAL, L2R_L2LOSS_SVR = 11, L2R_L2LOSS_SVR_DUAL, L2R_L1LOSS_SVR_DUAL }; /* solver_type */
+
+struct parameter
+{
+	int solver_type;
+
+	/* these are for training only */
+	double eps;	        /* stopping criteria */
+	double C;
+	int nr_weight;
+	int *weight_label;
+	double* weight;
+	double p;
+};
+
+struct model
+{
+	struct parameter param;
+	int nr_class;		/* number of classes */
+	int nr_feature;
+	double *w;
+	int *label;		/* label of each class */
+	double bias;
+};
+
+struct model* train(const struct problem *prob, const struct parameter *param);
+void cross_validation(const struct problem *prob, const struct parameter *param, int nr_fold, double *target);
+
+double predict_values(const struct model *model_, const struct feature_node *x, double* dec_values);
+double predict(const struct model *model_, const struct feature_node *x);
+double predict_probability(const struct model *model_, const struct feature_node *x, double* prob_estimates);
+
+int save_model(const char *model_file_name, const struct model *model_);
+struct model *load_model(const char *model_file_name);
+
+int get_nr_feature(const struct model *model_);
+int get_nr_class(const struct model *model_);
+void get_labels(const struct model *model_, int* label);
+
+void free_model_content(struct model *model_ptr);
+void free_and_destroy_model(struct model **model_ptr_ptr);
+void destroy_param(struct parameter *param);
+
+const char *check_parameter(const struct problem *prob, const struct parameter *param);
+int check_probability_model(const struct model *model);
+void set_print_string_function(void (*print_func) (const char*));
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _LIBLINEAR_H */
+

data/ext/linear.rb

@@ -0,0 +1,357 @@
+require 'liblinear'
+include Liblinear
+
+def _int_array(seq)
+  size = seq.size
+  array = new_int(size)
+  i = 0
+  for item in seq
+    int_setitem(array,i,item)
+    i = i + 1
+  end
+  return array
+end
+
+def _double_array(seq)
+  size = seq.size
+  array = new_double(size)
+  i = 0
+  for item in seq
+    double_setitem(array,i,item)
+    i = i + 1
+  end
+  return array
+end
+
+def _free_int_array(x)
+  if !x.nil? # and !x.empty?
+    delete_int(x)
+  end
+end
+
+def _free_double_array(x)
+  if !x.nil? # and !x.empty?
+    delete_double(x)
+  end
+end
+
+def _int_array_to_list(x,n)
+  list = []
+   (0..n-1).each {|i| list << int_getitem(x,i) }
+  return list
+end
+
+def _double_array_to_list(x,n)
+  list = []
+   (0..n-1).each {|i| list << double_getitem(x,i) }
+  return list
+end    
+
+class LParameter
+  attr_accessor :param
+  
+  def initialize(*args)
+    @param = Liblinear::Parameter.new
+    @param.solver_type = L2R_LR
+    @param.C = 1
+    @param.eps = 0.01
+    @param.nr_weight = 0
+    @param.weight_label = _int_array([])
+    @param.weight = _double_array([])
+    
+    args[0].each {|k,v| 
+      self.send("#{k}=",v)
+    } if !args[0].nil?
+  end
+  
+  def method_missing(m, *args)
+    #print m.to_s
+    #puts args.inspect
+    if m.to_s == 'weight_label='
+      @weight_label_len = args[0].size
+      pargs = _int_array(args[0])
+      _free_int_array(@param.weight_label)
+    elsif m.to_s == 'weight='
+      @weight_len = args[0].size
+      pargs = _double_array(args[0])
+      _free_double_array(@param.weight)
+    else
+      pargs = args[0]
+    end
+    
+    if m.to_s.index('=')
+      @param.send("#{m}",pargs)
+    else
+      @param.send("#{m}")
+    end
+    
+  end
+  
+  def inspect
+    "LParameter: solver_type=#{@param.solver_type} C=#{@param.C} eps=#{@param.eps}"
+  end
+  
+  def destroy
+    _free_int_array(@param.weight_label)
+    _free_double_array(@param.weight)
+    delete_parameter(@param)
+    @param = nil
+  end
+end
+
+def _convert_to_feature_node_array(x, maxlen, bias=-1)
+  # convert a sequence or mapping to an feature_node array
+  
+  # Find non zero elements
+  iter_range = []
+  if x.class == Hash
+    x.each {|k, v|
+      # all zeros kept due to the precomputed kernel; no good solution yet
+      iter_range << k #if v != 0
+    }
+  elsif x.class == Array
+    x.each_index {|j| 
+      iter_range << j #if x[j] != 0
+    }
+  else
+    raise TypeError,"data must be a hash or an array"
+  end
+  
+  iter_range.sort!
+  if bias >=0
+    data = feature_node_array(iter_range.size+2)
+    #puts "bias element (#{iter_range.size},#{bias})"
+    feature_node_array_set(data,iter_range.size,maxlen+1,bias)
+    feature_node_array_set(data,iter_range.size+1,-1,0)
+  else
+    data = feature_node_array(iter_range.size+1)
+    feature_node_array_set(data,iter_range.size,-1,0)
+  end
+  
+  j = 0
+  for k in iter_range
+    #puts "element #{j}= (#{k},#{x[k]})"
+    feature_node_array_set(data,j,k,x[k])
+    j = j + 1
+  end
+  return data
+end
+
+
+class LProblem
+  attr_accessor :prob, :maxlen, :size
+  
+  def initialize(y,x,bias)
+#    assert_equal(y.size, x.size)
+    @prob = prob = Liblinear::Problem.new 
+    @size = size = y.size
+    
+    @y_array = y_array = new_double(size)
+    for i in (0..size-1)
+      double_setitem(@y_array,i,y[i])
+    end
+    
+    @x_matrix = x_matrix = feature_node_matrix(size)
+    @data = []
+    @maxlen = 0  #max number of features
+    len_array=[]
+
+    for i in (0..size-1)
+      data = _convert_to_feature_node_array(x[i], @maxlen, bias)
+      @data << data
+      feature_node_matrix_set(x_matrix,i,data)
+
+      if x[i].class == Hash
+        if x[i].size > 0
+          @maxlen = [@maxlen,x[i].keys.max].max          
+        end
+      else
+        @maxlen = [@maxlen,x[i].size].max
+      end
+      len_array << x[i].size      
+    end
+
+    if bias >= 0
+      set_bias_index(x_matrix, size, @maxlen, _int_array(len_array))
+    end
+    
+    prob.y = y_array
+    prob.x = x_matrix
+    prob.bias = bias
+    prob.l = size
+    prob.n = @maxlen
+    if bias >= 0
+      prob.n += 1
+    end
+  end
+  
+  def inspect
+    "LProblem: size = #{size} n=#{prob.n} bias=#{prob.bias} maxlen=#{@maxlen}"
+  end
+  
+  def destroy
+    delete_problem(@prob)
+    delete_int(@y_array)
+    for i in (0..size-1)
+      feature_node_array_destroy(@data[i])
+    end
+    feature_node_matrix_destroy(@x_matrix)
+  end
+end
+
+class LModel
+  attr_accessor :model, :probability
+  
+  def initialize(arg1,arg2=nil)
+    if arg2 == nil
+      # create model from file
+      filename = arg1
+      @model = load_model(filename)
+    else
+      # create model from problem and parameter
+      prob,param = arg1,arg2
+      @prob = prob
+      msg = check_parameter(prob.prob,param.param)
+      raise "ValueError", msg if msg
+      @model = Liblinear::train(prob.prob,param.param)
+    end
+    #setup some classwide variables
+    @nr_class = Liblinear::get_nr_class(@model)
+    #create labels(classes)
+    intarr = new_int(@nr_class)
+    Liblinear::get_labels(@model,intarr)
+    @labels = _int_array_to_list(intarr, @nr_class)
+    delete_int(intarr)
+  end
+  
+  def predict(x)
+    data = _convert_to_feature_node_array(x, @model.nr_feature, @model.bias)
+    ret = Liblinear::predict(@model,data)
+    feature_node_array_destroy(data)
+    return ret
+  end
+  
+  
+  def get_nr_class
+    return @nr_class
+  end
+  
+  def get_labels
+    return @labels
+  end
+  
+  def predict_values_raw(x)
+    #convert x into feature_node, allocate a double array for return
+    n = (@nr_class*(@nr_class-1)/2).floor
+    data = _convert_to_feature_node_array(x, @model.nr_feature, @model.bias)
+    dblarr = new_double(n)
+    Liblinear::predict_values(@model, data, dblarr)
+    ret = _double_array_to_list(dblarr, n)
+    delete_double(dblarr)
+    feature_node_array_destroy(data)
+    return ret
+  end
+  
+  def predict_values(x)
+    v=predict_values_raw(x)
+    #puts v.inspect
+    if false
+      #if @svm_type == NU_SVR or @svm_type == EPSILON_SVR or @svm_type == ONE_CLASS
+      return v[0]
+    else #self.svm_type == C_SVC or self.svm_type == NU_SVC
+      count = 0
+      
+      # create a width x height array
+      width = @labels.size
+      height = @labels.size
+      d = Array.new(width)
+      d.map! { Array.new(height) }
+      
+      for i in (0..@labels.size-1)
+        for j in (i+1..@labels.size-1)
+          d[@labels[i]][@labels[j]] = v[count]
+          d[@labels[j]][@labels[i]] = -v[count]
+          count += 1
+        end
+      end
+      return  d
+    end
+  end
+  
+  def predict_probability(x)
+#    if not @probability
+#      raise TypeError, "model does not support probabiliy estimates"
+#    end
+    
+    #convert x into feature_node, alloc a double array to receive probabilities
+    data = _convert_to_feature_node_array(x, @model.nr_feature, @model.bias)
+    dblarr = new_double(@nr_class)
+    pred = Liblinear::predict_probability(@model, data, dblarr)
+    pv = _double_array_to_list(dblarr, @nr_class)
+    delete_double(dblarr)
+    feature_node_array_destroy(data)
+    p = {}
+    for i in (0..@labels.size-1)
+      p[@labels[i]] = pv[i]
+    end
+    return pred, p
+  end
+  
+  # def get_svr_probability
+  #   #leave the Error checking to svm.cpp code
+  #   ret = Liblinear::get_svr_probability(@model)
+  #   if ret == 0
+  #     raise TypeError, "not a regression model or probability information not available"
+  #   end
+  #   return ret
+  # end
+  
+  # def get_svr_pdf
+  #   #get_svr_probability will handle error checking
+  #   sigma = get_svr_probability()
+  #   return Proc.new{|z| exp(-z.abs/sigma)/(2*sigma)}  # TODO: verify this works
+  # end
+  
+  def save(filename)
+    save_model(filename,@model)
+  end
+  
+  def destroy
+    destroy_model(@model)
+  end
+end
+
+def cross_validation(prob, param, fold)
+  target = new_int(prob.size)
+  Liblinear::cross_validation(prob.prob, param.param, fold, target)
+  ret = _int_array_to_list(target, prob.size)
+  delete_int(target)
+  return ret
+end
+
+def read_file filename
+  labels = []
+  samples = []
+  max_index = 0
+
+  f = File.open(filename)
+  f.each do |line|
+    elems = line.split
+    sample = {}
+    for e in elems[1..-1]
+       points = e.split(":")
+       sample[points[0].to_i] = points[1].to_f
+       if points[0].to_i < max_index
+          max_index = points[0].to_i
+       end
+    end
+    labels << elems[0].to_i
+    samples << sample
+  #print elems[0].to_i
+  #print " - "
+  #puts sample.inspect
+  end
+  puts "#{filename}: #{samples.size} samples loaded."
+  return labels,samples
+end
+

data/ext/tron.cpp

@@ -0,0 +1,235 @@
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdarg.h>
+#include "tron.h"
+
+#ifndef min
+template <class T> static inline T min(T x,T y) { return (x<y)?x:y; }
+#endif
+
+#ifndef max
+template <class T> static inline T max(T x,T y) { return (x>y)?x:y; }
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern double dnrm2_(int *, double *, int *);
+extern double ddot_(int *, double *, int *, double *, int *);
+extern int daxpy_(int *, double *, double *, int *, double *, int *);
+extern int dscal_(int *, double *, double *, int *);
+
+#ifdef __cplusplus
+}
+#endif
+
+static void default_print(const char *buf)
+{
+	fputs(buf,stdout);
+	fflush(stdout);
+}
+
+void TRON::info(const char *fmt,...)
+{
+	char buf[BUFSIZ];
+	va_list ap;
+	va_start(ap,fmt);
+	vsprintf(buf,fmt,ap);
+	va_end(ap);
+	(*tron_print_string)(buf);
+}
+
+TRON::TRON(const function *fun_obj, double eps, int max_iter)
+{
+	this->fun_obj=const_cast<function *>(fun_obj);
+	this->eps=eps;
+	this->max_iter=max_iter;
+	tron_print_string = default_print;
+}
+
+TRON::~TRON()
+{
+}
+
+void TRON::tron(double *w)
+{
+	// Parameters for updating the iterates.
+	double eta0 = 1e-4, eta1 = 0.25, eta2 = 0.75;
+
+	// Parameters for updating the trust region size delta.
+	double sigma1 = 0.25, sigma2 = 0.5, sigma3 = 4;
+
+	int n = fun_obj->get_nr_variable();
+	int i, cg_iter;
+	double delta, snorm, one=1.0;
+	double alpha, f, fnew, prered, actred, gs;
+	int search = 1, iter = 1, inc = 1;
+	double *s = new double[n];
+	double *r = new double[n];
+	double *w_new = new double[n];
+	double *g = new double[n];
+
+	for (i=0; i<n; i++)
+		w[i] = 0;
+
+        f = fun_obj->fun(w);
+	fun_obj->grad(w, g);
+	delta = dnrm2_(&n, g, &inc);
+	double gnorm1 = delta;
+	double gnorm = gnorm1;
+
+	if (gnorm <= eps*gnorm1)
+		search = 0;
+
+	iter = 1;
+
+	while (iter <= max_iter && search)
+	{
+		cg_iter = trcg(delta, g, s, r);
+
+		memcpy(w_new, w, sizeof(double)*n);
+		daxpy_(&n, &one, s, &inc, w_new, &inc);
+
+		gs = ddot_(&n, g, &inc, s, &inc);
+		prered = -0.5*(gs-ddot_(&n, s, &inc, r, &inc));
+                fnew = fun_obj->fun(w_new);
+
+		// Compute the actual reduction.
+	        actred = f - fnew;
+
+		// On the first iteration, adjust the initial step bound.
+		snorm = dnrm2_(&n, s, &inc);
+		if (iter == 1)
+			delta = min(delta, snorm);
+
+		// Compute prediction alpha*snorm of the step.
+		if (fnew - f - gs <= 0)
+			alpha = sigma3;
+		else
+			alpha = max(sigma1, -0.5*(gs/(fnew - f - gs)));
+
+		// Update the trust region bound according to the ratio of actual to predicted reduction.
+		if (actred < eta0*prered)
+			delta = min(max(alpha, sigma1)*snorm, sigma2*delta);
+		else if (actred < eta1*prered)
+			delta = max(sigma1*delta, min(alpha*snorm, sigma2*delta));
+		else if (actred < eta2*prered)
+			delta = max(sigma1*delta, min(alpha*snorm, sigma3*delta));
+		else
+			delta = max(delta, min(alpha*snorm, sigma3*delta));
+
+		info("iter %2d act %5.3e pre %5.3e delta %5.3e f %5.3e |g| %5.3e CG %3d\n", iter, actred, prered, delta, f, gnorm, cg_iter);
+
+		if (actred > eta0*prered)
+		{
+			iter++;
+			memcpy(w, w_new, sizeof(double)*n);
+			f = fnew;
+		        fun_obj->grad(w, g);
+
+			gnorm = dnrm2_(&n, g, &inc);
+			if (gnorm <= eps*gnorm1)
+				break;
+		}
+		if (f < -1.0e+32)
+		{
+			info("WARNING: f < -1.0e+32\n");
+			break;
+		}
+		if (fabs(actred) <= 0 && prered <= 0)
+		{
+			info("WARNING: actred and prered <= 0\n");
+			break;
+		}
+		if (fabs(actred) <= 1.0e-12*fabs(f) &&
+		    fabs(prered) <= 1.0e-12*fabs(f))
+		{
+			info("WARNING: actred and prered too small\n");
+			break;
+		}
+	}
+
+	delete[] g;
+	delete[] r;
+	delete[] w_new;
+	delete[] s;
+}
+
+int TRON::trcg(double delta, double *g, double *s, double *r)
+{
+	int i, inc = 1;
+	int n = fun_obj->get_nr_variable();
+	double one = 1;
+	double *d = new double[n];
+	double *Hd = new double[n];
+	double rTr, rnewTrnew, alpha, beta, cgtol;
+
+	for (i=0; i<n; i++)
+	{
+		s[i] = 0;
+		r[i] = -g[i];
+		d[i] = r[i];
+	}
+	cgtol = 0.1*dnrm2_(&n, g, &inc);
+
+	int cg_iter = 0;
+	rTr = ddot_(&n, r, &inc, r, &inc);
+	while (1)
+	{
+		if (dnrm2_(&n, r, &inc) <= cgtol)
+			break;
+		cg_iter++;
+		fun_obj->Hv(d, Hd);
+
+		alpha = rTr/ddot_(&n, d, &inc, Hd, &inc);
+		daxpy_(&n, &alpha, d, &inc, s, &inc);
+		if (dnrm2_(&n, s, &inc) > delta)
+		{
+			info("cg reaches trust region boundary\n");
+			alpha = -alpha;
+			daxpy_(&n, &alpha, d, &inc, s, &inc);
+
+			double std = ddot_(&n, s, &inc, d, &inc);
+			double sts = ddot_(&n, s, &inc, s, &inc);
+			double dtd = ddot_(&n, d, &inc, d, &inc);
+			double dsq = delta*delta;
+			double rad = sqrt(std*std + dtd*(dsq-sts));
+			if (std >= 0)
+				alpha = (dsq - sts)/(std + rad);
+			else
+				alpha = (rad - std)/dtd;
+			daxpy_(&n, &alpha, d, &inc, s, &inc);
+			alpha = -alpha;
+			daxpy_(&n, &alpha, Hd, &inc, r, &inc);
+			break;
+		}
+		alpha = -alpha;
+		daxpy_(&n, &alpha, Hd, &inc, r, &inc);
+		rnewTrnew = ddot_(&n, r, &inc, r, &inc);
+		beta = rnewTrnew/rTr;
+		dscal_(&n, &beta, d, &inc);
+		daxpy_(&n, &one, r, &inc, d, &inc);
+		rTr = rnewTrnew;
+	}
+
+	delete[] d;
+	delete[] Hd;
+
+	return(cg_iter);
+}
+
+double TRON::norm_inf(int n, double *x)
+{
+	double dmax = fabs(x[0]);
+	for (int i=1; i<n; i++)
+		if (fabs(x[i]) >= dmax)
+			dmax = fabs(x[i]);
+	return(dmax);
+}
+
+void TRON::set_print_string(void (*print_string) (const char *buf))
+{
+	tron_print_string = print_string;
+}