thera 0.0.1

data/lib/quarry/src/classifier/multinomial_bayes/multinomial_bayes_classifier.cpp

@@ -0,0 +1,40 @@
+#include "multinomial_bayes_classifier.h"
+#include "data_set/data_set.h"
+#include <math.h>
+
+double Classifier::MultinomialBayesClassifier::score(DataSet::Category *category, DataSet::Example *example) {
+
+}
+
+void Classifier::MultinomialBayesClassifier::prepare() {
+  numeric_feature_probabilities.resize(data_set->categories_size() + 1);
+  nominal_feature_probabilities.resize(data_set->categories_size() + 1);
+  DataSet::NumericFeature *numeric_feature = NULL;
+  DataSet::NominalFeature *nominal_feature = NULL;
+  int feature_count = data_set->features.size();
+  double category_sum = 0.0;
+  
+  data_set->count();
+  
+  // determine the category probabilities for each feature
+  for(int i = 1; i <= data_set->categories_size(); i++) {
+    numeric_feature_probabilities[i].reserve(feature_count);
+    nominal_feature_probabilities[i].reserve(feature_count);
+    
+    // sum the counts of each numeric feature for this category
+    category_sum = 0.0
+    for(int j = 0; j < numeric_features.size(); j++)
+      category_sum += numeric_features[j]->category_sum(i);
+    category_sum += numeric_features.size();
+    
+    // weight each numeric feature only by the number of other numeric features; nominal features are handled separately
+    for(int j = 0; j < numeric_features.size(); j++)
+      numeric_feature_probabilities[i][j] = (1.0 + numeric_features[j]->category_sum(i)) / (category_sum);
+    
+    // each value of a nominal feature is treated as if it were another feature in itself
+    for(int j = 0; j < nominal_features.size(); j++) {
+      nominal_feature = nominal_features[j];
+      nominal_feature_probabilities[j].resize(nominal_feature->values.size());
+    }
+  }
+}

data/lib/quarry/src/classifier/multinomial_bayes/multinomial_bayes_classifier.h

@@ -0,0 +1,18 @@
+#ifndef __multinomial_bayes_classifier_h__
+#define __multinomial_bayes_classifier_h__
+#include "classifier/classifier.h"
+#include <vector>
+
+namespace Classifier {
+  class MultinomialBayesClassifier : public Classifier {
+    vector<vector<double> > numeric_feature_probabilities;
+    vector<vector<vector<double> > > nominal_feature_probabilities;
+    
+  public:
+    MultinomialBayesClassifier(DataSet::DataSet *data_set) : Classifier(data_set) {}
+    double score(DataSet::Category *category, DataSet::Example *example);
+    void prepare();
+  };
+}
+
+#endif

data/lib/quarry/src/classifier/naive_bayes/naive_bayes_classifier.cpp

@@ -0,0 +1,80 @@
+#include "naive_bayes_classifier.h"
+#include "data_set/data_set.h"
+#include "data_set/dense/dense_data_set.h"
+#include "storage/binary.h"
+#include <math.h>
+
+Classifier::NaiveBayesClassifier *Classifier::NaiveBayesClassifier::clone(DataSet::DataSet *new_data_set) {
+  return new NaiveBayesClassifier(new_data_set);
+}
+
+
+void Classifier::NaiveBayesClassifier::prepare() {
+  feature_caches.resize(data_set->categories_size() + 1);
+  int features_size = data_set->features_size();
+  DataSet::NumericFeature *feature = NULL;
+  
+  data_set->count();
+  category_probabilities = data_set->category_feature()->probabilities;
+  
+  for(int i = 1; i <= data_set->categories_size(); i++) {
+    feature_caches[i].resize(features_size);
+    
+    for(int j = 0; j < features_size; j++) {
+      if(!numeric_features[j])
+        continue;
+      feature = (DataSet::NumericFeature *) data_set->features[j];
+      feature_caches[i][j].denom = 2 * feature->category_variance(i);
+      feature_caches[i][j].lhs = 1 / sqrt(2 * M_PI * feature->category_variance(i));
+    }
+  }
+}
+
+
+double Classifier::NaiveBayesClassifier::score(int category, DataSet::Example *example) {
+  DataSet::SparseExample *sparse_example = NULL;
+  DataSet::SparseExample::Value *sparse_value = NULL;
+  DataSet::DenseExample *dense_example = NULL;
+  double dense_value = 0.0, probability = 0.0;
+  
+  if(typeid(*example) == typeid(DataSet::SparseExample)) {
+    sparse_example = (DataSet::SparseExample *) example;
+    for(int i = 0; i < example->size; i++) {
+      sparse_value = &(sparse_example->values[i]);
+      if(numeric_features[sparse_value->index])
+        score_numeric_feature(sparse_value->index, sparse_value->value, category, &probability);
+      else if(sparse_value->index != data_set->category_index)
+        score_nominal_feature(sparse_value->index, sparse_value->value, category, &probability);
+    }
+    
+  } else {
+    dense_example = (DataSet::DenseExample *) example;
+    for(int i = 0; i < example->size; i++) {
+      dense_value = dense_example->get_value(i);
+      if(numeric_features[i])
+        score_numeric_feature(i, dense_value, category, &probability);
+      else if(i != data_set->category_index)
+        score_nominal_feature(i, dense_value, category, &probability);
+    }
+  }
+  
+  return probability * category_probabilities[category];
+}
+
+void Classifier::NaiveBayesClassifier::write_binary(Storage::Binary *file) {
+  int category_count = feature_caches.size();
+  file->write_int(category_count);
+  file->write_vector<double>(&category_probabilities);
+  
+  for(int i = 1; i <= category_count; i++)
+    file->write_vector<NumericFeatureCache>(&feature_caches[i]);
+}
+
+void Classifier::NaiveBayesClassifier::read_binary(Storage::Binary *file) {
+  int category_count = file->read_int();
+  feature_caches.resize(category_count + 1);
+  category_probabilities = *(file->read_vector<double>());
+  
+  for(int i = 1; i <= category_count; i++)
+    feature_caches[i] = *(file->read_vector<NumericFeatureCache>());
+}

data/lib/quarry/src/classifier/naive_bayes/naive_bayes_classifier.h

@@ -0,0 +1,52 @@
+#ifndef __naive_bayes_classifier_h__
+#define __naive_bayes_classifier_h__
+#include "classifier/classifier.h"
+#include <vector>
+
+namespace Classifier {
+  class NaiveBayesClassifier : public Classifier {
+    typedef struct {
+      double  denom;
+      double  lhs;
+    } NumericFeatureCache;
+    vector<vector<NumericFeatureCache> > feature_caches; // features_caches[category_index][numeric_feature]
+    vector<double> category_probabilities;
+    
+    // (1 / sqrt(2PI * var)) * e^(-((value - mean) ^ 2) / (2 * var))
+    void score_numeric_feature(int index, double value, int category, double *probability) {
+      DataSet::NumericFeature *numeric_feature = (DataSet::NumericFeature *) data_set->features[index];
+      if(feature_caches[category][index].denom != 0.0) {
+        double numerator = -1 * pow(value - numeric_feature->category_mean(category), 2);
+        if(*probability == 0.0)
+          *probability = 1.0;
+        *probability = *probability * (feature_caches[category][index].lhs * exp(numerator / feature_caches[category][index].denom));
+      }
+    }
+    
+    void score_nominal_feature(int index, double value, int category, double *probability) {
+      DataSet::NominalFeature *nominal_feature = (DataSet::NominalFeature *) data_set->features[index];
+      *probability = *probability * nominal_feature->category_value_probability(category, (int)value);
+    }
+    
+  public:
+    static const uint32_t file_mark = 'naiv';
+    NaiveBayesClassifier(DataSet::DataSet *data_set) : Classifier(data_set) {}
+    NaiveBayesClassifier *clone(DataSet::DataSet *new_data_set);
+    
+    double score(int category, DataSet::Example *example);
+    void prepare();
+    void write_binary(Storage::Binary *file);
+    void read_binary(Storage::Binary *file);
+    uint32_t mark() { return file_mark; }
+    
+    void print() {
+      cout << "NB:" << endl;
+      for(unsigned int i = 0; i < feature_caches.size(); i++) {
+        for(unsigned int j = 0; j < feature_caches[i].size(); j++)
+          cout << "C" << i << "F" << j << ":" << feature_caches[i][j].denom << ", l:" << feature_caches[i][j].lhs << endl;
+      }
+    }
+  };
+}
+
+#endif

data/lib/quarry/src/data_set/data_set.cpp

@@ -0,0 +1,130 @@
+#include <iostream>
+#include "data_set.h"
+#include "classifier/classifier.h"
+#include "metrics/confusion_matrix.h"
+#include "dense/dense_data_set.h"
+#include "dense/dense_example.h"
+#include "sparse/sparse_data_set.h"
+#include "sparse/sparse_example.h"
+
+DataSet::NominalFeature *DataSet::DataSet::new_nominal_feature(string name) {
+  NominalFeature *feature = new NominalFeature(name, features.size());
+  feature_names[name] = feature;
+  features.push_back(feature);
+  return feature;
+}
+
+
+DataSet::NumericFeature *DataSet::DataSet::new_numeric_feature(string name) {
+  NumericFeature *feature = new NumericFeature(name, features.size());
+  feature_names[name] = feature;
+  features.push_back(feature);
+  return feature;
+}
+
+
+void DataSet::DataSet::set_category_index(int index) {
+  category_index = index;
+  for(vector<Feature *>::iterator feature = features.begin(); feature < features.end(); feature++)
+    (*feature)->reset();
+  counted = false;
+  indexed = false;
+}
+
+
+void DataSet::DataSet::count() {
+  if(counted)
+    return;
+  
+  // initialise each feature for counting
+  for(vector<Feature *>::iterator feature = features.begin(); feature < features.end(); feature++)
+    (*feature)->prepare_for_counting(this);
+  
+  // implementation optimised count
+  perform_count();
+  
+  // calculate and finalise counts
+  for(vector<Feature *>::iterator feature = features.begin(); feature < features.end(); feature++)
+    (*feature)->finalise_counting(this);
+
+  counted = true;
+}
+
+
+void DataSet::DataSet::index() {
+  if(indexed)
+    return;
+  
+  // initialise each feature for indexing
+  for(vector<Feature *>::iterator feature = features.begin(); feature < features.end(); feature++)
+    (*feature)->prepare_for_indexing(this);
+  
+  // index
+  for(vector<Example *>::iterator example = examples.begin(); example < examples.end(); example++) {
+    for(int i = 0; i < features.size(); i++)
+      features[i]->index_example((*example)->get_value(i), *example);
+  }
+  
+  // finalise indexing
+  for(vector<Feature *>::iterator feature = features.begin(); feature < features.end(); feature++)
+    (*feature)->finalise_indexing(this);
+  
+  indexed = true;
+}
+
+
+// TODO: this isn't really stratification; categories should be proportionally represented
+vector<vector<DataSet::Example *> > *DataSet::DataSet::stratify(int number_of_folds) {
+  int examples_per_fold = examples.size() / number_of_folds;
+  Example *example;
+  vector<vector<Example *> > *folds = new vector<vector<Example *> >(number_of_folds, vector<Example *>(examples_per_fold, NULL));
+  
+  for(int fold = 0; fold < number_of_folds; fold++) {
+    for(int i = 0; i < examples_per_fold; i++) {
+      (*folds)[fold][i] = examples[fold + (i * number_of_folds)];
+    }
+  }
+  
+  return folds;
+}
+
+
+ConfusionMatrix *DataSet::DataSet::cross_fold_validation(Classifier::Classifier *classifier, int number_of_folds) {
+  vector<vector<Example *> > *folds = stratify(number_of_folds);
+  ConfusionMatrix *overall_matrix = new ConfusionMatrix(this);
+  Classifier::Classifier *test_classifier = NULL;
+  ConfusionMatrix *result = NULL;
+  DataSet *test_set = NULL;
+  Example *example = NULL;
+  int predicted = 0;
+  
+  int examples_per_fold = examples.size() / number_of_folds;
+  int examples_per_test = examples_per_fold * (number_of_folds - 1);
+  
+  for(int fold = 0; fold < number_of_folds; fold++) {
+    cout << "Running fold " << fold << endl;
+    test_set = clone_without_examples();
+    test_set->examples.reserve(examples_per_test);
+    for(int i = 0; i < number_of_folds; i++) {
+      if(i != fold)
+        test_set->examples.insert(test_set->examples.begin() + (examples_per_fold * (i > fold ? i - 1 : i)), (*folds)[i].begin(), (*folds)[i].end());
+    }
+    
+    test_classifier = classifier->clone(test_set);
+    result = new ConfusionMatrix(test_set);
+    test_classifier->prepare();
+    
+    for(int i = 0; i < examples_per_fold; i++) {
+      example = (*folds)[fold][i];
+      predicted = test_classifier->classify(example);
+      result->add(predicted, (int)example->get_value(category_index));
+    }
+    
+    overall_matrix->merge(result);
+    delete test_classifier;
+    delete test_set;
+    delete result;
+  }
+  
+  return overall_matrix;
+}

data/lib/quarry/src/data_set/data_set.h

@@ -0,0 +1,78 @@
+#ifndef __data_set_h__
+#define __data_set_h__
+#include <tr1/unordered_map>
+#include <vector>
+#include <string>
+#include "example.h"
+#include "feature.h"
+#include "features/numeric_feature.h"
+#include "features/nominal_feature.h"
+using namespace std;
+
+namespace Classifier { class Classifier; }
+class ConfusionMatrix;
+
+namespace DataSet {
+  class DataSet {
+    virtual void perform_count() {}
+    virtual void perform_index() {}
+  public:
+    DataSet() : category_index(-1), counted(false), indexed(false) {}
+    DataSet(DataSet *other) : name(other->name), category_index(other->category_index), counted(false), indexed(false) {
+      Feature *feature = NULL;
+      for(unsigned int i = 0; i < other->features.size(); i++) {
+        feature = other->features[i]->clone();
+        feature_names[feature->name] = feature;
+        features.push_back(feature);
+      }
+    }
+    virtual DataSet *clone_without_examples() { return NULL; }
+    
+    tr1::unordered_map<string, Feature *> feature_names;
+    vector<Feature *>   features;
+    vector<Example *>   examples;
+    string              name;
+    int                 category_index;
+    bool                counted;
+    bool                indexed;
+    
+    NumericFeature  *new_numeric_feature(string name);
+    NominalFeature  *new_nominal_feature(string name);
+    virtual Example *new_example() { return NULL; }
+    
+    void count();
+    void index();
+    
+    vector<vector<Example *> > *stratify(int number_of_folds);
+    ConfusionMatrix *cross_fold_validation(Classifier::Classifier *classifier, int number_of_folds);
+    
+    string  get_name()                    { return name; }
+    void    set_name(string new_name)     { name = new_name; }
+    int     get_category_index()          { return category_index; }
+    int     features_size()               { return features.size(); }
+    int     examples_size()               { return examples.size(); }
+    int     categories_size()             { return ((NominalFeature *)features[category_index])->indexes.size(); }
+    NominalFeature *category_feature()    { return (NominalFeature *)features[category_index]; }
+    void    set_category_index(int index);
+    
+    Feature  *get_feature_by_name(string name)  { return feature_names[name]; }
+    Feature  *get_feature_by_index(int index)   { return features[index]; }
+    Example  *get_example_by_index(int index)   { return examples[index]; }
+    
+    NumericFeature *get_or_create_numeric_feature_by_name(string name)  {
+      NumericFeature *feature = (NumericFeature *)feature_names[name];
+      if(feature == NULL)
+        feature = new_numeric_feature(name);
+      return feature;
+    }
+    
+    NominalFeature *get_or_create_nominal_feature_by_name(string name)  {
+      NominalFeature *feature = (NominalFeature *)feature_names[name];
+      if(feature == NULL)
+        feature = new_nominal_feature(name);
+      return feature;
+    }
+  };
+}
+
+#endif

data/lib/quarry/src/data_set/dense/dense_data_set.h

@@ -0,0 +1,39 @@
+#ifndef __dense_data_set_h__
+#define __dense_data_set_h__
+#include "dense_example.h"
+
+namespace DataSet {
+  class DenseDataSet : public DataSet {
+    void perform_count() {
+      int example_category_index = 0;
+      double value = 0.0;
+      
+      for(vector<Example *>::iterator example = examples.begin(); example < examples.end(); example++) {
+        example_category_index = (int)((*example)->get_value(category_index));
+        for(unsigned int i = 0; i < features.size(); i++) {
+          value = (*example)->get_value(i);
+          features[i]->count_example(value, example_category_index);
+        }
+      }
+    }
+    
+    void perform_index() {
+    }
+    
+  public:
+    DenseDataSet() : DataSet() {}
+    DenseDataSet(DataSet *other) : DataSet(other) {}
+    
+    DenseDataSet *clone_without_examples() {
+      return new DenseDataSet(this);
+    }
+    
+    DenseExample *new_example() {
+      DenseExample *example = new DenseExample(features.size());
+      examples.push_back(example);
+      return example;
+    }
+  };
+}
+
+#endif

data/lib/quarry/src/data_set/dense/dense_example.h

@@ -0,0 +1,44 @@
+#include <stdexcept>
+#ifndef __dense_data_set_example_h__
+#define __dense_data_set_example_h__
+#include "dense_data_set.h"
+#include <iostream>
+
+namespace DataSet {
+  class DenseExample : public Example {
+  public:
+    double *values;
+    
+    DenseExample(int size) : Example(size) {
+      values = (double *) calloc(size, sizeof(double));
+    }
+    
+    ~DenseExample() {
+      free(values);
+    }
+    
+    double get_value(int index) {
+      return values[index];
+    }
+    
+    void set_value(int index, double new_value) {
+      values[index] = new_value;
+    }
+    
+    double euclidean_distance(Example *other_example) {
+      return 0.0;
+    }
+    
+    double cosine_distance(Example *other_example) {
+      return 0.0;
+    }
+    
+    void print() {
+      for(int i = 0; i < size; i++)
+        cout << values[i] << ",";
+      cout << endl;
+    }
+  };
+}
+
+#endif

data/lib/quarry/src/data_set/example.cpp

@@ -0,0 +1,10 @@
+#include "data_set.h"
+#include "example.h"
+
+int DataSet::Example::category_index(DataSet *data_set) {
+  return (int)get_value(data_set->category_index);
+}
+
+void DataSet::Example::set_category_index(DataSet *data_set, int index) {
+  set_value(data_set->category_index, index);
+}

data/lib/quarry/src/data_set/example.h

@@ -0,0 +1,23 @@
+#ifndef __example_h__
+#define __example_h__
+
+namespace DataSet {
+  class DataSet;
+  class Example {
+  public:
+    int size;
+    
+    Example(int size) : size(size) {}
+    virtual ~Example() {}
+    
+    int  category_index(DataSet *data_set);
+    void set_category_index(DataSet *data_set, int index);
+    virtual double  get_value(int index) { return 0.0; }
+    virtual void    set_value(int index, double new_value) {}
+    virtual double  euclidean_distance(Example *other_example) { return 0.0; }
+    virtual double  cosine_distance(Example *other_example) { return 0.0; }
+    virtual void    print() {}
+  };
+}
+
+#endif

data/lib/quarry/src/data_set/feature.h

@@ -0,0 +1,36 @@
+#ifndef __feature_h__
+#define __feature_h__
+#include <string>
+using namespace std;
+
+namespace DataSet {
+  class DataSet;
+  
+  class Feature {
+  public:
+    string  name;
+    int     index;
+    
+    string  get_name()                  { return name; }
+    int     get_index()                 { return index; }
+    void    set_name(string new_name)   { name = new_name; }
+    void    set_index(int new_index)    { index = new_index; }
+    
+    Feature(string name, int index) : name(name), index(index) {}
+    virtual Feature *clone() { return NULL; }
+    virtual void reset() {}
+    virtual void print() {}
+
+    // counting
+    virtual void prepare_for_counting(DataSet *data_set) {}
+    virtual void count_example(double value, int category_index) {}
+    virtual void finalise_counting(DataSet *data_set) {}
+    
+    // indexing
+    virtual void prepare_for_indexing(DataSet *data_set) {}
+    virtual void index_example(double value, Example *example) {}
+    virtual void finalise_indexing(DataSet *data_set) {}
+  };
+}
+
+#endif

data/lib/quarry/src/data_set/features/nominal_feature.cpp

@@ -0,0 +1,57 @@
+#include "nominal_feature.h"
+#include "data_set/data_set.h"
+#include "data_set/example.h"
+
+
+void DataSet::NominalFeature::prepare_for_counting(DataSet *data_set) {
+  int categories_count = data_set->categories_size();
+  int values_count = names.size();
+  
+  category_frequencies.resize(categories_count + 1);
+  category_probabilities.resize(categories_count + 1);
+  probabilities.resize(values_count);
+  frequencies.resize(values_count);
+  
+  for(int i = 1; i <= categories_count; i++) {
+    category_frequencies[i].resize(values_count);
+    category_probabilities[i].resize(values_count);
+  }
+}
+
+void DataSet::NominalFeature::count_example(double value, int category_index) {
+  frequencies[(int)value] += 1;
+  category_frequencies[category_index][(int)value] += 1;
+}
+
+void DataSet::NominalFeature::finalise_counting(DataSet *data_set) {
+  int categories_count = data_set->categories_size();
+  int examples_count = data_set->examples.size();
+  int values_count = names.size();
+  
+  // overall value probabilities
+  for(int i = 0; i < values_count; i++)
+    probabilities[i] = ((double)frequencies[i] + 1) / examples_count;
+  
+  // value probabilities per category
+  if(index != data_set->category_index) {
+    for(int i = 1; i <= categories_count; i++) {
+      for(int j = 0; j < values_count; j++)
+        category_probabilities[i][j] = ((double)category_frequencies[i][j] + 1) / data_set->category_feature()->value_frequency(i);
+    }
+  }
+}
+
+void DataSet::NominalFeature::prepare_for_indexing(DataSet *data_set) {
+  examples_with_value.resize(names.size());
+}
+
+void DataSet::NominalFeature::index_example(double value, Example *example) {
+  examples_with_value[(int)value].push_back(example);
+}
+
+void DataSet::NominalFeature::finalise_indexing(DataSet *data_set) {}
+
+void DataSet::NominalFeature::print() {
+  for(int i = 1; i < names.size(); i++)
+    cout << i << ": " << names[i] << endl;
+}