ml4r 0.1.4 → 0.1.5

data/ext/ml4r/MachineLearning/MLRunner.cpp

@@ -0,0 +1,183 @@
+#include "MachineLearning/MLRunner.h"
+#include "MachineLearning/MLData/MLData.h"
+#include "MachineLearning/MLOutput.h"
+#include "MachineLearning/MLExperiment.h"
+#include "MachineLearning/MLParameters.h"
+#include "MachineLearning/MLEstimator.h"
+#include "MachineLearning/MLEstimatorFactory.h"
+
+#include "utils/VlcMessage.h"
+
+// #ifdef TBB_USE_THREADING_TOOLS
+// #undef TBB_USE_THREADING_TOOLS
+// #endif
+// #define TBB_USE_THREADING_TOOLS 1
+// #include "tbb/task_scheduler_init.h"
+// #include "tbb/parallel_for.h"
+// #include "tbb/blocked_range.h"
+// #include "tbb/explicit_range.h"
+
+#include <boost/foreach.hpp>
+
+MLRunner::MLRunner()
+    : m_data(0)
+{
+
+}
+
+MLRunner::~MLRunner()
+{
+
+}
+
+void MLRunner::execute()
+{
+    checks();
+    config();
+    input();
+    estimate();
+    output();
+}
+
+void MLRunner::checks()
+{
+    if (m_data == 0)
+        throw std::runtime_error("[MLRunner::checks()] - MLData is not defined - have you provided input data?");
+}
+
+void MLRunner::config()
+{
+    
+}
+
+void MLRunner::input()
+{
+
+}
+
+void MLRunner::estimate()
+{
+    vector<int>& foldNumbers = m_data->getFoldNumbers();
+    long numFolds             = foldNumbers.size();
+    long numThreads           = numFolds; // TODO: change this!
+
+    // tbb::task_scheduler_init init(numFolds);
+    // static tbb::simple_partitioner sp;
+
+    //int grainSize       = numFolds / numThreads;
+
+    m_outputObjects.resize(numFolds);
+    m_estimators.resize(numFolds);
+
+    //tbb::parallel_for(explicit_range<size_t>(0, numFolds, grainSize),
+    //    [&](const explicit_range<size_t>& r) {
+    //        int threadNumber = r.begin() / grainSize;
+    //        for(size_t foldIndex=r.begin(); foldIndex!=r.end(); ++foldIndex)
+            for (long foldIndex = 0; foldIndex < numFolds; ++foldIndex)
+            {
+                vlcMessage.Begin("Estimating");
+                int foldNumber = foldNumbers.at(foldIndex);
+
+                shared_ptr<MLEstimator> estimator = createEstimator(m_data, m_data->getTrainingExperiments(foldNumber));
+                m_estimators.at(foldIndex)    = estimator;
+                m_outputObjects.at(foldIndex) = estimator->estimate();
+                
+                vlcMessage.End();
+            }
+    //}, sp);
+}
+
+void MLRunner::output()
+{
+
+}
+
+void MLRunner::setData( MLData* data )
+{
+    m_data = data;
+}
+
+MLData* MLRunner::getData()
+{
+    return m_data;
+}
+
+vector<double> MLRunner::getPredictions( MLData* newData )
+{
+    if (m_data->initialPredictionsDefined() && !newData->initialPredictionsDefined())
+        throw std::runtime_error("Cannot apply model to new data as initial predictions are not defined (but were in initial data).");
+
+    if (m_data->getFeatures() != newData->getFeatures())
+        throw std::runtime_error("Features in prediction dataset do not match those in the estimation dataset (order is important)");
+
+    return getPredictions(newData->getExperiments());
+}
+
+vector<double> MLRunner::getPredictions( vector<shared_ptr<MLExperiment> > experiments )
+{
+    // we can get a prediction from each of our outputs, when then need to be averaged.
+    vector<double> predictions;
+    predictions.reserve(experiments.size());
+
+    BOOST_FOREACH(shared_ptr<MLExperiment> experiment, experiments)
+    {
+        vector<double> experimentPredictions;
+        experimentPredictions.reserve(m_outputObjects.size());
+
+        BOOST_FOREACH(shared_ptr<MLOutput> outputObject, m_outputObjects)
+        {
+            experimentPredictions.push_back(outputObject->predictForExperiment(experiment));
+        }
+        predictions.push_back(m_outputObjects.front()->calculateAveragePredictions(experimentPredictions));
+    }
+    return predictions;
+}
+
+vector<double> MLRunner::getMeanTrainingPredictions()
+{
+    long experimentCount = m_data->getExperiments().size();
+    vector<double> meanPredictions;
+    meanPredictions.reserve(experimentCount);
+
+    vector<vector<double> > predictionsForEachFold(experimentCount);
+    
+    BOOST_FOREACH(vector<double>& experimentPredictions, predictionsForEachFold)
+        experimentPredictions.reserve(m_outputObjects.size() - 1);
+    
+    BOOST_FOREACH(shared_ptr<MLOutput> outputObject, m_outputObjects)
+    {
+        BOOST_FOREACH(int experimentIndex, outputObject->getTrainingExperimentIndicies())
+        {
+            double prediction = outputObject->predictForExperiment(m_data->getExperiment(experimentIndex));
+            predictionsForEachFold.at(experimentIndex).push_back(prediction);
+        }
+    }
+
+    // take the mean of our predictions
+    BOOST_FOREACH(vector<double>& experimentPredictions, predictionsForEachFold)
+        meanPredictions.push_back(m_outputObjects.front()->calculateAveragePredictions(experimentPredictions));
+
+    return meanPredictions;
+}
+
+vector<double> MLRunner::getCrossValidationPredictions()
+{
+    int experimentCount = (int) m_data->getExperiments().size();
+    vector<double> predictions(experimentCount);
+    
+    int foldIndex = -1;
+    vector<int> foldNumbers = m_data->getFoldNumbers();
+
+    BOOST_FOREACH(shared_ptr<MLOutput> outputObject, m_outputObjects)
+    {
+        ++foldIndex;
+        int foldNumber = foldNumbers.at(foldIndex);
+
+        BOOST_FOREACH(shared_ptr<MLExperiment> experiment, m_data->getCrossValidationExperiments(foldNumber))
+        {
+            double prediction = outputObject->predictForExperiment(experiment);
+            predictions.at(experiment->getExperimentIndex()) = prediction;
+        }
+    }
+    return predictions;
+}

data/ext/ml4r/MachineLearning/MLUtils.cpp

@@ -0,0 +1,15 @@
+#include "MachineLearning/MLUtils.h"
+
+#include <boost/foreach.hpp>
+
+double MLUtils::getMeanY(vector<shared_ptr<MLExperiment> > experiments)
+{   
+    double sumY = 0.0, sumWeight = 0.0;
+    BOOST_FOREACH(shared_ptr<MLExperiment>& e, experiments)
+    {
+        sumY        += e->getY() * e->getWeight();
+        sumWeight   += e->getWeight();
+    }
+
+    return sumY / sumWeight;
+}

data/ext/ml4r/MachineLearning/RandomForest/RandomForestEstimator.cpp

@@ -0,0 +1,172 @@
+#include "MachineLearning/RandomForest/RandomForestEstimator.h"
+#include "MachineLearning/MLData/MLData.h"
+#include "MachineLearning/DecisionTree/DecisionTreeExperiment.h"
+#include "MachineLearning/DecisionTree/DecisionTreeNode.h"
+#include "MachineLearning/DecisionTree/NodeSplitterCategorical.h"
+#include "MachineLearning/DecisionTree/NodeSplitterContinuous.h"
+#include "MachineLearning/DecisionTree/SplitDefinition.h"
+#include "MachineLearning/RandomForest/RandomForestOutput.h"
+#include "MachineLearning/MLUtils.h"
+
+#include "utils/VlcMessage.h"
+
+#include <boost/make_shared.hpp>
+using boost::make_shared;
+
+RandomForestEstimator::RandomForestEstimator(MLData* data, 
+                                             vector<shared_ptr<MLExperiment> > experiments, 
+                                             shared_ptr<RandomForestParameters> parameters)
+                                             : MLEstimator(data, experiments), m_parameters(parameters)
+{
+    m_decisionTreeExperiments.reserve(experiments.size());
+    BOOST_FOREACH(shared_ptr<MLExperiment>& experiment, experiments)
+        m_decisionTreeExperiments.push_back(make_shared<DecisionTreeExperiment>(experiment));
+
+    vector<int> experimentIndicies;
+    experimentIndicies.reserve(experiments.size());
+    BOOST_FOREACH(shared_ptr<MLExperiment>& experiment, experiments)
+        experimentIndicies.push_back(experiment->getExperimentIndex());
+
+    m_output = shared_ptr<RandomForestOutput>(new RandomForestOutput(m_data, experimentIndicies, m_parameters));
+}
+
+RandomForestEstimator::~RandomForestEstimator()
+{
+
+}
+
+shared_ptr<MLOutput> RandomForestEstimator::estimate()
+{
+    initializeEstimator();
+    updateZ();
+
+    for (int iteration = 0; iteration < m_parameters->numIterations; ++iteration)
+    {
+        if (m_parameters->verbose)
+            vlcMessage.Begin((string("Iteration ") + boost::lexical_cast<string>(iteration + 1)).c_str());
+
+        performIteration();
+
+        if (m_parameters->verbose)
+            vlcMessage.End();
+    }
+    return shared_ptr<MLOutput>(m_output);
+}
+
+shared_ptr<MLOutput> RandomForestEstimator::estimateMore(int numTrees)
+{
+    initializeEstimator();
+    updateZ();
+    int numberOfExistingTrees = m_output->getNumTrees();
+
+    for (int iteration = 0; iteration < numTrees; ++iteration)
+    {
+        if (m_parameters->verbose)
+            vlcMessage.Begin((string("Iteration ") + boost::lexical_cast<string>(numberOfExistingTrees + iteration + 1)).c_str());
+
+        performIteration();
+
+        if (m_parameters->verbose)
+            vlcMessage.End();
+    }
+    return shared_ptr<MLOutput>(m_output);
+}
+
+void RandomForestEstimator::updateZ()
+{
+    BOOST_FOREACH(shared_ptr<DecisionTreeExperiment> e, m_decisionTreeExperiments)
+        e->setZ(e->getY());
+}
+
+void RandomForestEstimator::performIteration()
+{
+    vector<shared_ptr<DecisionTreeExperiment> > experiments;
+    size_t bagSize = m_decisionTreeExperiments.size() * m_parameters->bagFraction;
+
+    if (m_parameters->withReplacement)
+        experiments = MLUtils::bagObjectsWithReplacement<shared_ptr<DecisionTreeExperiment> >(m_decisionTreeExperiments, (int) bagSize);
+    else
+    {
+        pair<vector<shared_ptr<DecisionTreeExperiment> >,vector<shared_ptr<DecisionTreeExperiment> > > inAndOutOfBag =
+            MLUtils::bagObjectsWithoutReplacement<shared_ptr<DecisionTreeExperiment> >(m_decisionTreeExperiments, (int) std::min(m_decisionTreeExperiments.size(), bagSize));
+        experiments = inAndOutOfBag.first;
+    }
+
+    if (m_parameters->verbose)
+        vlcMessage.Begin("Constructing decision tree");
+
+    constructDecisionTree(experiments);
+
+    m_output->addHeadDecisionTreeNode(m_decisionTreeHead);
+    m_decisionTreeHead->clearExperimentsWithinTree();
+
+    if (m_parameters->verbose)
+        vlcMessage.End();
+}
+
+void RandomForestEstimator::constructDecisionTree(vector<shared_ptr<DecisionTreeExperiment> >& experiments)
+{
+    vector<shared_ptr<DecisionTreeNode> > currentGeneration;
+    vector<shared_ptr<DecisionTreeNode> > nextGeneration;
+
+    // create a head DecisionTreeNode
+    double sumZ = 0.0, sumW = 0.0;
+    BOOST_FOREACH(shared_ptr<DecisionTreeExperiment>& e, experiments)
+    {
+        double w = e->getWeight();
+        sumW += w;
+        sumZ += w * e->getZ();
+    }
+
+    m_decisionTreeHead = shared_ptr<DecisionTreeNode>(new DecisionTreeNode(experiments, sumZ, sumW, ROOT, shared_ptr<SplitDefinition>()));
+    currentGeneration.push_back(m_decisionTreeHead);
+    
+    NodeSplitter splitter(m_data, m_parameters->minObservations, m_parameters->scale);
+
+    while (!currentGeneration.empty())
+    {
+        BOOST_FOREACH(shared_ptr<DecisionTreeNode> nodeToSplit, currentGeneration)
+        {
+            if (nodeToSplit->getSumW() == 0)
+                continue;
+
+            // choose M variables to test splitting on
+            // find terminal node with best improvement for any of those variables
+            pair<vector<int>,vector<int> > inAndOut = MLUtils::bagObjectsWithoutReplacement<int>(m_featureIndices, std::min((int)m_featureIndices.size(), m_parameters->tryMVariables));
+            vector<int> featuresToConsider = inAndOut.first;
+
+            double bestImprovement = 0.0;
+            shared_ptr<SplitDefinition> bestSplit;
+
+            vector<shared_ptr<DecisionTreeNode> > children = splitter.splitNode(nodeToSplit, featuresToConsider);
+            BOOST_FOREACH(shared_ptr<DecisionTreeNode>& child, children)
+            {
+                nextGeneration.push_back(child);
+            }
+        }
+        currentGeneration = nextGeneration;
+        nextGeneration.clear();
+    }
+    
+
+}
+
+void RandomForestEstimator::initializeEstimator()
+{
+    m_missingValueDefined = m_data->missingValueDefined();
+    if (m_missingValueDefined)
+        m_missingValue        = m_data->getMissingValue();
+        
+
+    constructFeatureIndices();
+    // sortTrainingExperiments();
+}
+
+void RandomForestEstimator::constructFeatureIndices()
+{
+    BOOST_FOREACH(string feature, m_parameters->featuresToRun)
+    {
+        // note that in a given run, we may not "run" with all loaded variables.
+        m_featureIndices.push_back(m_data->getFeatureIndex(feature));
+    }
+}

data/ext/ml4r/MachineLearning/RandomForest/RandomForestOutput.cpp

@@ -0,0 +1,66 @@
+#include "MachineLearning/RandomForest/RandomForestOutput.h"
+#include "MachineLearning/RandomForest/RandomForestParameters.h"
+#include "MachineLearning/DecisionTree/DecisionTreeNode.h"
+#include "MachineLearning/DecisionTree/DecisionTreeExperiment.h"
+
+#include "utils/VlcMessage.h"
+
+#include <boost/make_shared.hpp>
+using boost::make_shared;
+
+RandomForestOutput::RandomForestOutput( MLData* trainingData, vector<int> trainingExperimentIndicies, shared_ptr<RandomForestParameters> parameters )
+    : MLOutput(trainingData, trainingExperimentIndicies), m_parameters(parameters)
+{
+
+}
+
+RandomForestOutput::~RandomForestOutput()
+{
+
+}
+
+shared_ptr<RandomForestParameters> RandomForestOutput::getParameters()
+{
+    return m_parameters;
+}
+
+double RandomForestOutput::predictForExperiment( shared_ptr<MLExperiment> experiment )
+{
+    shared_ptr<DecisionTreeExperiment> dtExperiment = make_shared<DecisionTreeExperiment>(experiment);
+    
+    double sumPrediction = 0.0;
+    int count = 0;
+    BOOST_FOREACH(shared_ptr<DecisionTreeNode>& head, m_headNodes)
+    {
+        shared_ptr<DecisionTreeNode> node = head->getTerminalNodeForExperiment(dtExperiment);
+        if (node.get() == 0)
+            node = head;
+
+        if (node->getSumW() == 0)
+        {
+            
+            vlcMessage.Write("Zero weight!! WTF!!");
+            vlcMessage.Write("SumZ: "+ boost::lexical_cast<string>(node->getSumZ()));
+            vlcMessage.Write("exp.size() " + boost::lexical_cast<string>(node->getExperiments().size()));
+            vlcMessage.Write("Node is head: " + boost::lexical_cast<string>(node == head));
+        }
+
+        if (node->isTerminalNode())
+        {
+            sumPrediction += node->getSumZ() / node->getSumW();
+            count++;
+        }
+        
+    }
+    return sumPrediction / count;
+}
+
+void RandomForestOutput::addHeadDecisionTreeNode( shared_ptr<DecisionTreeNode> node )
+{
+    m_headNodes.push_back(node);
+}
+
+int RandomForestOutput::getNumTrees()
+{
+    return (int) m_headNodes.size();
+}

data/ext/ml4r/MachineLearning/RandomForest/RandomForestRunner.cpp

@@ -0,0 +1,84 @@
+#include "MachineLearning/RandomForest/RandomForestRunner.h"
+#include "MachineLearning/RandomForest/RandomForestOutput.h"
+#include "MachineLearning/RandomForest/RandomForestEstimator.h"
+#include "MachineLearning/DecisionTree/DecisionTreeNode.h"
+#include "MachineLearning/MLData/MLData.h"
+
+#include "utils/VlcMessage.h"
+
+// #ifdef TBB_USE_THREADING_TOOLS
+// #undef TBB_USE_THREADING_TOOLS
+// #endif
+// #define TBB_USE_THREADING_TOOLS 1
+// #include "tbb/task_scheduler_init.h"
+// #include "tbb/parallel_for.h"
+// #include "tbb/blocked_range.h"
+// #include "tbb/explicit_range.h"
+
+#include <boost/pointer_cast.hpp>
+using boost::dynamic_pointer_cast;
+#include <math.h>
+
+RandomForestRunner::RandomForestRunner()
+{
+
+}
+
+RandomForestRunner::~RandomForestRunner()
+{
+
+}
+
+void RandomForestRunner::estimateMore(int numTrees)
+{
+    int numFolds   = m_data->getNumFolds();
+    int numThreads = m_data->getNumFolds();
+
+    // tbb::task_scheduler_init init(numFolds);
+    // static tbb::simple_partitioner sp;
+
+    int grainSize       = numFolds / numThreads;
+
+    // tbb::parallel_for(explicit_range<size_t>(0, numFolds, grainSize),
+    //     [&](const explicit_range<size_t>& r) {
+    //         int threadNumber = r.begin() / grainSize;
+    //         for(size_t foldIndex=r.begin(); foldIndex!=r.end(); ++foldIndex)
+            for(int foldIndex=numFolds; foldIndex<numFolds; ++foldIndex)
+            {
+                vlcMessage.Begin("Estimating more...");
+
+                shared_ptr<RandomForestEstimator> estimator = dynamic_pointer_cast<RandomForestEstimator>(m_estimators.at(foldIndex));
+                estimator->estimateMore(numTrees);
+
+                vlcMessage.End();
+            }
+    // }, sp);
+}
+
+void RandomForestRunner::config()
+{
+    vector<string>& dataFeatures = m_data->getFeatures();
+
+    if (parameters->featuresToRun.empty())
+        parameters->featuresToRun = dataFeatures;
+    else
+    {    
+        BOOST_FOREACH(string& feature, parameters->featuresToRun)
+        {
+            if (Utils::vectorIndex(dataFeatures, feature) == -1)
+                throw std::runtime_error("Feature '" + feature + "' specified as part of parameter 'featuresToRun', but feature not found in data");
+        }
+    }
+    if (parameters->featuresToRun.empty())
+        throw std::runtime_error("There are no features to run!");
+
+    if (m_data->missingValueDefined())
+        DecisionTreeNode::setMissingValue(m_data->getMissingValue());
+}
+
+
+
+shared_ptr<MLEstimator> RandomForestRunner::createEstimator( MLData* data, vector<shared_ptr<MLExperiment> > trainingExperiments )
+{
+    return shared_ptr<MLEstimator>(shared_ptr<RandomForestEstimator>(new RandomForestEstimator(data, trainingExperiments, parameters)));
+}