nlptoolkit-classification 1.0.0

package/source/Classifier/RandomForest.ts

@@ -0,0 +1,29 @@
+import {Classifier} from "./Classifier";
+import {InstanceList} from "../InstanceList/InstanceList";
+import {Parameter} from "../Parameter/Parameter";
+import {RandomForestParameter} from "../Parameter/RandomForestParameter";
+import {DecisionTree} from "../Model/DecisionTree/DecisionTree";
+import {DecisionNode} from "../Model/DecisionTree/DecisionNode";
+import {TreeEnsembleModel} from "../Model/TreeEnsembleModel";
+
+export class RandomForest extends Classifier{
+
+    /**
+     * Training algorithm for random forest classifier. Basically the algorithm creates K distinct decision trees from
+     * K bootstrap samples of the original training set.
+     *
+     * @param trainSet   Training data given to the algorithm
+     * @param parameters Parameters of the bagging trees algorithm. ensembleSize returns the number of trees in the random forest.
+     */
+    train(trainSet: InstanceList, parameters: Parameter): void {
+        let forestSize = (<RandomForestParameter> parameters).getEnsembleSize();
+        let forest = new Array<DecisionTree>();
+        for (let i = 0; i < forestSize; i++){
+            let bootstrap = trainSet.bootstrap(i);
+            let tree = new DecisionTree(new DecisionNode(new InstanceList(bootstrap.getSample()), undefined, <RandomForestParameter> parameters, false));
+            forest.push(tree);
+        }
+        this.model = new TreeEnsembleModel(forest);
+    }
+
+}

package/source/DataSet/DataDefinition.ts

@@ -0,0 +1,106 @@
+import {AttributeType} from "../Attribute/AttributeType";
+import {FeatureSubSet} from "../FeatureSelection/FeatureSubSet";
+
+export class DataDefinition {
+
+    private attributeTypes: Array<AttributeType>
+
+    /**
+     * Constructor for creating a new {@link DataDefinition} with given attribute types.
+     *
+     * @param attributeTypes Attribute types of the data definition.
+     */
+    constructor(attributeTypes?: Array<AttributeType>) {
+        if (attributeTypes != undefined){
+            this.attributeTypes = attributeTypes
+        }
+    }
+
+    /**
+     * Returns the number of attribute types.
+     *
+     * @return Number of attribute types.
+     */
+    attributeCount(): number{
+        return this.attributeTypes.length
+    }
+
+    /**
+     * Counts the occurrences of binary and discrete type attributes.
+     *
+     * @return Count of binary and discrete type attributes.
+     */
+    discreteAttributeCount(): number{
+        let count = 0;
+        for (let attributeType of this.attributeTypes) {
+            if (attributeType == AttributeType.DISCRETE || attributeType == AttributeType.BINARY) {
+                count++;
+            }
+        }
+        return count;
+    }
+
+    /**
+     * Counts the occurrences of binary and continuous type attributes.
+     *
+     * @return Count of of binary and continuous type attributes.
+     */
+    continuousAttributeCount(): number{
+        let count = 0;
+        for (let attributeType of this.attributeTypes) {
+            if (attributeType == AttributeType.CONTINUOUS) {
+                count++;
+            }
+        }
+        return count;
+    }
+
+    /**
+     * Returns the attribute type of the corresponding item at given index.
+     *
+     * @param index Index of the attribute type.
+     * @return Attribute type of the corresponding item at given index.
+     */
+    getAttributeType(index: number): AttributeType{
+        return this.attributeTypes[index]
+    }
+
+    /**
+     * Adds an attribute type to the list of attribute types.
+     *
+     * @param attributeType Attribute type to add to the list of attribute types.
+     */
+    addAttribute(attributeType: AttributeType){
+        this.attributeTypes.push(attributeType)
+    }
+
+    /**
+     * Removes the attribute type at given index from the list of attributes.
+     *
+     * @param index Index to remove attribute type from list.
+     */
+    removeAttribute(index: number){
+        this.attributeTypes.splice(index, 1)
+    }
+
+    /**
+     * Clears all the attribute types from list.
+     */
+    removeAllAttributes(){
+        this.attributeTypes = new Array<AttributeType>()
+    }
+
+    /**
+     * Generates new subset of attribute types by using given feature subset.
+     *
+     * @param featureSubSet {@link FeatureSubSet} input.
+     * @return DataDefinition with new subset of attribute types.
+     */
+    getSubSetOfFeatures(featureSubSet: FeatureSubSet): DataDefinition{
+        let newAttributeTypes = new Array<AttributeType>();
+        for (let i = 0; i < featureSubSet.size(); i++) {
+            newAttributeTypes.push(this.attributeTypes[featureSubSet.get(i)]);
+        }
+        return new DataDefinition(newAttributeTypes);
+    }
+}

package/source/DataSet/DataSet.ts

@@ -0,0 +1,279 @@
+import {InstanceList} from "../InstanceList/InstanceList";
+import {DataDefinition} from "./DataDefinition";
+import * as fs from "fs";
+import {AttributeType} from "../Attribute/AttributeType";
+import {Instance} from "../Instance/Instance";
+import {CompositeInstance} from "../Instance/CompositeInstance";
+import {ContinuousAttribute} from "../Attribute/ContinuousAttribute";
+import {DiscreteAttribute} from "../Attribute/DiscreteAttribute";
+import {BinaryAttribute} from "../Attribute/BinaryAttribute";
+import {DiscreteIndexedAttribute} from "../Attribute/DiscreteIndexedAttribute";
+import {Partition} from "../InstanceList/Partition";
+import {FeatureSubSet} from "../FeatureSelection/FeatureSubSet";
+
+export class DataSet {
+
+    private instances: InstanceList = new InstanceList()
+    private definition: DataDefinition = undefined
+
+    /**
+     * Constructor for generating a new {@link DataSet} with given {@link DataDefinition}.
+     *
+     * @param definition Data definition of the data set.
+     * @param separator  Separator character which separates the attribute values in the data file.
+     * @param fileName   Name of the data set file.
+     */
+    constructor(definition?: any, separator?: string, fileName?: string) {
+        if (definition != undefined){
+            if (definition instanceof DataDefinition){
+                this.definition = definition
+            } else {
+                if (separator == undefined){
+                    let data = fs.readFileSync(fileName, 'utf8')
+                    let i = 0
+                    let lines = data.split("\n")
+                    for (let line of lines){
+                        let attributes = line.split(",");
+                        if (i == 0) {
+                            for (let j = 0; j < attributes.length - 1; j++) {
+                                if (Number.isNaN(attributes[j])){
+                                    definition.addAttribute(AttributeType.DISCRETE);
+                                } else {
+                                    Number.parseFloat(attributes[j]);
+                                    definition.addAttribute(AttributeType.CONTINUOUS);
+                                }
+                            }
+                        } else {
+                            if (attributes.length != definition.attributeCount() + 1) {
+                                continue;
+                            }
+                        }
+                        let instance : Instance
+                        if (!attributes[attributes.length - 1].includes(";")) {
+                            instance = new Instance(attributes[attributes.length - 1]);
+                        } else {
+                            let labels = attributes[attributes.length - 1].split(";");
+                            instance = new CompositeInstance(labels[0]);
+                            (<CompositeInstance> instance).setPossibleClassLabels(labels.slice(1))
+                        }
+                        for (let j = 0; j < attributes.length - 1; j++) {
+                            switch (definition.getAttributeType(j)) {
+                                case AttributeType.CONTINUOUS:
+                                    instance.addAttribute(new ContinuousAttribute(Number.parseFloat(attributes[j])));
+                                    break;
+                                case AttributeType.DISCRETE:
+                                    instance.addAttribute(new DiscreteAttribute(attributes[j]));
+                                    break;
+                            }
+                        }
+                        if (instance.attributeSize() == definition.attributeCount()) {
+                            this.instances.add(instance);
+                        }
+                        i++;
+                    }
+                }
+                else {
+                    this.definition = definition;
+                    this.instances = new InstanceList(definition, separator, fileName);
+                }
+            }
+        }
+    }
+
+    /**
+     * Checks the correctness of the attribute type, for instance, if the attribute of given instance is a Binary attribute,
+     * and the attribute type of the corresponding item of the data definition is also a Binary attribute, it then
+     * returns true, and false otherwise.
+     *
+     * @param instance {@link Instance} to checks the attribute type.
+     * @return true if attribute types of given {@link Instance} and data definition matches.
+     */
+    private checkDefinition(instance: Instance): boolean{
+        for (let i = 0; i < instance.attributeSize(); i++) {
+            if (instance.getAttribute(i) instanceof BinaryAttribute) {
+                if (this.definition.getAttributeType(i) != AttributeType.BINARY)
+                    return false;
+            } else {
+                if (instance.getAttribute(i) instanceof DiscreteIndexedAttribute) {
+                    if (this.definition.getAttributeType(i) != AttributeType.DISCRETE_INDEXED)
+                        return false;
+                } else {
+                    if (instance.getAttribute(i) instanceof DiscreteAttribute) {
+                        if (this.definition.getAttributeType(i) != AttributeType.DISCRETE)
+                            return false;
+                    } else {
+                        if (instance.getAttribute(i) instanceof ContinuousAttribute) {
+                            if (this.definition.getAttributeType(i) != AttributeType.CONTINUOUS)
+                                return false;
+                        }
+                    }
+                }
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Adds the attribute types according to given {@link Instance}. For instance, if the attribute type of given {@link Instance}
+     * is a Discrete type, it than adds a discrete attribute type to the list of attribute types.
+     *
+     * @param instance {@link Instance} input.
+     */
+    private setDefinition(instance: Instance){
+        let attributeTypes = new Array<AttributeType>();
+        for (let i = 0; i < instance.attributeSize(); i++) {
+            if (instance.getAttribute(i) instanceof BinaryAttribute) {
+                attributeTypes.push(AttributeType.BINARY);
+            } else {
+                if (instance.getAttribute(i) instanceof DiscreteIndexedAttribute) {
+                    attributeTypes.push(AttributeType.DISCRETE_INDEXED);
+                } else {
+                    if (instance.getAttribute(i) instanceof DiscreteAttribute) {
+                        attributeTypes.push(AttributeType.DISCRETE);
+                    } else {
+                        if (instance.getAttribute(i) instanceof ContinuousAttribute) {
+                            attributeTypes.push(AttributeType.CONTINUOUS);
+                        }
+                    }
+                }
+            }
+        }
+        this.definition = new DataDefinition(attributeTypes);
+    }
+
+    /**
+     * Returns the size of the {@link InstanceList}.
+     *
+     * @return Size of the {@link InstanceList}.
+     */
+    sampleSize(): number{
+        return this.instances.size()
+    }
+
+    /**
+     * Returns the size of the class label distribution of {@link InstanceList}.
+     *
+     * @return Size of the class label distribution of {@link InstanceList}.
+     */
+    classCount(): number{
+        return this.instances.classDistribution().size
+    }
+
+    /**
+     * Returns the number of attribute types at {@link DataDefinition} list.
+     *
+     * @return The number of attribute types at {@link DataDefinition} list.
+     */
+    attributeCount(): number{
+        return this.definition.attributeCount()
+    }
+
+    /**
+     * Returns the number of discrete attribute types at {@link DataDefinition} list.
+     *
+     * @return The number of discrete attribute types at {@link DataDefinition} list.
+     */
+    discreteAttributeCount(): number{
+        return this.definition.discreteAttributeCount()
+    }
+
+    /**
+     * Returns the number of continuous attribute types at {@link DataDefinition} list.
+     *
+     * @return The number of continuous attribute types at {@link DataDefinition} list.
+     */
+    continuousAttributeCount(): number{
+        return this.definition.continuousAttributeCount()
+    }
+
+    /**
+     * Returns the accumulated {@link String} of class labels of the {@link InstanceList}.
+     *
+     * @return The accumulated {@link String} of class labels of the {@link InstanceList}.
+     */
+    getClasses(): string{
+        let classLabels = this.instances.getDistinctClassLabels();
+        let result = classLabels[0];
+        for (let i = 1; i < classLabels.length; i++) {
+            result = result + ";" + classLabels[i];
+        }
+        return result;
+    }
+
+    /**
+     * Adds a new instance to the {@link InstanceList}.
+     *
+     * @param current {@link Instance} to add.
+     */
+    addInstance(current: Instance){
+        if (this.definition == undefined) {
+            this.setDefinition(current);
+            this.instances.add(current);
+        } else {
+            if (this.checkDefinition(current)) {
+                this.instances.add(current);
+            }
+        }
+    }
+
+    /**
+     * Adds all the instances of given instance list to the {@link InstanceList}.
+     *
+     * @param instanceList {@link InstanceList} to add instances from.
+     */
+    addInstanceList(instanceList: Array<Instance>){
+        for (let instance of instanceList) {
+            this.addInstance(instance);
+        }
+    }
+
+    /**
+     * Returns the instances of {@link InstanceList}.
+     *
+     * @return The instances of {@link InstanceList}.
+     */
+    getInstances(): Array<Instance>{
+        return this.instances.getInstances()
+    }
+
+    /**
+     * Returns instances of the items at the list of instance lists from the partitions.
+     *
+     * @return Instances of the items at the list of instance lists from the partitions.
+     */
+    getClassInstances(): Array<Array<Instance>>{
+        return new Partition(this.instances).getLists()
+    }
+
+    /**
+     * Accessor for the {@link InstanceList}.
+     *
+     * @return The {@link InstanceList}.
+     */
+    getInstanceList(): InstanceList{
+        return this.instances
+    }
+
+    /**
+     * Accessor for the data definition.
+     *
+     * @return The data definition.
+     */
+    getDataDefinition(): DataDefinition{
+        return this.definition
+    }
+
+    /**
+     * Return a subset generated via the given {@link FeatureSubSet}.
+     *
+     * @param featureSubSet {@link FeatureSubSet} input.
+     * @return Subset generated via the given {@link FeatureSubSet}.
+     */
+    getSubSetOfFeatures(featureSubSet: FeatureSubSet): DataSet{
+        let result = new DataSet(this.definition.getSubSetOfFeatures(featureSubSet));
+        for (let i = 0; i < this.instances.size(); i++) {
+            result.addInstance(this.instances.get(i).getSubSetOfFeatures(featureSubSet));
+        }
+        return result;
+    }
+}

package/source/DistanceMetric/DistanceMetric.ts

@@ -0,0 +1,6 @@
+import {Instance} from "../Instance/Instance";
+
+export interface DistanceMetric {
+
+    distance(instance1: Instance, instance2: Instance): number
+}

package/source/DistanceMetric/EuclidianDistance.ts

@@ -0,0 +1,32 @@
+import {DistanceMetric} from "./DistanceMetric";
+import {Instance} from "../Instance/Instance";
+import {DiscreteAttribute} from "../Attribute/DiscreteAttribute";
+import {ContinuousAttribute} from "../Attribute/ContinuousAttribute";
+
+export class EuclidianDistance implements DistanceMetric{
+
+    /**
+     * Calculates Euclidian distance between two instances. For continuous features: \sum_{i=1}^d (x_i^(1) - x_i^(2))^2,
+     * For discrete features: \sum_{i=1}^d 1(x_i^(1) == x_i^(2))
+     *
+     * @param instance1 First instance
+     * @param instance2 Second instance
+     * @return Euclidian distance between two instances.
+     */
+    distance(instance1: Instance, instance2: Instance): number {
+        let result = 0;
+        for (let i = 0; i < instance1.attributeSize(); i++) {
+            if (instance1.getAttribute(i) instanceof DiscreteAttribute && instance2.getAttribute(i) instanceof DiscreteAttribute) {
+                if ((<DiscreteAttribute> instance1.getAttribute(i)).getValue() != null && (<DiscreteAttribute> instance1.getAttribute(i)).getValue() != (<DiscreteAttribute> instance2.getAttribute(i)).getValue()) {
+                    result += 1;
+                }
+            } else {
+                if (instance1.getAttribute(i) instanceof ContinuousAttribute && instance2.getAttribute(i) instanceof ContinuousAttribute) {
+                    result += Math.pow((<ContinuousAttribute> instance1.getAttribute(i)).getValue() - (<ContinuousAttribute> instance2.getAttribute(i)).getValue(), 2);
+                }
+            }
+        }
+        return result;
+    }
+
+}

package/source/DistanceMetric/MahalanobisDistance.ts

@@ -0,0 +1,33 @@
+import {DistanceMetric} from "./DistanceMetric";
+import {Instance} from "../Instance/Instance";
+import {Matrix} from "nlptoolkit-math/dist/Matrix";
+
+export class MahalanobisDistance implements DistanceMetric{
+
+    private covarianceInverse: Matrix
+
+    /**
+     * Constructor for the MahalanobisDistance class. Basically sets the inverse of the covariance matrix.
+     *
+     * @param covarianceInverse Inverse of the covariance matrix.
+     */
+    constructor(covarianceInverse: Matrix) {
+        this.covarianceInverse = covarianceInverse
+    }
+
+    /**
+     * Calculates Mahalanobis distance between two instances. (x^(1) - x^(2)) S (x^(1) - x^(2))^T
+     *
+     * @param instance1 First instance.
+     * @param instance2 Second instance.
+     * @return Mahalanobis distance between two instances.
+     */
+    distance(instance1: Instance, instance2: Instance): number {
+        let v1 = instance1.toVector();
+        let v2 = instance2.toVector();
+        v1.subtract(v2);
+        let v3 = this.covarianceInverse.multiplyWithVectorFromLeft(v1);
+        return v3.dotProduct(v1);
+    }
+
+}

package/source/Experiment/BootstrapRun.ts

@@ -0,0 +1,37 @@
+import {MultipleRun} from "./MultipleRun";
+import {Experiment} from "./Experiment";
+import {ExperimentPerformance} from "../Performance/ExperimentPerformance";
+import {Bootstrap} from "nlptoolkit-sampling/dist/Bootstrap";
+import {InstanceList} from "../InstanceList/InstanceList";
+import {Instance} from "../Instance/Instance";
+
+export class BootstrapRun implements MultipleRun{
+
+    private numberOfBootstraps: number
+
+    /**
+     * Constructor for BootstrapRun class. Basically sets the number of bootstrap runs.
+     *
+     * @param numberOfBootstraps Number of bootstrap runs.
+     */
+    constructor(numberOfBootstraps: number) {
+        this.numberOfBootstraps = numberOfBootstraps
+    }
+
+    /**
+     * Execute the bootstrap run with the given classifier on the given data set using the given parameters.
+     *
+     * @param experiment Experiment to be run.
+     * @return An ExperimentPerformance instance.
+     */
+    execute(experiment: Experiment): ExperimentPerformance {
+        let result = new ExperimentPerformance();
+        for (let i = 0; i < this.numberOfBootstraps; i++) {
+            let bootstrap = new Bootstrap<Instance>(experiment.getDataSet().getInstances(), i + experiment.getParameter().getSeed());
+            let bootstrapSample = new InstanceList(bootstrap.getSample());
+            experiment.getClassifier().train(bootstrapSample, experiment.getParameter());
+            result.add(experiment.getClassifier().test(experiment.getDataSet().getInstanceList()));
+        }
+        return result;
+    }
+}

package/source/Experiment/Experiment.ts

@@ -0,0 +1,57 @@
+import {Classifier} from "../Classifier/Classifier";
+import {Parameter} from "../Parameter/Parameter";
+import {DataSet} from "../DataSet/DataSet";
+import {FeatureSubSet} from "../FeatureSelection/FeatureSubSet";
+
+export class Experiment {
+
+    private classifier: Classifier
+    private parameter: Parameter
+    private dataSet: DataSet
+
+    /**
+     * Constructor for a specific machine learning experiment
+     * @param classifier Classifier used in the machine learning experiment
+     * @param parameter Parameter(s) of the classifier.
+     * @param dataSet DataSet on which the classifier is run.
+     */
+    constructor(classifier: Classifier, parameter: Parameter, dataSet: DataSet) {
+        this.classifier = classifier
+        this.parameter = parameter
+        this.dataSet = dataSet
+    }
+
+    /**
+     * Accessor for the classifier attribute.
+     * @return Classifier attribute.
+     */
+    getClassifier(): Classifier{
+        return this.classifier
+    }
+
+    /**
+     * Accessor for the parameter attribute.
+     * @return Parameter attribute.
+     */
+    getParameter(): Parameter{
+        return this.parameter
+    }
+
+    /**
+     * Accessor for the dataSet attribute.
+     * @return DataSet attribute.
+     */
+    getDataSet(): DataSet{
+        return this.dataSet
+    }
+
+    /**
+     * Construct and returns a feature selection experiment.
+     * @param featureSubSet Feature subset used in the feature selection experiment
+     * @return Experiment constructed
+     */
+    featureSelectedExperiment(featureSubSet: FeatureSubSet): Experiment{
+        return new Experiment(this.classifier, this.parameter, this.dataSet.getSubSetOfFeatures(featureSubSet));
+    }
+
+}

package/source/Experiment/KFoldRun.ts

@@ -0,0 +1,44 @@
+import {MultipleRun} from "./MultipleRun";
+import {Experiment} from "./Experiment";
+import {ExperimentPerformance} from "../Performance/ExperimentPerformance";
+import {Classifier} from "../Classifier/Classifier";
+import {Parameter} from "../Parameter/Parameter";
+import {CrossValidation} from "nlptoolkit-sampling/dist/CrossValidation";
+import {Instance} from "../Instance/Instance";
+import {InstanceList} from "../InstanceList/InstanceList";
+import {KFoldCrossValidation} from "nlptoolkit-sampling/dist/KFoldCrossValidation";
+
+export class KFoldRun implements MultipleRun{
+
+    protected K: number
+
+    constructor(K: number) {
+        this.K = K
+    }
+
+    protected runExperiment(classifier: Classifier,
+                            parameter: Parameter,
+                            experimentPerformance: ExperimentPerformance,
+                            crossValidation: CrossValidation<Instance>){
+        for (let i = 0; i < this.K; i++) {
+            let trainSet = new InstanceList(crossValidation.getTrainFold(i));
+            let testSet = new InstanceList(crossValidation.getTestFold(i));
+            classifier.train(trainSet, parameter);
+            experimentPerformance.add(classifier.test(testSet));
+        }
+    }
+
+    /**
+     * Execute K-fold cross-validation with the given classifier on the given data set using the given parameters.
+     *
+     * @param experiment Experiment to be run.
+     * @return An ExperimentPerformance instance.
+     */
+    execute(experiment: Experiment): ExperimentPerformance {
+        let result = new ExperimentPerformance();
+        let crossValidation = new KFoldCrossValidation<Instance>(experiment.getDataSet().getInstances(), this.K, experiment.getParameter().getSeed());
+        this.runExperiment(experiment.getClassifier(), experiment.getParameter(), result, crossValidation);
+        return result;
+    }
+
+}

package/source/Experiment/KFoldRunSeparateTest.ts

@@ -0,0 +1,50 @@
+import {KFoldRun} from "./KFoldRun";
+import {Classifier} from "../Classifier/Classifier";
+import {Parameter} from "../Parameter/Parameter";
+import {ExperimentPerformance} from "../Performance/ExperimentPerformance";
+import {InstanceList} from "../InstanceList/InstanceList";
+import {CrossValidation} from "nlptoolkit-sampling/dist/CrossValidation";
+import {Instance} from "../Instance/Instance";
+import {Experiment} from "./Experiment";
+import {KFoldCrossValidation} from "nlptoolkit-sampling/dist/KFoldCrossValidation";
+import {Partition} from "../InstanceList/Partition";
+
+export class KFoldRunSeparateTest extends KFoldRun{
+
+    /**
+     * Constructor for KFoldRunSeparateTest class. Basically sets K parameter of the K-fold cross-validation.
+     *
+     * @param K K of the K-fold cross-validation.
+     */
+    constructor(K: number) {
+        super(K);
+    }
+
+    protected runExperiment(classifier: Classifier,
+                            parameter: Parameter,
+                            experimentPerformance: ExperimentPerformance,
+                            crossValidation: CrossValidation<Instance>,
+                            testSet?: InstanceList) {
+        for (let i = 0; i < this.K; i++) {
+            let trainSet = new InstanceList(crossValidation.getTrainFold(i));
+            classifier.train(trainSet, parameter);
+            experimentPerformance.add(classifier.test(testSet));
+        }
+    }
+
+    /**
+     * Execute K-fold cross-validation with separate test set with the given classifier on the given data set using the given parameters.
+     *
+     * @param experiment Experiment to be run.
+     * @return An ExperimentPerformance instance.
+     */
+    execute(experiment: Experiment): ExperimentPerformance {
+        let result = new ExperimentPerformance();
+        let instanceList = experiment.getDataSet().getInstanceList();
+        let partition = new Partition(instanceList, 0.25, true);
+        let crossValidation = new KFoldCrossValidation<Instance>(partition.get(1).getInstances(), this.K, experiment.getParameter().getSeed());
+        this.runExperiment(experiment.getClassifier(), experiment.getParameter(), result, crossValidation, partition.get(0));
+        return result;
+    }
+
+}