nlptoolkit-classification 1.0.0

package/source/Model/DecisionTree/DecisionCondition.ts

@@ -0,0 +1,74 @@
+import {Attribute} from "../../Attribute/Attribute";
+import {Instance} from "../../Instance/Instance";
+import {DiscreteIndexedAttribute} from "../../Attribute/DiscreteIndexedAttribute";
+import {DiscreteAttribute} from "../../Attribute/DiscreteAttribute";
+import {ContinuousAttribute} from "../../Attribute/ContinuousAttribute";
+
+export class DecisionCondition {
+
+    private attributeIndex: number = -1
+    private comparison: string = undefined
+    private value: Attribute
+
+    /**
+     * A constructor that sets attributeIndex, comparison and {@link Attribute} value.
+     *
+     * @param attributeIndex Integer number that shows attribute index.
+     * @param value          The value of the {@link Attribute}.
+     * @param comparison     Comparison character.
+     */
+    constructor(attributeIndex: number, value: Attribute, comparison?: string) {
+        this.attributeIndex = attributeIndex;
+        this.comparison = comparison;
+        this.value = value;
+    }
+
+    getAttributeIndex(): number{
+        return this.attributeIndex
+    }
+
+    getValue(): Attribute{
+        return this.value
+    }
+
+    /**
+     * The satisfy method takes an {@link Instance} as an input.
+     * <p>
+     * If defined {@link Attribute} value is a {@link DiscreteIndexedAttribute} it compares the index of {@link Attribute} of instance at the
+     * attributeIndex and the index of {@link Attribute} value and returns the result.
+     * <p>
+     * If defined {@link Attribute} value is a {@link DiscreteAttribute} it compares the value of {@link Attribute} of instance at the
+     * attributeIndex and the value of {@link Attribute} value and returns the result.
+     * <p>
+     * If defined {@link Attribute} value is a {@link ContinuousAttribute} it compares the value of {@link Attribute} of instance at the
+     * attributeIndex and the value of {@link Attribute} value and returns the result according to the comparison character whether it is
+     * less than or greater than signs.
+     *
+     * @param instance Instance to compare.
+     * @return True if gicen instance satisfies the conditions.
+     */
+    satisfy(instance: Instance): boolean{
+        if (this.value instanceof DiscreteIndexedAttribute) {
+            if ((<DiscreteIndexedAttribute> this.value).getIndex() != -1) {
+                return (<DiscreteIndexedAttribute> instance.getAttribute(this.attributeIndex)).getIndex() == (<DiscreteIndexedAttribute> this.value).getIndex();
+            } else {
+                return true;
+            }
+        } else {
+            if (this.value instanceof DiscreteAttribute) {
+                return ((<DiscreteAttribute> instance.getAttribute(this.attributeIndex)).getValue() == (<DiscreteAttribute> this.value).getValue());
+            } else {
+                if (this.value instanceof ContinuousAttribute) {
+                    if (this.comparison == "<") {
+                        return (<ContinuousAttribute> instance.getAttribute(this.attributeIndex)).getValue() <= (<ContinuousAttribute> this.value).getValue();
+                    } else {
+                        if (this.comparison == ">") {
+                            return (<ContinuousAttribute> instance.getAttribute(this.attributeIndex)).getValue() > (<ContinuousAttribute> this.value).getValue();
+                        }
+                    }
+                }
+            }
+        }
+        return false;
+    }
+}

package/source/Model/DecisionTree/DecisionNode.ts

@@ -0,0 +1,257 @@
+import {InstanceList} from "../../InstanceList/InstanceList";
+import {DecisionCondition} from "./DecisionCondition";
+import {RandomForestParameter} from "../../Parameter/RandomForestParameter";
+import {Model} from "../Model";
+import {DiscreteIndexedAttribute} from "../../Attribute/DiscreteIndexedAttribute";
+import {DiscreteAttribute} from "../../Attribute/DiscreteAttribute";
+import {ContinuousAttribute} from "../../Attribute/ContinuousAttribute";
+import {DiscreteDistribution} from "nlptoolkit-math/dist/DiscreteDistribution";
+import {Partition} from "../../InstanceList/Partition";
+import {Instance} from "../../Instance/Instance";
+import {CompositeInstance} from "../../Instance/CompositeInstance";
+
+export class DecisionNode {
+
+    children: Array<DecisionNode> = undefined
+    private EPSILON: number = 0.0000000001;
+    private readonly data : InstanceList = undefined
+    private readonly classLabel : string = undefined
+    leaf: boolean = false
+    private condition: DecisionCondition = undefined
+
+    /**
+     * The DecisionNode method takes {@link InstanceList} data as input, and then it sets the class label parameter by finding
+     * the most occurred class label of given data, it then gets distinct class labels as class labels ArrayList. Later, it adds ordered
+     * indices to the indexList and shuffles them randomly. Then, it gets the class distribution of given data and finds the best entropy value
+     * of these class distribution.
+     * <p>
+     * If an attribute of given data is {@link DiscreteIndexedAttribute}, it creates a Distribution according to discrete indexed attribute class distribution
+     * and finds the entropy. If it is better than the last best entropy it reassigns the best entropy, best attribute and best split value according to
+     * the newly founded best entropy's index. At the end, it also adds new distribution to the class distribution .
+     * <p>
+     * If an attribute of given data is {@link DiscreteAttribute}, it directly finds the entropy. If it is better than the last best entropy it
+     * reassigns the best entropy, best attribute and best split value according to the newly founded best entropy's index.
+     * <p>
+     * If an attribute of given data is {@link ContinuousAttribute}, it creates two distributions; left and right according to class distribution
+     * and discrete distribution respectively, and finds the entropy. If it is better than the last best entropy it reassigns the best entropy,
+     * best attribute and best split value according to the newly founded best entropy's index. At the end, it also adds new distribution to
+     * the right distribution and removes from left distribution .
+     *
+     * @param data      {@link InstanceList} input.
+     * @param condition {@link DecisionCondition} to check.
+     * @param parameter RandomForestParameter like seed, ensembleSize, attributeSubsetSize.
+     * @param isStump   Refers to decision trees with only 1 splitting rule.
+     */
+    constructor(data: InstanceList, condition: DecisionCondition, parameter: RandomForestParameter, isStump: boolean) {
+        let bestAttribute = -1
+        let bestSplitValue = 0
+        this.condition = condition;
+        this.data = data;
+        this.classLabel = Model.getMaximum(data.getClassLabels());
+        this.leaf = true;
+        let classLabels = data.getDistinctClassLabels();
+        if (classLabels.length == 1) {
+            return;
+        }
+        if (isStump && condition != null) {
+            return;
+        }
+        let indexList = new Array<number>();
+        for (let i = 0; i < data.get(0).attributeSize(); i++) {
+            indexList.push(i);
+        }
+        let size
+        if (parameter != undefined && parameter.getAttributeSubsetSize() < data.get(0).attributeSize()) {
+            size = parameter.getAttributeSubsetSize();
+        } else {
+            size = data.get(0).attributeSize();
+        }
+        let classDistribution = data.classDistribution();
+        let bestEntropy = data.classDistribution().entropy();
+        for (let j = 0; j < size; j++) {
+            let index = indexList[j];
+            if (data.get(0).getAttribute(index) instanceof DiscreteIndexedAttribute) {
+                for (let k = 0; k < (<DiscreteIndexedAttribute>data.get(0).getAttribute(index)).getMaxIndex(); k++) {
+                    let distribution = data.discreteIndexedAttributeClassDistribution(index, k);
+                    if (distribution.getSum() > 0) {
+                        classDistribution.removeDistribution(distribution);
+                        let entropy = (classDistribution.entropy() * classDistribution.getSum() + distribution.entropy() * distribution.getSum()) / data.size();
+                        if (entropy + this.EPSILON < bestEntropy) {
+                            bestEntropy = entropy;
+                            bestAttribute = index;
+                            bestSplitValue = k;
+                        }
+                        classDistribution.addDistribution(distribution);
+                    }
+                }
+            } else {
+                if (data.get(0).getAttribute(index) instanceof DiscreteAttribute) {
+                    let entropy = this.entropyForDiscreteAttribute(index);
+                    if (entropy + this.EPSILON < bestEntropy) {
+                        bestEntropy = entropy;
+                        bestAttribute = index;
+                    }
+                } else {
+                    if (data.get(0).getAttribute(index) instanceof ContinuousAttribute) {
+                        data.sort(index);
+                        let previousValue = Number.NEGATIVE_INFINITY;
+                        let leftDistribution = data.classDistribution();
+                        let rightDistribution = new DiscreteDistribution();
+                        for (let k = 0; k < data.size(); k++) {
+                            let instance = data.get(k);
+                            if (k == 0) {
+                                previousValue = (<ContinuousAttribute>instance.getAttribute(index)).getValue();
+                            } else {
+                                if ((<ContinuousAttribute>instance.getAttribute(index)).getValue() != previousValue) {
+                                    let splitValue = (previousValue + (<ContinuousAttribute>instance.getAttribute(index)).getValue()) / 2;
+                                    previousValue = (<ContinuousAttribute>instance.getAttribute(index)).getValue();
+                                    let entropy = (leftDistribution.getSum() / data.size()) * leftDistribution.entropy() + (rightDistribution.getSum() / data.size()) * rightDistribution.entropy();
+                                    if (entropy + this.EPSILON < bestEntropy) {
+                                        bestEntropy = entropy;
+                                        bestSplitValue = splitValue;
+                                        bestAttribute = index;
+                                    }
+                                }
+                            }
+                            leftDistribution.removeItem(instance.getClassLabel());
+                            rightDistribution.addItem(instance.getClassLabel());
+                        }
+                    }
+                }
+            }
+        }
+        if (bestAttribute != -1) {
+            this.leaf = false;
+            if (data.get(0).getAttribute(bestAttribute) instanceof DiscreteIndexedAttribute) {
+                this.createChildrenForDiscreteIndexed(bestAttribute, bestSplitValue, parameter, isStump);
+            } else {
+                if (data.get(0).getAttribute(bestAttribute) instanceof DiscreteAttribute) {
+                    this.createChildrenForDiscrete(bestAttribute, parameter, isStump);
+                } else {
+                    if (data.get(0).getAttribute(bestAttribute) instanceof ContinuousAttribute) {
+                        this.createChildrenForContinuous(bestAttribute, bestSplitValue, parameter, isStump);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * The entropyForDiscreteAttribute method takes an attributeIndex and creates an ArrayList of DiscreteDistribution.
+     * Then loops through the distributions and calculates the total entropy.
+     *
+     * @param attributeIndex Index of the attribute.
+     * @return Total entropy for the discrete attribute.
+     */
+    private entropyForDiscreteAttribute(attributeIndex: number): number{
+        let sum = 0.0;
+        let distributions = this.data.attributeClassDistribution(attributeIndex);
+        for (let distribution of distributions) {
+            sum += (distribution.getSum() / this.data.size()) * distribution.entropy();
+        }
+        return sum;
+    }
+
+    /**
+     * The createChildrenForDiscreteIndexed method creates an ArrayList of DecisionNodes as children and a partition with respect to
+     * indexed attribute.
+     *
+     * @param attributeIndex Index of the attribute.
+     * @param attributeValue Value of the attribute.
+     * @param parameter      RandomForestParameter like seed, ensembleSize, attributeSubsetSize.
+     * @param isStump        Refers to decision trees with only 1 splitting rule.
+     */
+    private createChildrenForDiscreteIndexed(attributeIndex: number, attributeValue: number, parameter: RandomForestParameter, isStump: boolean){
+        let childrenData = new Partition(this.data, attributeIndex, attributeValue);
+        this.children = new Array<DecisionNode>();
+        this.children.push(new DecisionNode(childrenData.get(0), new DecisionCondition(attributeIndex, new DiscreteIndexedAttribute("", attributeValue, (<DiscreteIndexedAttribute> this.data.get(0).getAttribute(attributeIndex)).getMaxIndex())), parameter, isStump));
+        this.children.push(new DecisionNode(childrenData.get(1), new DecisionCondition(attributeIndex, new DiscreteIndexedAttribute("", -1, (<DiscreteIndexedAttribute> this.data.get(0).getAttribute(attributeIndex)).getMaxIndex())), parameter, isStump));
+    }
+
+    /**
+     * The createChildrenForDiscrete method creates an ArrayList of values, a partition with respect to attributes and an ArrayList
+     * of DecisionNodes as children.
+     *
+     * @param attributeIndex Index of the attribute.
+     * @param parameter      RandomForestParameter like seed, ensembleSize, attributeSubsetSize.
+     * @param isStump        Refers to decision trees with only 1 splitting rule.
+     */
+    private createChildrenForDiscrete(attributeIndex: number, parameter: RandomForestParameter, isStump: boolean){
+        let valueList = this.data.getAttributeValueList(attributeIndex);
+        let childrenData = new Partition(this.data, attributeIndex);
+        this.children = new Array<DecisionNode>();
+        for (let i = 0; i < valueList.length; i++) {
+            this.children.push(new DecisionNode(childrenData.get(i), new DecisionCondition(attributeIndex, new DiscreteAttribute(valueList[i])), parameter, isStump));
+        }
+    }
+
+    /**
+     * The createChildrenForContinuous method creates an ArrayList of DecisionNodes as children and a partition with respect to
+     * continuous attribute and the given split value.
+     *
+     * @param attributeIndex Index of the attribute.
+     * @param parameter      RandomForestParameter like seed, ensembleSize, attributeSubsetSize.
+     * @param isStump        Refers to decision trees with only 1 splitting rule.
+     * @param splitValue     Split value is used for partitioning.
+     */
+    private createChildrenForContinuous(attributeIndex: number, splitValue: number, parameter: RandomForestParameter, isStump: boolean){
+        let childrenData = new Partition(this.data, attributeIndex, splitValue);
+        this.children = new Array<DecisionNode>();
+        this.children.push(new DecisionNode(childrenData.get(0), new DecisionCondition(attributeIndex, new ContinuousAttribute(splitValue), "<"), parameter, isStump));
+        this.children.push(new DecisionNode(childrenData.get(1), new DecisionCondition(attributeIndex, new ContinuousAttribute(splitValue), ">"), parameter, isStump));
+    }
+
+    /**
+     * The predict method takes an {@link Instance} as input and performs prediction on the DecisionNodes and returns the prediction
+     * for that instance.
+     *
+     * @param instance Instance to make prediction.
+     * @return The prediction for given instance.
+     */
+    predict(instance: Instance): string{
+        if (instance instanceof CompositeInstance) {
+            let possibleClassLabels = (<CompositeInstance> instance).getPossibleClassLabels();
+            let distribution = this.data.classDistribution();
+            let predictedClass = distribution.getMaxItem(possibleClassLabels);
+            if (this.leaf) {
+                return predictedClass;
+            } else {
+                for (let node of this.children) {
+                    if (node.condition.satisfy(instance)) {
+                        let childPrediction = node.predict(instance);
+                        if (childPrediction != undefined) {
+                            return childPrediction;
+                        } else {
+                            return predictedClass;
+                        }
+                    }
+                }
+                return predictedClass;
+            }
+        } else {
+            if (this.leaf) {
+                return this.classLabel;
+            } else {
+                for (let node of this.children) {
+                    if (node.condition.satisfy(instance)) {
+                        return node.predict(instance);
+                    }
+                }
+                return this.classLabel;
+            }
+        }
+    }
+
+    predictProbabilityDistribution(instance: Instance): Map<string, number>{
+        if (this.leaf) {
+            return this.data.classDistribution().getProbabilityDistribution();
+        } else {
+            for (let node of this.children) {
+                if (node.condition.satisfy(instance)) {
+                    return node.predictProbabilityDistribution(instance);
+                }
+            }
+            return this.data.classDistribution().getProbabilityDistribution();
+        }
+    }
+}

package/source/Model/DecisionTree/DecisionTree.ts

@@ -0,0 +1,71 @@
+import {ValidatedModel} from "../ValidatedModel";
+import {DecisionNode} from "./DecisionNode";
+import {Instance} from "../../Instance/Instance";
+import {CompositeInstance} from "../../Instance/CompositeInstance";
+import {InstanceList} from "../../InstanceList/InstanceList";
+
+export class DecisionTree extends ValidatedModel{
+
+    private root: DecisionNode
+
+    /**
+     * Constructor that sets root node of the decision tree.
+     *
+     * @param root DecisionNode type input.
+     */
+    constructor(root: DecisionNode) {
+        super();
+        this.root = root
+    }
+
+    /**
+     * The predict method  performs prediction on the root node of given instance, and if it is null, it returns the possible class labels.
+     * Otherwise it returns the returned class labels.
+     *
+     * @param instance Instance make prediction.
+     * @return Possible class labels.
+     */
+    predict(instance: Instance): string {
+        let predictedClass = this.root.predict(instance);
+        if ((predictedClass == null) && ((instance instanceof CompositeInstance))) {
+            predictedClass = (<CompositeInstance> instance).getPossibleClassLabels()[0];
+        }
+        return predictedClass;
+    }
+
+    predictProbability(instance: Instance): Map<string, number> {
+        return this.root.predictProbabilityDistribution(instance)
+    }
+
+    /**
+     * The prune method takes a {@link DecisionNode} and an {@link InstanceList} as inputs. It checks the classification performance
+     * of given InstanceList before pruning, i.e making a node leaf, and after pruning. If the after performance is better than the
+     * before performance it prune the given InstanceList from the tree.
+     *
+     * @param node     DecisionNode that will be pruned if conditions hold.
+     * @param pruneSet Small subset of tree that will be removed from tree.
+     */
+    pruneNode(node: DecisionNode, pruneSet: InstanceList){
+        if (node.leaf){
+            return;
+        }
+        let before = this.testClassifier(pruneSet);
+        node.leaf = true;
+        let after = this.testClassifier(pruneSet);
+        if (after.getAccuracy() < before.getAccuracy()) {
+            node.leaf = false;
+            for (let child of node.children) {
+                this.pruneNode(child, pruneSet);
+            }
+        }
+    }
+
+    /**
+     * The prune method takes an {@link InstanceList} and  performs pruning to the root node.
+     *
+     * @param pruneSet {@link InstanceList} to perform pruning.
+     */
+    prune(pruneSet: InstanceList){
+        this.pruneNode(this.root, pruneSet)
+    }
+}

package/source/Model/DeepNetworkModel.ts

@@ -0,0 +1,156 @@
+import {NeuralNetworkModel} from "./NeuralNetworkModel";
+import {Matrix} from "nlptoolkit-math/dist/Matrix";
+import {ActivationFunction} from "../Parameter/ActivationFunction";
+import {DeepNetworkParameter} from "../Parameter/DeepNetworkParameter";
+import {InstanceList} from "../InstanceList/InstanceList";
+import {Vector} from "nlptoolkit-math/dist/Vector";
+import {ClassificationPerformance} from "../Performance/ClassificationPerformance";
+
+export class DeepNetworkModel extends NeuralNetworkModel{
+
+    private weights: Array<Matrix>
+    private hiddenLayerSize: number
+    private readonly activationFunction: ActivationFunction
+
+    /**
+     * The allocateWeights method takes {@link DeepNetworkParameter}s as an input. First it adds random weights to the {@link Array}
+     * of {@link Matrix} weights' first layer. Then it loops through the layers and adds random weights till the last layer.
+     * At the end it adds random weights to the last layer and also sets the hiddenLayerSize value.
+     *
+     * @param parameters {@link DeepNetworkParameter} input.
+     */
+    private allocateWeights(parameters: DeepNetworkParameter){
+        this.weights = new Array<Matrix>();
+        this.weights.push(this.allocateLayerWeights(parameters.getHiddenNodes(0), this.d + 1));
+        for (let i = 0; i < parameters.layerSize() - 1; i++) {
+            this.weights.push(this.allocateLayerWeights(parameters.getHiddenNodes(i + 1), parameters.getHiddenNodes(i) + 1));
+        }
+        this.weights.push(this.allocateLayerWeights(this.K, parameters.getHiddenNodes(parameters.layerSize() - 1) + 1));
+        this.hiddenLayerSize = parameters.layerSize();
+    }
+
+    /**
+     * The setBestWeights method creates an {@link Array} of Matrix as bestWeights and clones the values of weights {@link Array}
+     * into this newly created {@link Array}.
+     *
+     * @return An {@link Array} clones from the weights ArrayList.
+     */
+    private setBestWeights(): Array<Matrix>{
+        let bestWeights = new Array<Matrix>();
+        for (let m of this.weights) {
+            bestWeights.push(m.clone());
+        }
+        return bestWeights;
+    }
+
+    /**
+     * Constructor that takes two {@link InstanceList} train set and validation set and {@link DeepNetworkParameter} as inputs.
+     * First it sets the class labels, their sizes as K and the size of the continuous attributes as d of given train set and
+     * allocates weights and sets the best weights. At each epoch, it shuffles the train set and loops through the each item of that train set,
+     * it multiplies the weights Matrix with input Vector than applies the sigmoid function and stores the result as hidden and add bias.
+     * Then updates weights and at the end it compares the performance of these weights with validation set. It updates the bestClassificationPerformance and
+     * bestWeights according to the current situation. At the end it updates the learning rate via etaDecrease value and finishes
+     * with clearing the weights.
+     *
+     * @param trainSet      {@link InstanceList} to be used as trainSet.
+     * @param validationSet {@link InstanceList} to be used as validationSet.
+     * @param parameters    {@link DeepNetworkParameter} input.
+     */
+    constructor(trainSet: InstanceList, validationSet: InstanceList, parameters: DeepNetworkParameter) {
+        super(trainSet);
+        let tmpHidden = new Vector(0, 0);
+        let deltaWeights = new Array<Matrix>();
+        let hidden = new Array<Vector>();
+        let hiddenBiased = new Array<Vector>();
+        this.activationFunction = parameters.getActivationFunction();
+        this.allocateWeights(parameters);
+        let bestWeights = this.setBestWeights();
+        let bestClassificationPerformance = new ClassificationPerformance(0.0);
+        let epoch = parameters.getEpoch();
+        let learningRate = parameters.getLearningRate();
+        for (let i = 0; i < epoch; i++) {
+            trainSet.shuffle(parameters.getSeed());
+            for (let j = 0; j < trainSet.size(); j++) {
+                this.createInputVector(trainSet.get(j));
+                hidden = new Array<Vector>();
+                hiddenBiased = new Array<Vector>();
+                deltaWeights = new Array<Matrix>();
+                for (let k = 0; k < this.hiddenLayerSize; k++) {
+                    if (k == 0) {
+                        hidden.push(this.calculateHidden(this.x, this.weights[k], this.activationFunction));
+                    } else {
+                        hidden.push(this.calculateHidden(hiddenBiased[k - 1], this.weights[k], this.activationFunction));
+                    }
+                    hiddenBiased.push(hidden[k].biased());
+                }
+                let rMinusY = this.calculateRMinusY(trainSet.get(j), hiddenBiased[this.hiddenLayerSize - 1], this.weights[this.weights.length - 1]);
+                deltaWeights.splice(0, 0, new Matrix(rMinusY, hiddenBiased[this.hiddenLayerSize - 1]));
+                for (let k = this.weights.length - 2; k >= 0; k--) {
+                    let tmph
+                    if (k == this.weights.length - 2){
+                        tmph = this.weights[k + 1].multiplyWithVectorFromLeft(rMinusY);
+                    } else {
+                        tmph = this.weights[k + 1].multiplyWithVectorFromLeft(tmpHidden);
+                    }
+                    tmph.remove(0);
+                    let activationDerivative
+                    switch (this.activationFunction){
+                        case ActivationFunction.SIGMOID:
+                        default:
+                            let oneMinusHidden = this.calculateOneMinusHidden(hidden[k]);
+                            activationDerivative = oneMinusHidden.elementProduct(hidden[k]);
+                            break;
+                        case ActivationFunction.TANH:
+                            let one = new Vector(hidden.length, 1.0);
+                            hidden[k].tanh();
+                            activationDerivative = one.difference(hidden[k].elementProduct(hidden[k]));
+                            break;
+                        case ActivationFunction.RELU:
+                            hidden[k].reluDerivative();
+                            activationDerivative = hidden[k];
+                            break;
+                    }
+                    tmpHidden = tmph.elementProduct(activationDerivative);
+                    if (k == 0) {
+                        deltaWeights.splice(0, 0, new Matrix(tmpHidden, this.x));
+                    } else {
+                        deltaWeights.splice(0, 0, new Matrix(tmpHidden, hiddenBiased[k - 1]));
+                    }
+                }
+                for (let k = 0; k < this.weights.length; k++) {
+                    deltaWeights[k].multiplyWithConstant(learningRate);
+                    this.weights[k].add(deltaWeights[k]);
+                }
+            }
+            let currentClassificationPerformance = this.testClassifier(validationSet);
+            if (currentClassificationPerformance.getAccuracy() > bestClassificationPerformance.getAccuracy()) {
+                bestClassificationPerformance = currentClassificationPerformance;
+                bestWeights = this.setBestWeights();
+            }
+            learningRate *= parameters.getEtaDecrease();
+        }
+        this.weights = new Array<Matrix>();
+        for (let m of bestWeights) {
+            this.weights.push(m);
+        }
+    }
+
+    /**
+     * The calculateOutput method loops size of the weights times and calculate one hidden layer at a time and adds bias term.
+     * At the end it updates the output y value.
+     */
+    protected calculateOutput(): void {
+        let hiddenBiased
+        for (let i = 0; i < this.weights.length - 1; i++) {
+            let hidden
+            if (i == 0) {
+                hidden = this.calculateHidden(this.x, this.weights[i], this.activationFunction);
+            } else {
+                hidden = this.calculateHidden(hiddenBiased, this.weights[i], this.activationFunction);
+            }
+            hiddenBiased = hidden.biased();
+        }
+        this.y = this.weights[this.weights.length - 1].multiplyWithVectorFromRight(hiddenBiased);
+    }
+
+}

package/source/Model/DummyModel.ts

@@ -0,0 +1,40 @@
+import {Model} from "./Model";
+import {Instance} from "../Instance/Instance";
+import {DiscreteDistribution} from "nlptoolkit-math/dist/DiscreteDistribution";
+import {InstanceList} from "../InstanceList/InstanceList";
+import {CompositeInstance} from "../Instance/CompositeInstance";
+
+export class DummyModel extends Model{
+
+    private distribution: DiscreteDistribution
+
+    /**
+     * Constructor which sets the distribution using the given {@link InstanceList}.
+     *
+     * @param trainSet {@link InstanceList} which is used to get the class distribution.
+     */
+    constructor(trainSet: InstanceList) {
+        super();
+        this.distribution = trainSet.classDistribution();
+    }
+
+    /**
+     * The predict method takes an Instance as an input and returns the entry of distribution which has the maximum value.
+     *
+     * @param instance Instance to make prediction.
+     * @return The entry of distribution which has the maximum value.
+     */
+    predict(instance: Instance): string {
+        if ((instance instanceof CompositeInstance)) {
+            let possibleClassLabels = (<CompositeInstance> instance).getPossibleClassLabels();
+            return this.distribution.getMaxItem(possibleClassLabels);
+        } else {
+            return this.distribution.getMaxItem();
+        }
+    }
+
+    predictProbability(instance: Instance): Map<string, number> {
+        return this.distribution.getProbabilityDistribution();
+    }
+
+}

package/source/Model/GaussianModel.ts

@@ -0,0 +1,58 @@
+import {DiscreteDistribution} from "nlptoolkit-math/dist/DiscreteDistribution";
+import {Instance} from "../Instance/Instance";
+import {ValidatedModel} from "./ValidatedModel";
+import {CompositeInstance} from "../Instance/CompositeInstance";
+
+export abstract class GaussianModel extends ValidatedModel{
+
+    protected priorDistribution: DiscreteDistribution
+
+    /**
+     * Abstract method calculateMetric takes an {@link Instance} and a String as inputs.
+     *
+     * @param instance {@link Instance} input.
+     * @param Ci       String input.
+     * @return A double value as metric.
+     */
+    abstract calculateMetric(instance: Instance, Ci: string): number
+
+    /**
+     * The predict method takes an Instance as an input. First it gets the size of prior distribution and loops this size times.
+     * Then it gets the possible class labels and and calculates metric value. At the end, it returns the class which has the
+     * maximum value of metric.
+     *
+     * @param instance {@link Instance} to predict.
+     * @return The class which has the maximum value of metric.
+     */
+    predict(instance: Instance): string {
+        let maxMetric = Number.NEGATIVE_INFINITY;
+        let predictedClass, size
+        if (instance instanceof CompositeInstance) {
+            predictedClass = (<CompositeInstance> instance).getPossibleClassLabels()[0];
+            size = (<CompositeInstance> instance).getPossibleClassLabels().length;
+        } else {
+            predictedClass = this.priorDistribution.getMaxItem();
+            size = this.priorDistribution.size;
+        }
+        for (let i = 0; i < size; i++) {
+            let Ci
+            if (instance instanceof CompositeInstance) {
+                Ci = (<CompositeInstance> instance).getPossibleClassLabels()[i];
+            } else {
+                Ci = this.priorDistribution.getItem(i);
+            }
+            if (this.priorDistribution.containsItem(Ci)) {
+                let metric = this.calculateMetric(instance, Ci);
+                if (metric > maxMetric) {
+                    maxMetric = metric;
+                    predictedClass = Ci;
+                }
+            }
+        }
+        return predictedClass;
+    }
+
+    predictProbability(instance: Instance): Map<string, number> {
+        return undefined;
+    }
+}