moo_fann 0.1.0

data/ext/moo_fann/fann_activation.h

@@ -0,0 +1,144 @@
+/*
+Fast Artificial Neural Network Library (fann)
+Copyright (C) 2003-2012 Steffen Nissen (sn@leenissen.dk)
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifndef __fann_activation_h__
+#define __fann_activation_h__
+/* internal include file, not to be included directly
+ */
+
+/* Implementation of the activation functions
+ */
+
+/* stepwise linear functions used for some of the activation functions */
+
+/* defines used for the stepwise linear functions */
+
+#define fann_linear_func(v1, r1, v2, r2, sum) (((((r2)-(r1)) * ((sum)-(v1)))/((v2)-(v1))) + (r1))
+#define fann_stepwise(v1, v2, v3, v4, v5, v6, r1, r2, r3, r4, r5, r6, min, max, sum) (sum < v5 ? (sum < v3 ? (sum < v2 ? (sum < v1 ? min : fann_linear_func(v1, r1, v2, r2, sum)) : fann_linear_func(v2, r2, v3, r3, sum)) : (sum < v4 ? fann_linear_func(v3, r3, v4, r4, sum) : fann_linear_func(v4, r4, v5, r5, sum))) : (sum < v6 ? fann_linear_func(v5, r5, v6, r6, sum) : max))
+
+/* FANN_LINEAR */
+/* #define fann_linear(steepness, sum) fann_mult(steepness, sum) */
+#define fann_linear_derive(steepness, value) (steepness)
+
+/* FANN_SIGMOID */
+/* #define fann_sigmoid(steepness, sum) (1.0f/(1.0f + exp(-2.0f * steepness * sum))) */
+#define fann_sigmoid_real(sum) (1.0f/(1.0f + exp(-2.0f * sum)))
+#define fann_sigmoid_derive(steepness, value) (2.0f * steepness * value * (1.0f - value))
+
+/* FANN_SIGMOID_SYMMETRIC */
+/* #define fann_sigmoid_symmetric(steepness, sum) (2.0f/(1.0f + exp(-2.0f * steepness * sum)) - 1.0f) */
+#define fann_sigmoid_symmetric_real(sum) (2.0f/(1.0f + exp(-2.0f * sum)) - 1.0f)
+#define fann_sigmoid_symmetric_derive(steepness, value) steepness * (1.0f - (value*value))
+
+/* FANN_GAUSSIAN */
+/* #define fann_gaussian(steepness, sum) (exp(-sum * steepness * sum * steepness)) */
+#define fann_gaussian_real(sum) (exp(-sum * sum))
+#define fann_gaussian_derive(steepness, value, sum) (-2.0f * sum * value * steepness * steepness)
+
+/* FANN_GAUSSIAN_SYMMETRIC */
+/* #define fann_gaussian_symmetric(steepness, sum) ((exp(-sum * steepness * sum * steepness)*2.0)-1.0) */
+#define fann_gaussian_symmetric_real(sum) ((exp(-sum * sum)*2.0f)-1.0f)
+#define fann_gaussian_symmetric_derive(steepness, value, sum) (-2.0f * sum * (value+1.0f) * steepness * steepness)
+
+/* FANN_ELLIOT */
+/* #define fann_elliot(steepness, sum) (((sum * steepness) / 2.0f) / (1.0f + fann_abs(sum * steepness)) + 0.5f) */
+#define fann_elliot_real(sum) (((sum) / 2.0f) / (1.0f + fann_abs(sum)) + 0.5f)
+#define fann_elliot_derive(steepness, value, sum) (steepness * 1.0f / (2.0f * (1.0f + fann_abs(sum)) * (1.0f + fann_abs(sum))))
+
+/* FANN_ELLIOT_SYMMETRIC */
+/* #define fann_elliot_symmetric(steepness, sum) ((sum * steepness) / (1.0f + fann_abs(sum * steepness)))*/
+#define fann_elliot_symmetric_real(sum) ((sum) / (1.0f + fann_abs(sum)))
+#define fann_elliot_symmetric_derive(steepness, value, sum) (steepness * 1.0f / ((1.0f + fann_abs(sum)) * (1.0f + fann_abs(sum))))
+
+/* FANN_SIN_SYMMETRIC */
+#define fann_sin_symmetric_real(sum) (sin(sum))
+#define fann_sin_symmetric_derive(steepness, sum) (steepness*cos(steepness*sum))
+
+/* FANN_COS_SYMMETRIC */
+#define fann_cos_symmetric_real(sum) (cos(sum))
+#define fann_cos_symmetric_derive(steepness, sum) (steepness*-sin(steepness*sum))
+
+/* FANN_SIN */
+#define fann_sin_real(sum) (sin(sum)/2.0f+0.5f)
+#define fann_sin_derive(steepness, sum) (steepness*cos(steepness*sum)/2.0f)
+
+/* FANN_COS */
+#define fann_cos_real(sum) (cos(sum)/2.0f+0.5f)
+#define fann_cos_derive(steepness, sum) (steepness*-sin(steepness*sum)/2.0f)
+
+#define fann_activation_switch(activation_function, value, result) \
+switch(activation_function) \
+{ \
+	case FANN_LINEAR: \
+		result = (fann_type)value; \
+        break; \
+	case FANN_LINEAR_PIECE: \
+		result = (fann_type)((value < 0) ? 0 : (value > 1) ? 1 : value); \
+        break; \
+	case FANN_LINEAR_PIECE_SYMMETRIC: \
+		result = (fann_type)((value < -1) ? -1 : (value > 1) ? 1 : value); \
+        break; \
+	case FANN_SIGMOID: \
+		result = (fann_type)fann_sigmoid_real(value); \
+        break; \
+	case FANN_SIGMOID_SYMMETRIC: \
+		result = (fann_type)fann_sigmoid_symmetric_real(value); \
+        break; \
+	case FANN_SIGMOID_SYMMETRIC_STEPWISE: \
+		result = (fann_type)fann_stepwise(-2.64665293693542480469e+00, -1.47221934795379638672e+00, -5.49306154251098632812e-01, 5.49306154251098632812e-01, 1.47221934795379638672e+00, 2.64665293693542480469e+00, -9.90000009536743164062e-01, -8.99999976158142089844e-01, -5.00000000000000000000e-01, 5.00000000000000000000e-01, 8.99999976158142089844e-01, 9.90000009536743164062e-01, -1, 1, value); \
+        break; \
+	case FANN_SIGMOID_STEPWISE: \
+		result = (fann_type)fann_stepwise(-2.64665246009826660156e+00, -1.47221946716308593750e+00, -5.49306154251098632812e-01, 5.49306154251098632812e-01, 1.47221934795379638672e+00, 2.64665293693542480469e+00, 4.99999988824129104614e-03, 5.00000007450580596924e-02, 2.50000000000000000000e-01, 7.50000000000000000000e-01, 9.49999988079071044922e-01, 9.95000004768371582031e-01, 0, 1, value); \
+        break; \
+	case FANN_THRESHOLD: \
+		result = (fann_type)((value < 0) ? 0 : 1); \
+        break; \
+	case FANN_THRESHOLD_SYMMETRIC: \
+		result = (fann_type)((value < 0) ? -1 : 1); \
+        break; \
+	case FANN_GAUSSIAN: \
+		result = (fann_type)fann_gaussian_real(value); \
+        break; \
+	case FANN_GAUSSIAN_SYMMETRIC: \
+		result = (fann_type)fann_gaussian_symmetric_real(value); \
+        break; \
+	case FANN_ELLIOT: \
+		result = (fann_type)fann_elliot_real(value); \
+	    break; \
+	case FANN_ELLIOT_SYMMETRIC: \
+		result = (fann_type)fann_elliot_symmetric_real(value); \
+        break; \
+	case FANN_SIN_SYMMETRIC: \
+		result = (fann_type)fann_sin_symmetric_real(value); \
+        break; \
+	case FANN_COS_SYMMETRIC: \
+		result = (fann_type)fann_cos_symmetric_real(value); \
+        break; \
+	case FANN_SIN: \
+		result = (fann_type)fann_sin_real(value); \
+        break; \
+	case FANN_COS: \
+		result = (fann_type)fann_cos_real(value); \
+        break; \
+	case FANN_GAUSSIAN_STEPWISE: \
+        result = 0; \
+        break; \
+}
+
+#endif

data/ext/moo_fann/fann_augment.h

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2018 Maxine Michalski <maxine@furfind.net>
+ * Copyright 2013 ruby-fann contributors
+ * <https://github.com/tangledpath/ruby-fann#contributors>
+ *
+ * This file is part of moo_fann.
+ *
+ * moo_fann is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * moo_fann is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with moo_fann.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ruby.h"
+#include "ruby_compat.h"
+
+FANN_EXTERNAL struct fann_train_data * FANN_API fann_create_train_from_rb_ary2(
+	unsigned int num_data,
+	unsigned int num_input,
+  unsigned int num_output)
+{
+	return 0;
+}
+
+/*
+ * Copied from fann_create_train_from_callback/file & modified to ease 
+ * allocating from ruby arrays:
+ */
+FANN_EXTERNAL struct fann_train_data * FANN_API fann_create_train_from_rb_ary(
+	VALUE inputs, 
+	VALUE outputs 
+)
+{
+    unsigned int i, j;
+    fann_type *data_input, *data_output;
+    struct fann_train_data *data = (struct fann_train_data *)malloc(sizeof(struct fann_train_data));
+    unsigned int num_input = RARRAY_LEN(RARRAY_PTR(inputs)[0]);
+    unsigned int num_output =RARRAY_LEN(RARRAY_PTR(outputs)[0]);
+		unsigned int num_data = RARRAY_LEN(inputs);
+		
+    if(data == NULL) {
+        fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM);
+        return NULL;
+    }
+
+    fann_init_error_data((struct fann_error *) data);
+
+    data->num_data     = num_data;
+    data->num_input    = num_input;
+    data->num_output = num_output;
+
+    data->input = (fann_type **) calloc(num_data, sizeof(fann_type *));
+    if(data->input == NULL)
+    {
+        fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM);
+        fann_destroy_train(data);
+        return NULL;
+    }
+
+    data->output = (fann_type **) calloc(num_data, sizeof(fann_type *));
+    if(data->output == NULL)
+    {
+        fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM);
+        fann_destroy_train(data);
+        return NULL;
+    }
+
+    data_input = (fann_type *) calloc(num_input * num_data, sizeof(fann_type));
+    if(data_input == NULL)
+    {
+        fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM);
+        fann_destroy_train(data);
+        return NULL;
+    }
+
+    data_output = (fann_type *) calloc(num_output * num_data, sizeof(fann_type));
+    if(data_output == NULL)
+    {
+        fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM);
+        fann_destroy_train(data);
+        return NULL;
+    }
+
+		VALUE inputs_i, outputs_i;
+		for(i = 0; i != num_data; i++)
+		{
+			data->input[i] = data_input;
+			data_input += num_input;
+
+			inputs_i = RARRAY_PTR(inputs)[i];
+			outputs_i = RARRAY_PTR(outputs)[i];
+			
+			if(RARRAY_LEN(inputs_i) != num_input) 
+			{
+				rb_raise (
+					rb_eRuntimeError, 
+					"Number of inputs at [%d] is inconsistent: (%d != %d)", 
+					i, RARRAY_LEN(inputs_i), num_input);
+			}
+			
+			if(RARRAY_LEN(outputs_i) != num_output) 
+			{
+				rb_raise (
+					rb_eRuntimeError, 
+					"Number of outputs at [%d] is inconsistent: (%d != %d)", 
+					i, RARRAY_LEN(outputs_i), num_output);
+			}
+			
+			
+			for(j = 0; j != num_input; j++)
+			{
+				data->input[i][j]=NUM2DBL(RARRAY_PTR(inputs_i)[j]);
+			}
+
+			data->output[i] = data_output;
+			data_output += num_output;
+			
+			for(j = 0; j != num_output; j++)
+			{
+				data->output[i][j]=NUM2DBL(RARRAY_PTR(outputs_i)[j]);
+			}
+		}
+		
+    return data;
+}

data/ext/moo_fann/fann_cascade.c

@@ -0,0 +1,1048 @@
+/*
+  Fast Artificial Neural Network Library (fann)
+  Copyright (C) 2003-2012 Steffen Nissen (sn@leenissen.dk)
+  
+  This library is free software; you can redistribute it and/or
+  modify it under the terms of the GNU Lesser General Public
+  License as published by the Free Software Foundation; either
+  version 2.1 of the License, or (at your option) any later version.
+  
+  This library is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  Lesser General Public License for more details.
+  
+  You should have received a copy of the GNU Lesser General Public
+  License along with this library; if not, write to the Free Software
+  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#include "config.h"
+#include "fann.h"
+#include "string.h"
+
+#ifndef FIXEDFANN
+
+/* #define CASCADE_DEBUG */
+/* #define CASCADE_DEBUG_FULL */
+
+void fann_print_connections_raw(struct fann *ann)
+{
+	unsigned int i;
+
+	for(i = 0; i < ann->total_connections_allocated; i++)
+	{
+		if(i == ann->total_connections)
+		{
+			printf("* ");
+		}
+		printf("%f ", ann->weights[i]);
+	}
+	printf("\n\n");
+}
+
+/* Cascade training directly on the training data.
+   The connected_neurons pointers are not valid during training,
+   but they will be again after training.
+ */
+FANN_EXTERNAL void FANN_API fann_cascadetrain_on_data(struct fann *ann, struct fann_train_data *data,
+										unsigned int max_neurons,
+										unsigned int neurons_between_reports,
+										float desired_error)
+{
+	float error;
+	unsigned int i;
+	unsigned int total_epochs = 0;
+	int desired_error_reached;
+
+	if(neurons_between_reports && ann->callback == NULL)
+	{
+		printf("Max neurons %3d. Desired error: %.6f\n", max_neurons, desired_error);
+	}
+
+	for(i = 1; i <= max_neurons; i++)
+	{
+		/* train output neurons */
+		total_epochs += fann_train_outputs(ann, data, desired_error);
+		error = fann_get_MSE(ann);
+		desired_error_reached = fann_desired_error_reached(ann, desired_error);
+
+		/* print current error */
+		if(neurons_between_reports &&
+		   (i % neurons_between_reports == 0
+			|| i == max_neurons || i == 1 || desired_error_reached == 0))
+		{
+			if(ann->callback == NULL)
+			{
+				printf
+					("Neurons     %3d. Current error: %.6f. Total error:%8.4f. Epochs %5d. Bit fail %3d",
+					 i-1, error, ann->MSE_value, total_epochs, ann->num_bit_fail);
+				if((ann->last_layer-2) != ann->first_layer)
+				{
+					printf(". candidate steepness %.2f. function %s", 
+					   (ann->last_layer-2)->first_neuron->activation_steepness,
+					   FANN_ACTIVATIONFUNC_NAMES[(ann->last_layer-2)->first_neuron->activation_function]);
+				}
+				printf("\n");
+			}
+			else if((*ann->callback) (ann, data, max_neurons, 
+				neurons_between_reports, desired_error, total_epochs) == -1) 
+			{
+				/* you can break the training by returning -1 */
+				break;
+			}					 
+		}
+
+		if(desired_error_reached == 0)
+			break;
+
+		if(fann_initialize_candidates(ann) == -1)
+		{
+			/* Unable to initialize room for candidates */
+			break;
+		}
+
+		/* train new candidates */
+		total_epochs += fann_train_candidates(ann, data);
+
+		/* this installs the best candidate */
+		fann_install_candidate(ann);
+	}
+
+	/* Train outputs one last time but without any desired error */
+	total_epochs += fann_train_outputs(ann, data, 0.0);
+
+	if(neurons_between_reports && ann->callback == NULL)
+	{
+		printf("Train outputs    Current error: %.6f. Epochs %6d\n", fann_get_MSE(ann),
+			   total_epochs);
+	}
+
+	/* Set pointers in connected_neurons
+	 * This is ONLY done in the end of cascade training,
+	 * since there is no need for them during training.
+	 */
+	fann_set_shortcut_connections(ann);
+}
+
+FANN_EXTERNAL void FANN_API fann_cascadetrain_on_file(struct fann *ann, const char *filename,
+													  unsigned int max_neurons,
+													  unsigned int neurons_between_reports,
+													  float desired_error)
+{
+	struct fann_train_data *data = fann_read_train_from_file(filename);
+
+	if(data == NULL)
+	{
+		return;
+	}
+	fann_cascadetrain_on_data(ann, data, max_neurons, neurons_between_reports, desired_error);
+	fann_destroy_train(data);
+}
+
+int fann_train_outputs(struct fann *ann, struct fann_train_data *data, float desired_error)
+{
+	float error, initial_error, error_improvement;
+	float target_improvement = 0.0;
+	float backslide_improvement = -1.0e20f;
+	unsigned int i;
+	unsigned int max_epochs = ann->cascade_max_out_epochs;
+	unsigned int min_epochs = ann->cascade_min_out_epochs;
+	unsigned int stagnation = max_epochs;
+
+	/* TODO should perhaps not clear all arrays */
+	fann_clear_train_arrays(ann);
+
+	/* run an initial epoch to set the initital error */
+	initial_error = fann_train_outputs_epoch(ann, data);
+
+	if(fann_desired_error_reached(ann, desired_error) == 0)
+		return 1;
+
+	for(i = 1; i < max_epochs; i++)
+	{
+		error = fann_train_outputs_epoch(ann, data);
+
+		/*printf("Epoch %6d. Current error: %.6f. Bit fail %d.\n", i, error, ann->num_bit_fail); */
+
+		if(fann_desired_error_reached(ann, desired_error) == 0)
+		{
+#ifdef CASCADE_DEBUG
+			printf("Error %f < %f\n", error, desired_error);
+#endif
+			return i + 1;
+		}
+
+		/* Improvement since start of train */
+		error_improvement = initial_error - error;
+
+		/* After any significant change, set a new goal and
+		 * allow a new quota of epochs to reach it */
+		
+		if((target_improvement >= 0 &&
+			(error_improvement > target_improvement || error_improvement < backslide_improvement)) ||
+		(target_improvement < 0 &&
+			(error_improvement < target_improvement || error_improvement > backslide_improvement)))
+		{
+			/*printf("error_improvement=%f, target_improvement=%f, backslide_improvement=%f, stagnation=%d\n", error_improvement, target_improvement, backslide_improvement, stagnation); */
+
+			target_improvement = error_improvement * (1.0f + ann->cascade_output_change_fraction);
+			backslide_improvement = error_improvement * (1.0f - ann->cascade_output_change_fraction);
+			stagnation = i + ann->cascade_output_stagnation_epochs;
+		}
+
+		/* No improvement in allotted period, so quit */
+		if(i >= stagnation && i >= min_epochs)
+		{
+			return i + 1;
+		}
+	}
+
+	return max_epochs;
+}
+
+float fann_train_outputs_epoch(struct fann *ann, struct fann_train_data *data)
+{
+	unsigned int i;
+	
+	fann_reset_MSE(ann);
+
+	for(i = 0; i < data->num_data; i++)
+	{
+		fann_run(ann, data->input[i]);
+		fann_compute_MSE(ann, data->output[i]);
+		fann_update_slopes_batch(ann, ann->last_layer - 1, ann->last_layer - 1);
+	}
+
+	switch (ann->training_algorithm)
+	{
+		case FANN_TRAIN_RPROP:
+			fann_update_weights_irpropm(ann, (ann->last_layer - 1)->first_neuron->first_con,
+										ann->total_connections);
+			break;
+		case FANN_TRAIN_SARPROP:
+			fann_update_weights_sarprop(ann, ann->sarprop_epoch, (ann->last_layer - 1)->first_neuron->first_con,
+										ann->total_connections);
+			++(ann->sarprop_epoch);
+			break;
+		case FANN_TRAIN_QUICKPROP:
+			fann_update_weights_quickprop(ann, data->num_data,
+										  (ann->last_layer - 1)->first_neuron->first_con,
+										  ann->total_connections);
+			break;
+		case FANN_TRAIN_BATCH:
+		case FANN_TRAIN_INCREMENTAL:
+			fann_error((struct fann_error *) ann, FANN_E_CANT_USE_TRAIN_ALG);
+	}
+
+	return fann_get_MSE(ann);
+}
+
+int fann_reallocate_connections(struct fann *ann, unsigned int total_connections)
+{
+	/* The connections are allocated, but the pointers inside are
+	 * first moved in the end of the cascade training session.
+	 */
+
+#ifdef CASCADE_DEBUG
+	printf("realloc from %d to %d\n", ann->total_connections_allocated, total_connections);
+#endif
+	ann->connections =
+		(struct fann_neuron **) realloc(ann->connections,
+										total_connections * sizeof(struct fann_neuron *));
+	if(ann->connections == NULL)
+	{
+		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
+		return -1;
+	}
+
+	ann->weights = (fann_type *) realloc(ann->weights, total_connections * sizeof(fann_type));
+	if(ann->weights == NULL)
+	{
+		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
+		return -1;
+	}
+
+	ann->train_slopes =
+		(fann_type *) realloc(ann->train_slopes, total_connections * sizeof(fann_type));
+	if(ann->train_slopes == NULL)
+	{
+		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
+		return -1;
+	}
+
+	ann->prev_steps = (fann_type *) realloc(ann->prev_steps, total_connections * sizeof(fann_type));
+	if(ann->prev_steps == NULL)
+	{
+		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
+		return -1;
+	}
+
+	ann->prev_train_slopes =
+		(fann_type *) realloc(ann->prev_train_slopes, total_connections * sizeof(fann_type));
+	if(ann->prev_train_slopes == NULL)
+	{
+		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
+		return -1;
+	}
+
+	ann->total_connections_allocated = total_connections;
+
+	return 0;
+}
+
+int fann_reallocate_neurons(struct fann *ann, unsigned int total_neurons)
+{
+	struct fann_layer *layer_it;
+	struct fann_neuron *neurons;
+	unsigned int num_neurons = 0;
+	unsigned int num_neurons_so_far = 0;
+
+	neurons =
+		(struct fann_neuron *) realloc(ann->first_layer->first_neuron,
+									   total_neurons * sizeof(struct fann_neuron));
+	ann->total_neurons_allocated = total_neurons;
+
+	if(neurons == NULL)
+	{
+		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
+		return -1;
+	}
+
+	/* Also allocate room for more train_errors */
+	ann->train_errors = (fann_type *) realloc(ann->train_errors, total_neurons * sizeof(fann_type));
+	if(ann->train_errors == NULL)
+	{
+		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
+		return -1;
+	}
+
+	if(neurons != ann->first_layer->first_neuron)
+	{
+		/* Then the memory has moved, also move the pointers */
+
+#ifdef CASCADE_DEBUG_FULL
+		printf("Moving neuron pointers\n");
+#endif
+
+		/* Move pointers from layers to neurons */
+		for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++)
+		{
+			num_neurons = layer_it->last_neuron - layer_it->first_neuron;
+			layer_it->first_neuron = neurons + num_neurons_so_far;
+			layer_it->last_neuron = layer_it->first_neuron + num_neurons;
+			num_neurons_so_far += num_neurons;
+		}
+	}
+
+	return 0;
+}
+
+void initialize_candidate_weights(struct fann *ann, unsigned int first_con, unsigned int last_con, float scale_factor)
+{
+	fann_type prev_step;
+	unsigned int i = 0;
+	unsigned int bias_weight = first_con + (ann->first_layer->last_neuron - ann->first_layer->first_neuron) - 1;
+
+	if(ann->training_algorithm == FANN_TRAIN_RPROP)
+		prev_step = ann->rprop_delta_zero;
+	else
+		prev_step = 0;
+
+	for(i = first_con; i < last_con; i++)
+	{
+		if(i == bias_weight) 
+			ann->weights[i] = fann_rand(-scale_factor, scale_factor);
+		else
+			ann->weights[i] = fann_rand(0,scale_factor);
+					
+		ann->train_slopes[i] = 0;
+		ann->prev_steps[i] = prev_step;
+		ann->prev_train_slopes[i] = 0;
+	}
+}
+
+int fann_initialize_candidates(struct fann *ann)
+{
+	/* The candidates are allocated after the normal neurons and connections,
+	 * but there is an empty place between the real neurons and the candidate neurons,
+	 * so that it will be possible to make room when the chosen candidate are copied in
+	 * on the desired place.
+	 */
+	unsigned int neurons_to_allocate, connections_to_allocate;
+	unsigned int num_candidates = fann_get_cascade_num_candidates(ann);
+	unsigned int num_neurons = ann->total_neurons + num_candidates + 1;
+	unsigned int num_hidden_neurons = ann->total_neurons - ann->num_input - ann->num_output;
+	unsigned int candidate_connections_in = ann->total_neurons - ann->num_output;
+	unsigned int candidate_connections_out = ann->num_output;
+
+	/* the number of connections going into a and out of a candidate is
+	 * ann->total_neurons */
+	unsigned int num_connections =
+		ann->total_connections + (ann->total_neurons * (num_candidates + 1));
+	unsigned int first_candidate_connection = ann->total_connections + ann->total_neurons;
+	unsigned int first_candidate_neuron = ann->total_neurons + 1;
+	unsigned int connection_it, i, j, k, candidate_index;
+	struct fann_neuron *neurons;
+	float scale_factor;
+	
+	/* First make sure that there is enough room, and if not then allocate a
+	 * bit more so that we do not need to allocate more room each time.
+	 */
+	if(num_neurons > ann->total_neurons_allocated)
+	{
+		/* Then we need to allocate more neurons
+		 * Allocate half as many neurons as already exist (at least ten)
+		 */
+		neurons_to_allocate = num_neurons + num_neurons / 2;
+		if(neurons_to_allocate < num_neurons + 10)
+		{
+			neurons_to_allocate = num_neurons + 10;
+		}
+
+		if(fann_reallocate_neurons(ann, neurons_to_allocate) == -1)
+		{
+			return -1;
+		}
+	}
+
+	if(num_connections > ann->total_connections_allocated)
+	{
+		/* Then we need to allocate more connections
+		 * Allocate half as many connections as already exist
+		 * (at least enough for ten neurons)
+		 */
+		connections_to_allocate = num_connections + num_connections / 2;
+		if(connections_to_allocate < num_connections + ann->total_neurons * 10)
+		{
+			connections_to_allocate = num_connections + ann->total_neurons * 10;
+		}
+
+		if(fann_reallocate_connections(ann, connections_to_allocate) == -1)
+		{
+			return -1;
+		}
+	}
+
+	/* Some code to do semi Widrow + Nguyen initialization */
+	scale_factor = (float) (2.0 * pow(0.7f * (float)num_hidden_neurons, 1.0f / (float) ann->num_input));
+	if(scale_factor > 8)
+		scale_factor = 8;
+	else if(scale_factor < 0.5)
+		scale_factor = 0.5;
+
+	/* Set the neurons.
+	 */
+	connection_it = first_candidate_connection;
+	neurons = ann->first_layer->first_neuron;
+	candidate_index = first_candidate_neuron;
+
+	for(i = 0; i < ann->cascade_activation_functions_count; i++)
+	{
+		for(j = 0; j < ann->cascade_activation_steepnesses_count; j++)
+		{
+			for(k = 0; k < ann->cascade_num_candidate_groups; k++)
+			{
+				/* TODO candidates should actually be created both in
+				 * the last layer before the output layer, and in a new layer.
+				 */
+				neurons[candidate_index].value = 0;
+				neurons[candidate_index].sum = 0;
+				
+				neurons[candidate_index].activation_function =
+					ann->cascade_activation_functions[i];
+				neurons[candidate_index].activation_steepness =
+					ann->cascade_activation_steepnesses[j];
+				
+				neurons[candidate_index].first_con = connection_it;
+				connection_it += candidate_connections_in;
+				neurons[candidate_index].last_con = connection_it;
+				/* We have no specific pointers to the output weights, but they are
+				 * available after last_con */
+				connection_it += candidate_connections_out;
+				ann->train_errors[candidate_index] = 0;
+				initialize_candidate_weights(ann, neurons[candidate_index].first_con, neurons[candidate_index].last_con+candidate_connections_out, scale_factor);
+				candidate_index++;
+			}
+		}
+	}
+
+	
+	/* Now randomize the weights and zero out the arrays that needs zeroing out.
+	 */
+	 /*
+#ifdef CASCADE_DEBUG_FULL
+	printf("random cand weight [%d ... %d]\n", first_candidate_connection, num_connections - 1);
+#endif
+
+	for(i = first_candidate_connection; i < num_connections; i++)
+	{
+		
+		//ann->weights[i] = fann_random_weight();
+		ann->weights[i] = fann_rand(-2.0,2.0);
+		ann->train_slopes[i] = 0;
+		ann->prev_steps[i] = 0;
+		ann->prev_train_slopes[i] = initial_slope;
+	}
+	*/
+
+	return 0;
+}
+
+int fann_train_candidates(struct fann *ann, struct fann_train_data *data)
+{
+	fann_type best_cand_score = 0.0;
+	fann_type target_cand_score = 0.0;
+	fann_type backslide_cand_score = -1.0e20f;
+	unsigned int i;
+	unsigned int max_epochs = ann->cascade_max_cand_epochs;
+	unsigned int min_epochs = ann->cascade_min_cand_epochs;
+	unsigned int stagnation = max_epochs;
+
+	if(ann->cascade_candidate_scores == NULL)
+	{
+		ann->cascade_candidate_scores =
+			(fann_type *) malloc(fann_get_cascade_num_candidates(ann) * sizeof(fann_type));
+		if(ann->cascade_candidate_scores == NULL)
+		{
+			fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
+			return 0;
+		}
+	}
+
+	for(i = 0; i < max_epochs; i++)
+	{
+		best_cand_score = fann_train_candidates_epoch(ann, data);
+
+		if(best_cand_score / ann->MSE_value > ann->cascade_candidate_limit)
+		{
+#ifdef CASCADE_DEBUG
+			printf("above candidate limit %f/%f > %f", best_cand_score, ann->MSE_value,
+				   ann->cascade_candidate_limit);
+#endif
+			return i + 1;
+		}
+
+		if((best_cand_score > target_cand_score) || (best_cand_score < backslide_cand_score))
+		{
+#ifdef CASCADE_DEBUG_FULL
+			printf("Best candidate score %f, real score: %f\n", ann->MSE_value - best_cand_score,
+				   best_cand_score);
+			/* printf("best_cand_score=%f, target_cand_score=%f, backslide_cand_score=%f, stagnation=%d\n", best_cand_score, target_cand_score, backslide_cand_score, stagnation); */
+#endif
+
+			target_cand_score = best_cand_score * (1.0f + ann->cascade_candidate_change_fraction);
+			backslide_cand_score = best_cand_score * (1.0f - ann->cascade_candidate_change_fraction);
+			stagnation = i + ann->cascade_candidate_stagnation_epochs;
+		}
+
+		/* No improvement in allotted period, so quit */
+		if(i >= stagnation && i >= min_epochs)
+		{
+#ifdef CASCADE_DEBUG
+			printf("Stagnation with %d epochs, best candidate score %f, real score: %f\n", i + 1,
+				   ann->MSE_value - best_cand_score, best_cand_score);
+#endif
+			return i + 1;
+		}
+	}
+
+#ifdef CASCADE_DEBUG
+	printf("Max epochs %d reached, best candidate score %f, real score: %f\n", max_epochs,
+		   ann->MSE_value - best_cand_score, best_cand_score);
+#endif
+	return max_epochs;
+}
+
+void fann_update_candidate_slopes(struct fann *ann)
+{
+	struct fann_neuron *neurons = ann->first_layer->first_neuron;
+	struct fann_neuron *first_cand = neurons + ann->total_neurons + 1;
+	struct fann_neuron *last_cand = first_cand + fann_get_cascade_num_candidates(ann);
+	struct fann_neuron *cand_it;
+	unsigned int i, j, num_connections;
+	unsigned int num_output = ann->num_output;
+	fann_type max_sum, cand_sum, activation, derived, error_value, diff, cand_score;
+	fann_type *weights, *cand_out_weights, *cand_slopes, *cand_out_slopes;
+	fann_type *output_train_errors = ann->train_errors + (ann->total_neurons - ann->num_output);
+
+	for(cand_it = first_cand; cand_it < last_cand; cand_it++)
+	{
+		cand_score = ann->cascade_candidate_scores[cand_it - first_cand];
+		error_value = 0.0;
+
+		/* code more or less stolen from fann_run to fast forward pass
+		 */
+		cand_sum = 0.0;
+		num_connections = cand_it->last_con - cand_it->first_con;
+		weights = ann->weights + cand_it->first_con;
+
+		/* unrolled loop start */
+		i = num_connections & 3;	/* same as modulo 4 */
+		switch (i)
+		{
+			case 3:
+				cand_sum += weights[2] * neurons[2].value;
+			case 2:
+				cand_sum += weights[1] * neurons[1].value;
+			case 1:
+				cand_sum += weights[0] * neurons[0].value;
+			case 0:
+				break;
+		}
+
+		for(; i != num_connections; i += 4)
+		{
+			cand_sum +=
+				weights[i] * neurons[i].value +
+				weights[i + 1] * neurons[i + 1].value +
+				weights[i + 2] * neurons[i + 2].value + weights[i + 3] * neurons[i + 3].value;
+		}
+		/*
+		 * for(i = 0; i < num_connections; i++){
+		 * cand_sum += weights[i] * neurons[i].value;
+		 * }
+		 */
+		/* unrolled loop end */
+
+		max_sum = 150/cand_it->activation_steepness;
+		if(cand_sum > max_sum)
+			cand_sum = max_sum;
+		else if(cand_sum < -max_sum)
+			cand_sum = -max_sum;
+		
+		activation =
+			fann_activation(ann, cand_it->activation_function, cand_it->activation_steepness,
+							cand_sum);
+		/* printf("%f = sigmoid(%f);\n", activation, cand_sum); */
+
+		cand_it->sum = cand_sum;
+		cand_it->value = activation;
+
+		derived = fann_activation_derived(cand_it->activation_function,
+										  cand_it->activation_steepness, activation, cand_sum);
+
+		/* The output weights is located right after the input weights in
+		 * the weight array.
+		 */
+		cand_out_weights = weights + num_connections;
+
+		cand_out_slopes = ann->train_slopes + cand_it->first_con + num_connections;
+		for(j = 0; j < num_output; j++)
+		{
+			diff = (activation * cand_out_weights[j]) - output_train_errors[j];
+#ifdef CASCADE_DEBUG_FULL
+			/* printf("diff = %f = (%f * %f) - %f;\n", diff, activation, cand_out_weights[j], output_train_errors[j]); */
+#endif
+			cand_out_slopes[j] -= 2.0f * diff * activation;
+#ifdef CASCADE_DEBUG_FULL
+			/* printf("cand_out_slopes[%d] <= %f += %f * %f;\n", j, cand_out_slopes[j], diff, activation); */
+#endif
+			error_value += diff * cand_out_weights[j];
+			cand_score -= (diff * diff);
+#ifdef CASCADE_DEBUG_FULL
+			/* printf("cand_score[%d][%d] = %f -= (%f * %f)\n", cand_it - first_cand, j, cand_score, diff, diff); */
+
+			printf("cand[%d]: error=%f, activation=%f, diff=%f, slope=%f\n", cand_it - first_cand,
+				   output_train_errors[j], (activation * cand_out_weights[j]), diff,
+				   -2.0 * diff * activation);
+#endif
+		}
+
+		ann->cascade_candidate_scores[cand_it - first_cand] = cand_score;
+		error_value *= derived;
+
+		cand_slopes = ann->train_slopes + cand_it->first_con;
+		for(i = 0; i < num_connections; i++)
+		{
+			cand_slopes[i] -= error_value * neurons[i].value;
+		}
+	}
+}
+
+void fann_update_candidate_weights(struct fann *ann, unsigned int num_data)
+{
+	struct fann_neuron *first_cand = (ann->last_layer - 1)->last_neuron + 1;	/* there is an empty neuron between the actual neurons and the candidate neuron */
+	struct fann_neuron *last_cand = first_cand + fann_get_cascade_num_candidates(ann) - 1;
+
+	switch (ann->training_algorithm)
+	{
+		case FANN_TRAIN_RPROP:
+			fann_update_weights_irpropm(ann, first_cand->first_con,
+										last_cand->last_con + ann->num_output);
+			break;
+		case FANN_TRAIN_SARPROP:
+			/* TODO: increase epoch? */
+			fann_update_weights_sarprop(ann, ann->sarprop_epoch, first_cand->first_con,
+										last_cand->last_con + ann->num_output);
+			break;
+		case FANN_TRAIN_QUICKPROP:
+			fann_update_weights_quickprop(ann, num_data, first_cand->first_con,
+										  last_cand->last_con + ann->num_output);
+			break;
+		case FANN_TRAIN_BATCH:
+		case FANN_TRAIN_INCREMENTAL:
+			fann_error((struct fann_error *) ann, FANN_E_CANT_USE_TRAIN_ALG);
+			break;
+	}
+}
+
+fann_type fann_train_candidates_epoch(struct fann *ann, struct fann_train_data *data)
+{
+	unsigned int i, j;
+	unsigned int best_candidate;
+	fann_type best_score;
+	unsigned int num_cand = fann_get_cascade_num_candidates(ann);
+	fann_type *output_train_errors = ann->train_errors + (ann->total_neurons - ann->num_output);
+	struct fann_neuron *output_neurons = (ann->last_layer - 1)->first_neuron;
+
+	for(i = 0; i < num_cand; i++)
+	{
+		/* The ann->MSE_value is actually the sum squared error */
+		ann->cascade_candidate_scores[i] = ann->MSE_value;
+	}
+	/*printf("start score: %f\n", ann->MSE_value); */
+
+	for(i = 0; i < data->num_data; i++)
+	{
+		fann_run(ann, data->input[i]);
+
+		for(j = 0; j < ann->num_output; j++)
+		{
+			/* TODO only debug, but the error is in opposite direction, this might be usefull info */
+			/*          if(output_train_errors[j] != (ann->output[j] - data->output[i][j])){
+			 * printf("difference in calculated error at %f != %f; %f = %f - %f;\n", output_train_errors[j], (ann->output[j] - data->output[i][j]), output_train_errors[j], ann->output[j], data->output[i][j]);
+			 * } */
+
+			/*
+			 * output_train_errors[j] = (data->output[i][j] - ann->output[j])/2;
+			 * output_train_errors[j] = ann->output[j] - data->output[i][j];
+			 */
+
+			output_train_errors[j] = (data->output[i][j] - ann->output[j]);
+
+			switch (output_neurons[j].activation_function)
+			{
+				case FANN_LINEAR_PIECE_SYMMETRIC:
+				case FANN_SIGMOID_SYMMETRIC:
+				case FANN_SIGMOID_SYMMETRIC_STEPWISE:
+				case FANN_THRESHOLD_SYMMETRIC:
+				case FANN_ELLIOT_SYMMETRIC:
+				case FANN_GAUSSIAN_SYMMETRIC:
+				case FANN_SIN_SYMMETRIC:
+				case FANN_COS_SYMMETRIC:
+					output_train_errors[j] /= 2.0;
+					break;
+				case FANN_LINEAR:
+				case FANN_THRESHOLD:
+				case FANN_SIGMOID:
+				case FANN_SIGMOID_STEPWISE:
+				case FANN_GAUSSIAN:
+				case FANN_GAUSSIAN_STEPWISE:
+				case FANN_ELLIOT:
+				case FANN_LINEAR_PIECE:
+				case FANN_SIN:
+				case FANN_COS:
+					break;
+			}
+		}
+
+		fann_update_candidate_slopes(ann);
+	}
+
+	fann_update_candidate_weights(ann, data->num_data);
+
+	/* find the best candidate score */
+	best_candidate = 0;
+	best_score = ann->cascade_candidate_scores[best_candidate];
+	for(i = 1; i < num_cand; i++)
+	{
+		/*struct fann_neuron *cand = ann->first_layer->first_neuron + ann->total_neurons + 1 + i;
+		 * printf("candidate[%d] = activation: %s, steepness: %f, score: %f\n", 
+		 * i, FANN_ACTIVATIONFUNC_NAMES[cand->activation_function], 
+		 * cand->activation_steepness, ann->cascade_candidate_scores[i]); */
+
+		if(ann->cascade_candidate_scores[i] > best_score)
+		{
+			best_candidate = i;
+			best_score = ann->cascade_candidate_scores[best_candidate];
+		}
+	}
+
+	ann->cascade_best_candidate = ann->total_neurons + best_candidate + 1;
+#ifdef CASCADE_DEBUG
+	printf("Best candidate[%d]: with score %f, real score: %f\n", best_candidate,
+		   ann->MSE_value - best_score, best_score);
+#endif
+
+	return best_score;
+}
+
+/* add a layer ad the position pointed to by *layer */
+struct fann_layer *fann_add_layer(struct fann *ann, struct fann_layer *layer)
+{
+	int layer_pos = layer - ann->first_layer;
+	int num_layers = ann->last_layer - ann->first_layer + 1;
+	int i;
+
+	/* allocate the layer */
+	struct fann_layer *layers =
+		(struct fann_layer *) realloc(ann->first_layer, num_layers * sizeof(struct fann_layer));
+	if(layers == NULL)
+	{
+		fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
+		return NULL;
+	}
+
+	/* copy layers so that the free space is at the right location */
+	for(i = num_layers - 1; i >= layer_pos; i--)
+	{
+		layers[i] = layers[i - 1];
+	}
+
+	/* the newly allocated layer is empty */
+	layers[layer_pos].first_neuron = layers[layer_pos + 1].first_neuron;
+	layers[layer_pos].last_neuron = layers[layer_pos + 1].first_neuron;
+
+	/* Set the ann pointers correctly */
+	ann->first_layer = layers;
+	ann->last_layer = layers + num_layers;
+
+#ifdef CASCADE_DEBUG_FULL
+	printf("add layer at pos %d\n", layer_pos);
+#endif
+
+	return layers + layer_pos;
+}
+
+void fann_set_shortcut_connections(struct fann *ann)
+{
+	struct fann_layer *layer_it;
+	struct fann_neuron *neuron_it, **neuron_pointers, *neurons;
+	unsigned int num_connections = 0, i;
+
+	neuron_pointers = ann->connections;
+	neurons = ann->first_layer->first_neuron;
+
+	for(layer_it = ann->first_layer + 1; layer_it != ann->last_layer; layer_it++)
+	{
+		for(neuron_it = layer_it->first_neuron; neuron_it != layer_it->last_neuron; neuron_it++)
+		{
+
+			neuron_pointers += num_connections;
+			num_connections = neuron_it->last_con - neuron_it->first_con;
+
+			for(i = 0; i != num_connections; i++)
+			{
+				neuron_pointers[i] = neurons + i;
+			}
+		}
+	}
+}
+
+void fann_add_candidate_neuron(struct fann *ann, struct fann_layer *layer)
+{
+	unsigned int num_connections_in = layer->first_neuron - ann->first_layer->first_neuron;
+	unsigned int num_connections_out =
+		(ann->last_layer - 1)->last_neuron - (layer + 1)->first_neuron;
+	unsigned int num_connections_move = num_connections_out + num_connections_in;
+
+	unsigned int candidate_con, candidate_output_weight;
+	int i;
+
+	struct fann_layer *layer_it;
+	struct fann_neuron *neuron_it, *neuron_place, *candidate;
+
+	/* We know that there is enough room for the new neuron
+	 * (the candidates are in the same arrays), so move
+	 * the last neurons to make room for this neuron.
+	 */
+
+	/* first move the pointers to neurons in the layer structs */
+	for(layer_it = ann->last_layer - 1; layer_it != layer; layer_it--)
+	{
+#ifdef CASCADE_DEBUG_FULL
+		printf("move neuron pointers in layer %d, first(%d -> %d), last(%d -> %d)\n",
+			   layer_it - ann->first_layer,
+			   layer_it->first_neuron - ann->first_layer->first_neuron,
+			   layer_it->first_neuron - ann->first_layer->first_neuron + 1,
+			   layer_it->last_neuron - ann->first_layer->first_neuron,
+			   layer_it->last_neuron - ann->first_layer->first_neuron + 1);
+#endif
+		layer_it->first_neuron++;
+		layer_it->last_neuron++;
+	}
+
+	/* also move the last neuron in the layer that needs the neuron added */
+	layer->last_neuron++;
+
+	/* this is the place that should hold the new neuron */
+	neuron_place = layer->last_neuron - 1;
+
+#ifdef CASCADE_DEBUG_FULL
+	printf("num_connections_in=%d, num_connections_out=%d\n", num_connections_in,
+		   num_connections_out);
+#endif
+
+	candidate = ann->first_layer->first_neuron + ann->cascade_best_candidate;
+
+	/* the output weights for the candidates are located after the input weights */
+	candidate_output_weight = candidate->last_con;
+
+	/* move the actual output neurons and the indexes to the connection arrays */
+	for(neuron_it = (ann->last_layer - 1)->last_neuron - 1; neuron_it != neuron_place; neuron_it--)
+	{
+#ifdef CASCADE_DEBUG_FULL
+		printf("move neuron %d -> %d\n", neuron_it - ann->first_layer->first_neuron - 1,
+			   neuron_it - ann->first_layer->first_neuron);
+#endif
+		*neuron_it = *(neuron_it - 1);
+
+		/* move the weights */
+#ifdef CASCADE_DEBUG_FULL
+		printf("move weight[%d ... %d] -> weight[%d ... %d]\n", neuron_it->first_con,
+			   neuron_it->last_con - 1, neuron_it->first_con + num_connections_move - 1,
+			   neuron_it->last_con + num_connections_move - 2);
+#endif
+		for(i = neuron_it->last_con - 1; i >= (int)neuron_it->first_con; i--)
+		{
+#ifdef CASCADE_DEBUG_FULL
+			printf("move weight[%d] = weight[%d]\n", i + num_connections_move - 1, i);
+#endif
+			ann->weights[i + num_connections_move - 1] = ann->weights[i];
+		}
+
+		/* move the indexes to weights */
+		neuron_it->last_con += num_connections_move;
+		num_connections_move--;
+		neuron_it->first_con += num_connections_move;
+
+		/* set the new weight to the newly allocated neuron */
+		ann->weights[neuron_it->last_con - 1] =
+			(ann->weights[candidate_output_weight]) * ann->cascade_weight_multiplier;
+		candidate_output_weight++;
+	}
+
+	/* Now inititalize the actual neuron */
+	neuron_place->value = 0;
+	neuron_place->sum = 0;
+	neuron_place->activation_function = candidate->activation_function;
+	neuron_place->activation_steepness = candidate->activation_steepness;
+	neuron_place->last_con = (neuron_place + 1)->first_con;
+	neuron_place->first_con = neuron_place->last_con - num_connections_in;
+#ifdef CASCADE_DEBUG_FULL
+	printf("neuron[%d] = weights[%d ... %d] activation: %s, steepness: %f\n",
+		   neuron_place - ann->first_layer->first_neuron, neuron_place->first_con,
+		   neuron_place->last_con - 1, FANN_ACTIVATIONFUNC_NAMES[neuron_place->activation_function],
+		   neuron_place->activation_steepness);/* TODO remove */
+#endif
+
+	candidate_con = candidate->first_con;
+	/* initialize the input weights at random */
+#ifdef CASCADE_DEBUG_FULL
+	printf("move cand weights[%d ... %d] -> [%d ... %d]\n", candidate_con,
+		   candidate_con + num_connections_in - 1, neuron_place->first_con,
+		   neuron_place->last_con - 1);
+#endif
+
+	for(i = 0; i < (int)num_connections_in; i++)
+	{
+		ann->weights[i + neuron_place->first_con] = ann->weights[i + candidate_con];
+#ifdef CASCADE_DEBUG_FULL
+		printf("move weights[%d] -> weights[%d] (%f)\n", i + candidate_con,
+			   i + neuron_place->first_con, ann->weights[i + neuron_place->first_con]);
+#endif
+	}
+
+	/* Change some of main variables */
+	ann->total_neurons++;
+	ann->total_connections += num_connections_in + num_connections_out;
+
+	return;
+}
+
+void fann_install_candidate(struct fann *ann)
+{
+	struct fann_layer *layer;
+
+	layer = fann_add_layer(ann, ann->last_layer - 1);
+	fann_add_candidate_neuron(ann, layer);
+	return;
+}
+
+#endif /* FIXEDFANN */
+
+FANN_EXTERNAL unsigned int FANN_API fann_get_cascade_num_candidates(struct fann *ann)
+{
+	return ann->cascade_activation_functions_count *
+		ann->cascade_activation_steepnesses_count *
+		ann->cascade_num_candidate_groups;
+}
+
+FANN_GET_SET(float, cascade_output_change_fraction)
+FANN_GET_SET(unsigned int, cascade_output_stagnation_epochs)
+FANN_GET_SET(float, cascade_candidate_change_fraction)
+FANN_GET_SET(unsigned int, cascade_candidate_stagnation_epochs)
+FANN_GET_SET(unsigned int, cascade_num_candidate_groups)
+FANN_GET_SET(fann_type, cascade_weight_multiplier)
+FANN_GET_SET(fann_type, cascade_candidate_limit)
+FANN_GET_SET(unsigned int, cascade_max_out_epochs)
+FANN_GET_SET(unsigned int, cascade_max_cand_epochs)
+FANN_GET_SET(unsigned int, cascade_min_out_epochs)
+FANN_GET_SET(unsigned int, cascade_min_cand_epochs)
+
+FANN_GET(unsigned int, cascade_activation_functions_count)
+FANN_GET(enum fann_activationfunc_enum *, cascade_activation_functions)
+
+FANN_EXTERNAL void FANN_API fann_set_cascade_activation_functions(struct fann *ann,
+														 enum fann_activationfunc_enum *
+														 cascade_activation_functions,
+														 unsigned int 
+														 cascade_activation_functions_count)
+{
+	if(ann->cascade_activation_functions_count != cascade_activation_functions_count)
+	{
+		ann->cascade_activation_functions_count = cascade_activation_functions_count;
+		
+		/* reallocate mem */
+		ann->cascade_activation_functions = 
+			(enum fann_activationfunc_enum *)realloc(ann->cascade_activation_functions, 
+			ann->cascade_activation_functions_count * sizeof(enum fann_activationfunc_enum));
+		if(ann->cascade_activation_functions == NULL)
+		{
+			fann_error((struct fann_error*)ann, FANN_E_CANT_ALLOCATE_MEM);
+			return;
+		}
+	}
+	
+	memmove(ann->cascade_activation_functions, cascade_activation_functions, 
+		ann->cascade_activation_functions_count * sizeof(enum fann_activationfunc_enum));
+}
+
+FANN_GET(unsigned int, cascade_activation_steepnesses_count)
+FANN_GET(fann_type *, cascade_activation_steepnesses)
+
+FANN_EXTERNAL void FANN_API fann_set_cascade_activation_steepnesses(struct fann *ann,
+														   fann_type *
+														   cascade_activation_steepnesses,
+														   unsigned int 
+														   cascade_activation_steepnesses_count)
+{
+	if(ann->cascade_activation_steepnesses_count != cascade_activation_steepnesses_count)
+	{
+		ann->cascade_activation_steepnesses_count = cascade_activation_steepnesses_count;
+		
+		/* reallocate mem */
+		ann->cascade_activation_steepnesses = 
+			(fann_type *)realloc(ann->cascade_activation_steepnesses, 
+			ann->cascade_activation_steepnesses_count * sizeof(fann_type));
+		if(ann->cascade_activation_steepnesses == NULL)
+		{
+			fann_error((struct fann_error*)ann, FANN_E_CANT_ALLOCATE_MEM);
+			return;
+		}
+	}
+	
+	memmove(ann->cascade_activation_steepnesses, cascade_activation_steepnesses, 
+		ann->cascade_activation_steepnesses_count * sizeof(fann_type));
+}