newral 0.1

data/lib/newral/functions/base.rb

@@ -0,0 +1,102 @@
+module  Newral
+  module Functions
+    module Errors
+      class InvalidDirection < ::StandardError; end 
+      class NotImplemented < ::StandardError; end 
+    end 
+
+    class Base
+      attr_accessor :center
+     
+      def calculate
+        raise NotImplemented
+      end 
+      
+      # approximates the descent calculation at a certain point
+      def calculate_descent( input, difference: nil )
+        difference = (input/10000.0).abs unless difference
+       (calculate( input+difference )- calculate( input ))/difference
+      end
+
+      # finds the (local) minimum descent of a function
+      def find_minimum( start_input, max_iterations: 1000, treshold: 10**-9 , learning_rate: 0.01 )
+        descent = calculate_descent( start_input )
+        iterations = 0
+        input = start_input
+        while descent.abs >  treshold && iterations < max_iterations
+          old_input = input
+          input = input-descent.to_f*learning_rate
+          new_descent = calculate_descent( input )
+          learning_rate = learning_rate.to_f/10   if new_descent*descent < 0  # slow down if descent changes
+          descent = new_descent
+          iterations = iterations+1
+        end 
+        { input: input, descent: descent, learning_rate: learning_rate, output: calculate( input ), iterations: iterations }
+      end
+
+      def calculate_for_center_distance( vector1  ) 
+        calculate Newral::Tools.euclidian_distance( vector1, @center )
+      end
+
+      # if a function implements calculate,move+number_of_directions we can use it for 
+      # all training algorithms (like hill climbing)
+      # as shown in the Networks / network you do not need to derive from base function
+
+      def self.create_random( low_range: -9, high_range: 9 )
+        raise Errors::NotImplemented         
+      end
+
+      def number_of_directions
+        raise Errors::NotImplemented 
+      end 
+
+   
+      def move( direction: 0, step:0.01, step_percentage: nil )
+        raise Errors::NotImplemented 
+      end
+
+      def move_several( directions:[], step:0.01, step_percentage: nil )
+        directions.each do |direction| 
+          move( direction: direction, step: step, step_percentage: step_percentage)
+        end
+        self
+      end 
+
+      # moves all directions randomly
+      def move_random( low_range: -0.9, high_range: 0.9 ) 
+        number_of_directions.times do |direction|
+          step = low_range+rand()*(high_range.to_f-low_range.to_f) 
+          move( direction: direction, step: step )
+        end
+        self
+      end
+
+      def calculate_error( input: [],output: [] )
+        expected_values = [] # output can be longer than input
+        calculated_values = []
+        input.each_with_index do |x,idx|
+          calculated_values << calculate( x )
+          expected_values << output[idx] 
+        end 
+        Newral::ErrorCalculation.root_mean_square( calculated_values, expected_values  )
+      end
+
+      def error_gradient_approximation( direction: nil,  step: 0.01, input: nil, output: nil )
+        current_error = calculate_error( input: input, output: output)
+        new_pos = self.dup.move( direction: direction, step: step )
+        new_error = new_pos.calculate_error( input: input, output: output)
+        (new_error-current_error)/step
+      end 
+
+      # for  general functions we can only estimate the gradient of the error
+      # by taking small steps
+      def move_with_gradient( input:[], output:[], learning_rate: 0.01, step: 0.01 )
+         number_of_directions.times do |direction|
+          error_gradient = error_gradient_approximation( direction: direction, step: step, input: input, output: output )
+          move( direction: direction, step:(-error_gradient*learning_rate))
+        end
+        self 
+      end
+    end
+  end
+end

data/lib/newral/functions/block.rb

@@ -0,0 +1,34 @@
+module  Newral
+  module Functions
+    module Errors
+      class NoBlock < ::StandardError; end 
+    end 
+    attr_reader :calculate_block
+    # as its ruby we of course have to also offer the possibility for blocks
+    class Block < Base
+      
+      def initialize( directions:0, params:[], &block )
+        raise Errors::NoBlock unless block_given?
+        @calculate_block = block
+        @params = params
+        @directions = directions || params.size
+      end 
+
+      def calculate( input ) 
+        @calculate_block.call input, @params 
+      end
+
+      def move( direction: 0, step:0.01, step_percentage: nil )
+        raise Errors::InvalidDirection unless direction >0 && direction<@directions
+        @params[direction]=(step_percentage ? @params[direction]*(1+step_percentage.to_f/100) : @params[direction]+step )
+        self
+      end
+
+      def number_of_directions 
+        @directions
+      end
+
+  
+    end
+  end
+end

data/lib/newral/functions/gaussian.rb

@@ -0,0 +1,41 @@
+module  Newral
+  module Functions
+    class Gaussian < Base
+      attr_reader :factors
+      def initialize(  center:[0], factor:1 )
+        @factor = factor
+        @center = center.dup
+      end 
+
+      def calculate( input ) 
+       calculate_for_center_distance( [input])
+      end
+
+      def calculate_for_center_distance( vector1  ) 
+        distance = Newral::Tools.euclidian_distance( vector1, @center )
+        Math.exp(-distance**2)*@factor
+      end
+
+     def self.create_random(  low_range: -9, high_range: 9 )
+       self.new center:[low_range+rand(high_range-low_range)],factor: low_range+rand(high_range-low_range)
+     end
+
+     def number_of_directions
+       1+@center.size
+     end
+
+     def move( direction: 0, step:0.01, step_percentage: nil )
+        raise Errors::InvalidDirection if direction >= number_of_directions
+        if direction == 0
+          @factor = ( step_percentage ? @factor*(1+step_percentage/100) : @factor+step_percentage )
+        else 
+          @center = @center.dup
+          @center[direction-1] =  step_percentage ?  @center[direction-1]*(1+step_percentage/100) : @center[direction-1]+step
+        end
+        self
+      end 
+
+    
+    end
+  end
+end

data/lib/newral/functions/line.rb

@@ -0,0 +1,52 @@
+module  Newral
+  module Functions
+   
+
+    class Line < Base
+      attr_accessor :center
+     
+      def initialize( factor: 1, bias: 0, center: nil )
+        @factor = factor
+        @bias = bias
+        @center = center.dup if center
+      end 
+
+      def calculate( input ) 
+        @factor*input+@bias
+      end
+
+      def self.create_random( low_range: -9, high_range: 9 )
+        factor= low_range+rand(high_range-low_range)
+        bias = low_range+rand(high_range-low_range)
+        self.new( factor: factor, bias: bias )
+      end
+
+       def number_of_directions
+        2 
+      end 
+
+      def move( direction: 0, step:0.01, step_percentage: nil )
+        case direction
+          when 0 then @bias=(step_percentage ? @bias*(1+step_percentage.to_f/100) : @bias+step )
+          when 1 then @factor=(step_percentage ? @factor*(1+step_percentage.to_f/100)  : @factor+step )
+        else 
+          raise Errors::InvalidDirection
+        end
+        self
+      end
+
+      def move_with_gradient( input:[], output:[], learning_rate: 0.01 )
+         bias_gradient = 0
+         factor_gradient = 0
+         input.each_with_index do |x,idx|
+           bias_gradient = bias_gradient-2.0/input.size*( output[idx]-( @factor*x + @bias ))
+           factor_gradient = factor_gradient-2.0/input.size*x*( output[idx]-(@factor*x+@bias) )
+           # b_gradient += -(2/N) * (points[i].y - ((m_current*points[i].x) + b_current))
+            # m_gradient += -(2/N) * points[i].x * (points[i].y - ((m_current * points[i].x) + b_current))
+          end 
+        @bias = @bias - (learning_rate * bias_gradient)
+        @factor = @factor - (learning_rate * factor_gradient)
+      end
+    end 
+  end 
+end

data/lib/newral/functions/polynomial.rb

@@ -0,0 +1,48 @@
+module  Newral
+  module Functions
+    class Polynomial < Base
+      attr_reader :factors
+      def initialize(  factors:nil )
+        @factors = factors.dup || [1]
+        @length = @factors.size
+      end 
+
+      def calculate( input ) 
+        result = 0
+        @factors.each_with_index do |factor, idx|
+          result = result+input**(@length-idx-1)*factor # this way its more readable 
+        end 
+        result
+      end
+
+      # caluates descent at input
+      def calculate_descent( input )
+        descent = 0
+        @factors.each_with_index do |factor, idx|
+          descent = descent+input**(@length-idx-2)*factor*(@length-idx-1) if @length-idx-2>= 0 # this way its more readable 
+        end 
+        descent
+      end
+
+     def self.create_random(  length: 3, low_range: -9, high_range: 9 )
+        factors = []
+        length.times do 
+          factors << low_range+rand(high_range-low_range)
+        end
+        self.new( factors: factors )
+     end
+
+     def number_of_directions
+       @factors.size 
+     end 
+
+      def move( direction: 0, step:0.01, step_percentage: nil )
+        raise Errors::InvalidDirection if direction >= number_of_directions
+        @factors = @factors.dup
+        @factors[direction] = step_percentage ?  @factors[direction]*(1+step_percentage.to_f/100) : @factors[direction]+step
+        self
+      end 
+
+    end
+  end
+end

data/lib/newral/functions/radial_basis_function_network.rb

@@ -0,0 +1,54 @@
+module Newral 
+ module Functions
+  class RadialBasisFunctionNetwork < Base
+    attr_reader :weights, :functions
+    def initialize( centers:[], weights: [], bias_weight: 1, klass: Newral::Functions::Gaussian )
+      @klass = klass 
+      @weights = weights.dup
+      @centers = centers.dup
+      @functions = []
+      @weights.each_with_index do |weight,idx|
+        @functions << klass.new( center:centers[idx], factor: weight )
+      end
+      @functions << Line.new(factor:0,bias: bias_weight) if bias_weight != 0
+    end
+
+    def calculate( input )
+      result = 0
+      @functions.each do |function|
+        result = result + function.calculate( input )
+      end
+      result.to_f
+    end
+
+
+     def self.create_random(  length: 3, low_range: -9, high_range: 9, klass: Newral::Functions::Gaussian  )
+        weights = []
+        centers = []
+        length.times do 
+          weights << low_range+rand(high_range-low_range)
+          centers << [low_range+rand(high_range-low_range)]
+        end
+        self.new( centers:centers, weights: weights, bias_weight: low_range+rand(high_range-low_range), klass: klass  )
+     end
+
+     def number_of_directions
+       @weights.size+@centers.collect{ |c| c.size }.sum
+     end
+
+      def move( direction: 0, step:0.01, step_percentage: nil )
+        raise Errors::InvalidDirection if direction >= number_of_directions
+        if direction < @weights.size 
+          @weights = @weights.dup
+          @weights[direction] = step_percentage ? @weights[direction]*(1+step_percentage.to_f/100) : @weights[direction]+step
+        else
+          mod = @centers.first.size 
+          @centers = @centers.dup
+          @centers[(direction-@weights.size)/mod][(direction-@weights.size)%mod] = step_percentage ? @centers[(direction-@weights.size)/mod][(direction-@weights.size)%mod]*(1+step_percentage.to_f/100) : @centers[(direction-@weights.size)/mod][(direction-@weights.size)%mod]+step
+        end 
+        self
+      end
+
+    end
+  end
+end

data/lib/newral/functions/ricker_wavelet.rb

@@ -0,0 +1,13 @@
+module  Newral
+  module Functions
+    class RickerWavelet < Gaussian
+      # The mexican sombrero function
+
+      def calculate_for_center_distance( vector1  ) 
+        distance = Newral::Tools.euclidian_distance( vector1, @center )
+        (1-distance**2)*Math.exp(-distance.to_f**2/2)*@factor
+      end
+
+    end
+  end
+end

data/lib/newral/functions/vector.rb

@@ -0,0 +1,59 @@
+module  Newral
+  module Functions
+  
+
+    class Vector < Base
+      attr_accessor :weights, :bias
+      def initialize( vector: [1,1], bias: 0 )
+        @vector = vector
+        @bias = bias
+      end 
+
+      def calculate( input ) 
+        @vector.zip( input ).map{|x, y| x * y}.sum  + @bias
+      end
+
+      def self.create_random(  length: 2, low_range: -9, high_range: 9 )
+        vector = []
+        length.times do 
+          vector << low_range+rand(high_range-low_range)
+        end
+        self.new( vector: vector, bias: low_range+rand(high_range-low_range) )
+     end
+
+      def number_of_directions
+        @vector.size+1
+      end
+
+       def move( direction: 0, step:0.01, step_percentage: nil )
+        raise Errors::InvalidDirection if direction >= number_of_directions
+        if direction < @vector.size 
+          @vector[direction] = step_percentage ?  @vector[direction]*(1+step_percentage.to_f/100) : @vector[direction]+step
+        else
+          @bias = step_percentage ?  @bias*(1+step_percentage.to_f/100) : @bias+step
+        end
+        self
+      end 
+      
+      # step argument is ignored here 
+      def move_with_gradient( input:[], output:[], learning_rate: 0.01, step: nil )
+         bias_gradient = 0
+         vector_gradient = [0]*@vector.length
+         input.each_with_index do |input_vector,idx|
+           bias_gradient = bias_gradient-2.0/input.size*( output[idx]-(  @vector.zip( input_vector ).map{|x, y| x * y}.sum + @bias ))
+           vector_gradient.each_with_index do |v,idx_2|
+              vector_gradient[idx_2]=v-2.0/input.size*input_vector[idx_2]*( output[idx]-( @vector.zip( input_vector ).map{|x, y| x * y}.sum+@bias) )
+           end
+         end 
+        @bias = @bias - (learning_rate * bias_gradient)
+        new_vector = []
+        @vector.each_with_index do |value,idx| 
+          new_vector << value - (learning_rate * vector_gradient[idx])
+        end 
+        @vector = new_vector
+        self
+      end
+
+    end
+  end
+end

data/lib/newral/genetic/tree.rb

@@ -0,0 +1,70 @@
+module Newral
+  module Genetic
+    class Tree
+      attr_reader :value, :left_child, :right_child, :sub_node_count
+      attr_accessor :parent
+      OPERANDS = { 
+        '+' => 2,
+        '*' => 2,
+        '/' => 2,
+        '-' => 2,
+        'pow' => 2, 
+        'sqrt' => 1
+      }
+      def initialize( parent: nil, value:nil, right_child: nil, left_child: nil )
+        @parent = parent
+        @left_child = left_child
+        @right_child = right_child
+        update_node_count
+        @value = value
+      end     
+
+      def update_node_count 
+        @sub_node_count = ( @left_child && 1 ).to_i+( @left_child && @left_child.sub_node_count ).to_i+( @right_child && 1 ).to_i+( @right_child && @right_child.sub_node_count ).to_i
+        @parent.update_node_count if @parent
+        @sub_node_count
+      end 
+
+      def node_count 
+        @sub_node_count+1
+      end
+
+      def set_child_trees( left_child: nil, right_child: nil, force: false )
+        @left_child = left_child if left_child || force
+        @right_child = right_child if right_child || force
+        @right_child.parent = self if @right_child
+        @left_child.parent = self if @left_child
+        update_node_count
+        self
+      end
+
+      def eval
+        case @value 
+          when '+' then @left_child.eval+@right_child.eval
+          when '-' then @left_child.eval-@right_child.eval
+          when '*' then @left_child.eval*@right_child.eval
+          when '/' then @left_child.eval/@right_child.eval
+          when 'pow' then @left_child.eval**@right_child.eval
+          when 'sqrt' then @left_child.eval**0.5
+        else 
+          @value
+        end
+      end
+
+      def self.full_tree( depth:3, allowed_operands: OPERANDS.keys, terminal_nodes:[]  )
+        if depth > 1
+          value =  allowed_operands[ rand(allowed_operands.size) ]
+          tree = Tree.new( value: value )
+          tree.set_child_trees( left_child: Tree.full_tree( depth: depth-1, allowed_operands: allowed_operands, terminal_nodes: terminal_nodes ))
+          if OPERANDS[value] == 2 
+            tree.set_child_trees( right_child: Tree.full_tree( depth: depth-1, allowed_operands: allowed_operands, terminal_nodes: terminal_nodes ))
+          end
+          tree
+        else 
+          Tree.new( value: terminal_nodes[rand(terminal_nodes.size)])
+        end
+      end 
+
+    end
+  end
+end