mirlo 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d99924e52d6896e3346c45a30fb2c6d202ab2e63
+  data.tar.gz: 7e455ce32eb67d8ac4b1c239b5f87e0302ee4608
+SHA512:
+  metadata.gz: c1c3e145df40a265b77e28b4f466dff0b7f53d16efb432dc28c7b1184051a43be8cc6437c1019bc54606eddc99f8d46072ba11509073f82cd41a0b92f3f8dfdf
+  data.tar.gz: 6e6805bd9b68c22fe8b48b85dc19b671e19db0522a5359968424f7dc7a887daa5fa40fe0f7514306011a921b2dd8f1ba7b5e7aaac87ea65ac657f4b11726f89a

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+.DS_Store
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mirlo.gemspec
+gemspec
+
+group :development do
+  gem 'debugger'
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Alberto F. Capel
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# Mirlo
+
+Some Machine Learning algorithms implemented in Ruby.
+
+Currently implemented:
+
+* Perceptron
+* Multilayer Perceptron. Batch update of neuron weights with momentum.
+
+
+## Example
+
+```ruby
+
+mlp = Mirlo::ANN.build do
+  input_layer  2
+  hidden_layer 3
+  output_layer 1
+end
+# => #<Mirlo::MultilayerPerceptron:0x007fa0e997eff0 ...>
+
+data_set = Mirlo::XorDataSet.new
+# => #<Mirlo::XorDataSet:0x007fa0e9995430 ...>
+
+mlp.train_until(data_set, max_error: 0.0)
+
+mlp.classify([0,0])
+# => [0]
+
+mlp.classify([1,0])
+# => [1]
+
+mlp.classify([0,1])
+# => [1]
+
+mlp.classify([1,1])
+# => [0]
+
+```

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/mirlo.rb

@@ -0,0 +1,33 @@
+module Mirlo
+  ZERO = [0]
+  ONE  = [1]
+  POSITIVE = ONE
+  NEGATIVE = [-1]
+
+  DEFAULT_LEARNING_RATE = 0.05
+  DEFAULT_N_ITERATIONS  = 1000
+
+  require "gnuplot"
+  require_relative "mirlo/version"
+  require_relative "mirlo/plotting"
+  require_relative "mirlo/extensions/matrix"
+  require_relative "mirlo/sample"
+  require_relative "mirlo/sample_with_bias"
+  require_relative "mirlo/dataset"
+  require_relative "mirlo/test_result"
+  require_relative "mirlo/classifier"
+  require_relative "mirlo/ann/input_layer"
+  require_relative "mirlo/ann/neuron_layer"
+  require_relative "mirlo/ann/hidden_layer"
+  require_relative "mirlo/ann/output_layer"
+  require_relative "mirlo/ann/multilayer_perceptron"
+  require_relative "mirlo/ann/ann"
+
+  Dir.glob(File.expand_path('./mirlo/classifiers/*.rb', File.dirname(__FILE__))).each do |f|
+    require f
+  end
+
+  Dir.glob(File.expand_path('./mirlo/datasets/*.rb', File.dirname(__FILE__))).each do |f|
+    require f
+  end
+end

data/lib/mirlo/ann/ann.rb

@@ -0,0 +1,44 @@
+class Mirlo::ANN
+  attr_reader :ann
+
+  def self.build(*args, &block)
+    instance = new(*args)
+    instance.instance_eval(&block)
+    instance.ann
+  end
+
+  def initialize(*args)
+    @ann = Mirlo::MultilayerPerceptron.new(*args)
+  end
+
+  def learning_rate(l_rate)
+    @ann.learning_rate = l_rate
+  end
+
+  def input_layer(n_inputs)
+    @ann.input_layer = Mirlo::InputLayer.new(n_inputs)
+  end
+
+  def hidden_layer(n_neurons)
+    hidden_layer   = Mirlo::HiddenLayer.new(n_neurons)
+
+    connect_with_last_layer(hidden_layer)
+
+    @ann.hidden_layers << hidden_layer
+  end
+
+  def output_layer(n_outputs)
+    output_layer = Mirlo::OutputLayer.new(n_outputs)
+
+    connect_with_last_layer(output_layer)
+
+    @ann.output_layer = output_layer
+  end
+
+  def connect_with_last_layer(layer)
+    previous_layer = @ann.layers.last
+
+    previous_layer.next_layer = layer
+    layer.previous_layer = previous_layer
+  end
+end

data/lib/mirlo/ann/hidden_layer.rb

@@ -0,0 +1,11 @@
+class Mirlo::HiddenLayer < Mirlo::NeuronLayer
+  attr_accessor :next_layer, :errors
+
+  def calculate_errors
+    error_signal = next_layer.errors * next_layer.weights.transpose
+
+    @errors = @activations.apply_elementwise error_signal do |activation, delta|
+      activation * (1.0 - activation) * delta
+    end
+  end
+end

data/lib/mirlo/ann/input_layer.rb

@@ -0,0 +1,23 @@
+module Mirlo
+  class InputLayer
+    attr_accessor :next_layer, :inputs
+
+    def initialize(input_size)
+      @input_size = input_size
+      @inputs = Matrix.zero(1, input_size)
+    end
+
+    def size
+      @input_size + 1
+    end
+
+    def input=(input)
+      sample = input.is_a?(Mirlo::Sample) ? input :  Mirlo::SampleWithBias.new(features: input)
+      @inputs = Matrix.row_vector(sample.features)
+    end
+
+    def activation_matrix
+      @inputs
+    end
+  end
+end

data/lib/mirlo/ann/multilayer_perceptron.rb

@@ -0,0 +1,44 @@
+module Mirlo
+  class MultilayerPerceptron < Classifier
+
+    attr_accessor :learning_rate, :momentum, :input_layer, :hidden_layers, :output_layer
+
+    def initialize(learning_rate: DEFAULT_LEARNING_RATE, momentum: 0.9)
+      @learning_rate, @momentum = learning_rate, momentum
+      @hidden_layers = []
+    end
+
+    def iterate
+      # train_set.shuffle!
+
+      input_layer.inputs = train_set.input_matrix
+      output_layer.expected_targets = train_set.target_matrix
+
+      move_forward
+      move_backward
+    end
+
+    def move_forward(inputs = nil)
+      hidden_layers.each(&:calculate_activations)
+      output_layer.calculate_activations
+    end
+
+    def move_backward
+      output_layer.calculate_errors
+      hidden_layers.reverse.each { |layer| layer.calculate_errors }
+
+      output_layer.update_weights(learning_rate, momentum)
+      hidden_layers.reverse.each { |layer| layer.update_weights(learning_rate, momentum) }
+    end
+
+    def classify(input)
+      input_layer.input = input
+      move_forward
+      output_layer.outputs.first.collect(&:round)
+    end
+
+    def layers
+      [input_layer, hidden_layers, output_layer].flatten.compact
+    end
+  end
+end

data/lib/mirlo/ann/neuron_layer.rb

@@ -0,0 +1,53 @@
+class Mirlo::NeuronLayer
+  attr_accessor :previous_layer, :size, :activations, :errors, :build_weight_function
+
+  def initialize(size)
+    @size = size
+    @errors = Array.new(size, 0)
+  end
+
+  def inputs_matrix
+    # debugger
+    previous_layer.activation_matrix * weights
+  end
+
+  def calculate_activations
+    @activations = activation_matrix
+  end
+
+  def activation_matrix
+    inputs_matrix.collect { |i| activation_function(i) }
+  end
+
+  def error_matrix
+    Matrix.row_vector(@errors)
+  end
+
+  def activation_function(input)
+    1.0/(1 + Math.exp(-input))
+  end
+
+  def weights
+    @weights ||= Matrix.build(previous_layer.size, size) { build_weight }
+  end
+
+  def update_weights(learning_rate, momentum = 0)
+    has_momentum = @weights_update && momentum > 0
+
+    if has_momentum
+      momentum_matrix = @weights_update.collect { |u| u * momentum }
+    end
+
+    @weights_update = learning_rate * (previous_layer.activation_matrix.transpose * errors)
+
+    if has_momentum
+      @weights_update = @weights_update + momentum_matrix
+    end
+
+    @weights = @weights + @weights_update
+  end
+
+  def build_weight
+    @build_weight_function ? @build_weight_function.call : rand(-0.5..0.5)
+  end
+end

data/lib/mirlo/ann/output_layer.rb

@@ -0,0 +1,17 @@
+class Mirlo::OutputLayer < Mirlo::NeuronLayer
+  attr_accessor :errors, :previous_layer
+
+  def expected_targets=(target_matrix)
+    @expected_targets = target_matrix
+  end
+
+  def outputs
+    @activations.row_vectors.collect(&:to_a)
+  end
+
+  def calculate_errors
+    num_samples = @expected_targets.row_count
+
+    @errors = (@expected_targets - @activations).collect { |elm| elm/num_samples }
+  end
+end

data/lib/mirlo/classifier.rb

@@ -0,0 +1,37 @@
+module Mirlo
+  class ClassifyError < StandardError; end
+
+  class Classifier
+    attr_accessor :train_set
+
+    def train(train_set, n_iterations = Mirlo::DEFAULT_N_ITERATIONS)
+      @train_set = train_set
+
+      n_iterations.times { |i| iterate }
+    end
+
+    def train_until(train_set, max_error: 0.01, max_iterations: Mirlo::DEFAULT_N_ITERATIONS)
+      @train_set = train_set
+
+      max_iterations.times do |i|
+        iterate
+        test_result = test_with(train_set)
+
+        break if test_result.mean_squared_error <= max_error
+
+        if i == max_iterations - 1
+          raise ClassifyError.new("Could not reach a standard error of #{max_error} after #{max_iterations} iterations")
+        end
+      end
+    end
+
+    def test_with(test_set)
+      TestResult.new.tap do |tr|
+        test_set.samples.each do |sample|
+          prediction = classify(sample)
+          tr.add(sample, prediction)
+        end
+      end
+    end
+  end
+end

data/lib/mirlo/classifiers/perceptron.rb

@@ -0,0 +1,33 @@
+module Mirlo
+  class Perceptron < Classifier
+    attr_accessor :learning_rate
+
+    def initialize(learning_rate = DEFAULT_LEARNING_RATE)
+      @learning_rate = learning_rate
+    end
+
+    def activations(inputs = train_set.input_matrix)
+      (inputs * weights).collect { |v| v > 0 ? 1 : 0 }
+    end
+
+    def classify(input)
+      input = SampleWithBias.new(features: input) unless input.is_a?(Mirlo::Sample)
+
+      input_vector = Matrix[input.features]
+      result = (input_vector * weights).row(0).to_a
+      result.collect { |v| v > 0 ? 1 : 0 }
+    end
+
+    def weight_updates
+      train_set.input_matrix.transpose * (train_set.target_matrix - activations)
+    end
+
+    def iterate
+      @weights = weights + learning_rate * weight_updates
+    end
+
+    def weights
+      @weights ||= Matrix.build(train_set.num_features, train_set.num_outputs) { rand(-0.05..0.05) }
+    end
+  end
+end

data/lib/mirlo/dataset.rb

@@ -0,0 +1,103 @@
+module Mirlo
+
+  # Public: Dataset class to store a set of samples with their associated targets.
+  #
+  class Dataset
+    include Enumerable
+    include Plotting
+
+    DEFAULT_LABELS = {
+      [0]  => 'Zero',
+      [1]  => 'Positive',
+      [-1] => 'Negative'
+    }
+
+    attr_reader :title, :samples, :feature_names
+
+    def initialize(samples: [], targets: nil, feature_names: [], title: "Dataset", add_bias: true, labels: DEFAULT_LABELS)
+      @feature_names ||= feature_names
+      @title         ||= title
+      @labels        ||= labels
+
+      @samples = if targets.nil?
+        samples
+      else
+        build_from_samples_and_targets(samples, targets)
+      end
+    end
+
+    def feature(feature_name_or_index)
+      index = if feature_names.include?(feature_name_or_index)
+        feature_names.index(feature_name_or_index)
+      else
+        feature_name_or_index
+      end
+
+      samples.collect { |sample| sample[index] }
+    end
+
+    def subset_with_target(target)
+      matching_samples = samples.find_all { |s| s.target == target }
+      Dataset.new(samples: matching_samples, feature_names: feature_names, title: target)
+    end
+
+    def targets_for(feature_values)
+      samples.select { |s| s.has_features?(feature_values) }.collect(&:target)
+    end
+
+    def label_for(val)
+      @labels[val] || val
+    end
+
+    def target_set
+      targets.uniq.sort
+    end
+
+    def targets
+      samples.collect(&:target)
+    end
+
+    def size
+      @samples.size
+    end
+
+    def each(*args, &block)
+      @samples.each(*args, &block)
+    end
+
+    def num_features
+      @num_features ||= samples.first.feature_size
+    end
+
+    def num_outputs
+      @num_outputs ||= samples.first.target_size
+    end
+
+    def input_matrix
+      @input_matrix ||= Matrix.rows(samples.collect(&:features), false)
+    end
+
+    def target_matrix
+      @target_matrix ||= Matrix.rows(samples.collect(&:target), false)
+    end
+
+    def shuffle!
+      @input_matrix = @target_matrix = nil
+
+      shuffled_positions = (0..size-1).to_a.shuffle
+
+      shuffled_samples = shuffled_positions.collect { |i| samples[i] }
+      shuffled_targets = shuffled_positions.collect { |i| targets[i] }
+
+      @samples, @targets = shuffled_samples, shuffled_targets
+    end
+
+    private
+
+    def build_from_samples_and_targets(samples, targets)
+      samples.each_with_index.collect do |sample, index|
+        SampleWithBias.new(target: targets[index], features: sample)
+      end
+    end
+  end
+end