RubyGems - CooCoo - Versions diffs - 0.1.0 - Mend

CooCoo 0.1.0

Files changed (105) hide show

checksums.yaml +7 -0
data/.gitignore +16 -0
data/CooCoo.gemspec +47 -0
data/Gemfile +4 -0
data/Gemfile.lock +88 -0
data/README.md +123 -0
data/Rakefile +81 -0
data/bin/cuda-dev-info +25 -0
data/bin/cuda-free +28 -0
data/bin/cuda-free-trend +7 -0
data/bin/ffi-gen +267 -0
data/bin/spec_runner_html.sh +42 -0
data/bin/trainer +198 -0
data/bin/trend-cost +13 -0
data/examples/char-rnn.rb +405 -0
data/examples/cifar/cifar.rb +94 -0
data/examples/img-similarity.rb +201 -0
data/examples/math_ops.rb +57 -0
data/examples/mnist.rb +365 -0
data/examples/mnist_classifier.rb +293 -0
data/examples/mnist_dream.rb +214 -0
data/examples/seeds.rb +268 -0
data/examples/seeds_dataset.txt +210 -0
data/examples/t10k-images-idx3-ubyte +0 -0
data/examples/t10k-labels-idx1-ubyte +0 -0
data/examples/train-images-idx3-ubyte +0 -0
data/examples/train-labels-idx1-ubyte +0 -0
data/ext/buffer/Rakefile +50 -0
data/ext/buffer/buffer.pre.cu +727 -0
data/ext/buffer/matrix.pre.cu +49 -0
data/lib/CooCoo.rb +1 -0
data/lib/coo-coo.rb +18 -0
data/lib/coo-coo/activation_functions.rb +344 -0
data/lib/coo-coo/consts.rb +5 -0
data/lib/coo-coo/convolution.rb +298 -0
data/lib/coo-coo/core_ext.rb +75 -0
data/lib/coo-coo/cost_functions.rb +91 -0
data/lib/coo-coo/cuda.rb +116 -0
data/lib/coo-coo/cuda/device_buffer.rb +240 -0
data/lib/coo-coo/cuda/device_buffer/ffi.rb +109 -0
data/lib/coo-coo/cuda/error.rb +51 -0
data/lib/coo-coo/cuda/host_buffer.rb +117 -0
data/lib/coo-coo/cuda/runtime.rb +157 -0
data/lib/coo-coo/cuda/vector.rb +315 -0
data/lib/coo-coo/data_sources.rb +2 -0
data/lib/coo-coo/data_sources/xournal.rb +25 -0
data/lib/coo-coo/data_sources/xournal/bitmap_stream.rb +197 -0
data/lib/coo-coo/data_sources/xournal/document.rb +377 -0
data/lib/coo-coo/data_sources/xournal/loader.rb +144 -0
data/lib/coo-coo/data_sources/xournal/renderer.rb +101 -0
data/lib/coo-coo/data_sources/xournal/saver.rb +99 -0
data/lib/coo-coo/data_sources/xournal/training_document.rb +78 -0
data/lib/coo-coo/data_sources/xournal/training_document/constants.rb +15 -0
data/lib/coo-coo/data_sources/xournal/training_document/document_maker.rb +89 -0
data/lib/coo-coo/data_sources/xournal/training_document/document_reader.rb +105 -0
data/lib/coo-coo/data_sources/xournal/training_document/example.rb +37 -0
data/lib/coo-coo/data_sources/xournal/training_document/sets.rb +76 -0
data/lib/coo-coo/debug.rb +8 -0
data/lib/coo-coo/dot.rb +129 -0
data/lib/coo-coo/drawing.rb +4 -0
data/lib/coo-coo/drawing/cairo_canvas.rb +100 -0
data/lib/coo-coo/drawing/canvas.rb +68 -0
data/lib/coo-coo/drawing/chunky_canvas.rb +101 -0
data/lib/coo-coo/drawing/sixel.rb +214 -0
data/lib/coo-coo/enum.rb +17 -0
data/lib/coo-coo/from_name.rb +58 -0
data/lib/coo-coo/fully_connected_layer.rb +205 -0
data/lib/coo-coo/generation_script.rb +38 -0
data/lib/coo-coo/grapher.rb +140 -0
data/lib/coo-coo/image.rb +286 -0
data/lib/coo-coo/layer.rb +67 -0
data/lib/coo-coo/layer_factory.rb +26 -0
data/lib/coo-coo/linear_layer.rb +59 -0
data/lib/coo-coo/math.rb +607 -0
data/lib/coo-coo/math/abstract_vector.rb +121 -0
data/lib/coo-coo/math/functions.rb +39 -0
data/lib/coo-coo/math/interpolation.rb +7 -0
data/lib/coo-coo/network.rb +264 -0
data/lib/coo-coo/neuron.rb +112 -0
data/lib/coo-coo/neuron_layer.rb +168 -0
data/lib/coo-coo/option_parser.rb +18 -0
data/lib/coo-coo/platform.rb +17 -0
data/lib/coo-coo/progress_bar.rb +11 -0
data/lib/coo-coo/recurrence/backend.rb +99 -0
data/lib/coo-coo/recurrence/frontend.rb +101 -0
data/lib/coo-coo/sequence.rb +187 -0
data/lib/coo-coo/shell.rb +2 -0
data/lib/coo-coo/temporal_network.rb +291 -0
data/lib/coo-coo/trainer.rb +21 -0
data/lib/coo-coo/trainer/base.rb +67 -0
data/lib/coo-coo/trainer/batch.rb +82 -0
data/lib/coo-coo/trainer/batch_stats.rb +27 -0
data/lib/coo-coo/trainer/momentum_stochastic.rb +59 -0
data/lib/coo-coo/trainer/stochastic.rb +47 -0
data/lib/coo-coo/transformer.rb +272 -0
data/lib/coo-coo/vector_layer.rb +194 -0
data/lib/coo-coo/version.rb +3 -0
data/lib/coo-coo/weight_deltas.rb +23 -0
data/prototypes/convolution.rb +116 -0
data/prototypes/linear_drop.rb +51 -0
data/prototypes/recurrent_layers.rb +79 -0
data/www/images/screamer.png +0 -0
data/www/images/screamer.xcf +0 -0
data/www/index.html +82 -0
metadata +373 -0

@@ -0,0 +1,42 @@
+#!/bin/zsh
+INDEX=doc/spec/index.html
+# Start the index.html
+cat <<EOT > $INDEX
+<html>
+	<head>
+		<style>.PASSED { color: green; } .FAILED { color: red; }</style>
+	</head>
+	<body>
+		<ul>
+EOT
+# The spec runner
+function run_spec()
+{
+  local spec="$1"
+  local DIR=`dirname $spec`
+  local BASE=`basename -s .spec $spec`
+  echo -n "$spec "
+  mkdir -p doc/$DIR && bundle exec rspec -Ilib -Iexamples $spec -f html > doc/$DIR/$BASE.html && STATE=PASSED || STATE=FAILED
+  echo $STATE
+  cat <<-EOT >> $INDEX
+	<li class="$STATE"><a href="../$DIR/$BASE.html">$DIR/$BASE</a> &mdash; <span class="state">$STATE</span></li>
+EOT
+}
+# And the loop
+for spec in `find spec -name \*.spec | sort`; do
+  run_spec "$spec"
+done
+# Finalize the index
+cat <<EOT >> $INDEX
+		</ul>
+	</body>
+</html>
+EOT

data/bin/trainer ADDED

@@ -0,0 +1,198 @@
+#!/bin/env ruby
+ROOT = File.dirname(File.dirname(__FILE__))
+$: << File.join(ROOT, 'lib') << File.join(ROOT, 'examples')
+require 'colorize'
+require 'coo-coo'
+def load_script_opts(path)
+  CooCoo::GenerationScript.
+    new(path, $stdout).
+    opts
+end
+require 'pry'
+$pry = {
+  binding: binding,
+  main_thread: Thread.current
+}
+Signal.trap("USR1") do
+  $pry[:thread] ||= Thread.new do
+    $pry[:binding].pry
+  end
+end
+require 'ostruct'
+options = OpenStruct.new
+options.epochs = 1000
+options.dataset = nil
+options.model_path = nil
+options.prototype = nil
+options.trainer = nil
+options.num_tests = 20
+options.start_tests_at = 0
+options.test_cost = CooCoo::CostFunctions::MeanSquare
+opts = CooCoo::OptionParser.new do |o|
+  o.on('-d', '--dataset PATH') do |path|
+    options.dataset = path
+  end
+  o.on('-m', '--model PATH') do |path|
+    options.model_path = Pathname.new(path)
+    options.binary_blob = options.binary_blob || File.extname(options.model_path) == '.bin'
+  end
+  o.on('--prototype PATH') do |path|
+    options.prototype = path
+  end
+  o.on('--binary') do
+    options.binary_blob = true
+  end
+  o.on('-e', '--train NUMBER') do |n|
+    options.epochs = n.to_i
+  end
+  o.on('-p', '--predict NUMBER') do |n|
+    options.num_tests = n.to_i
+  end
+  o.on('-s', '--skip NUMBER') do |n|
+    options.start_tests_at = n.to_i
+  end
+  o.on('-t', '--trainer NAME') do |name|
+    options.trainer = name
+  end
+  o.on('--test-cost NAME') do |name|
+    options.test_cost = CooCoo::CostFunctions.from_name(name)
+  end
+  o.on('--verbose') do
+    options.verbose = true
+  end
+  o.on('-h', '--help') do
+    puts(opts)
+    if options.dataset
+      opts = load_script_opts(options.dataset.to_s)
+      puts(opts)
+    end
+    if options.prototype
+      opts = load_script_opts(options.prototype.to_s)
+      puts(opts)
+    end
+    if options.trainer
+      t = CooCoo::Trainer.from_name(options.trainer)
+      raise ArgumentError.new("Unknown trainer #{options.trainer}") unless t
+      opts, _ = t.options
+      puts(opts)
+    end
+    exit
+  end
+end
+argv = opts.parse!(ARGV)
+puts("Loading training set #{options.dataset}")
+training_set_gen = CooCoo::GenerationScript.new(options.dataset.to_s, $stdout)
+argv, training_set = training_set_gen.call(argv)
+if options.model_path && File.exists?(options.model_path.to_s)
+  puts("Loading network #{options.model_path}")
+  if options.binary_blob
+    net = Marshal.load(File.read(options.model_path))
+  else
+    net = CooCoo::Network.load(options.model_path)
+  end
+else
+  puts("Generating network from #{options.prototype}")
+  net_gen = CooCoo::GenerationScript.new(options.prototype.to_s, $stdout)
+  argv, net = net_gen.call(argv, training_set)
+end
+trainer = nil
+trainer_options = nil
+if options.trainer
+  trainer = CooCoo::Trainer.from_name(options.trainer)
+  t_opts, trainer_options = trainer.options
+  argv = t_opts.parse!(argv)
+end
+unless argv.empty?
+  raise ArgumentError.new("Unknown arguments: #{argv.inspect}")
+end
+puts("Net ready:")
+puts("\tAge: #{net.age}")
+puts("\tActivation: #{net.activation_function}")
+puts("\tInputs: #{net.num_inputs}")
+puts("\tOutputs: #{net.num_outputs}")
+puts("\tLayers: #{net.num_layers}")
+net.layers.each_with_index do |l, i|
+  puts("\t\t#{i}\t#{l.num_inputs}\t#{l.size}\t#{l.class}")
+end
+$stdout.flush
+if options.trainer
+  num_batches = options.epochs.to_i * training_set.size / trainer_options.batch_size
+  puts("Training in #{num_batches} batches of #{trainer_options.batch_size} examples for #{options.epochs} epochs at a rate of #{trainer_options.learning_rate} with #{trainer.name} using #{trainer_options.cost_function.name}.")
+  bar = CooCoo::ProgressBar.create(:total => num_batches)
+  trainer.train({ network: net,
+                  data: training_set.each.cycle(options.epochs)
+                }.merge(trainer_options.to_h)) do |stats|
+    avg_err = stats.average_loss
+    raise "Cost went to NAN" if avg_err.nan?
+    status = []
+    status << "Cost\t#{avg_err.average}\t#{options.verbose ? avg_err : nil}"
+    status << "Batch #{stats.batch}/#{num_batches} took #{stats.total_time} seconds"
+    if options.model_path
+      if options.binary_blob
+        File.write_to(options.model_path) do |f|
+          f.write(Marshal.dump(net))
+        end
+      else
+        net.save(options.model_path)
+      end
+      status << "Saved network to #{options.model_path}"
+    end
+    bar.log(status.join("\n"))
+    bar.increment
+  end
+end
+CHECKMARK = "\u2714".green
+CROSSMARK = "\u2718".red
+if options.num_tests
+  puts("Running #{options.num_tests} tests starting from #{options.start_tests_at}:")
+  e = training_set.each
+  e = e.drop(options.start_tests_at) if options.start_tests_at > 0
+  passes = 0
+  e.first(options.num_tests).
+    each_with_index do |(target, pixels), i|
+    out, hs = net.predict(pixels)
+    loss = options.test_cost.call(target, out)
+    target_max = target.each.with_index.max
+    out_max = out.each.with_index.max
+    passed = target_max[1] == out_max[1]
+    passes += 1 if passed
+    puts("#{passed ? CHECKMARK : CROSSMARK} #{options.start_tests_at + i}\t#{loss.average}\t#{target_max}\t#{out[target_max[1]]}\t#{out_max}\t#{target}\t#{out}")
+    $stdout.flush
+    GC.start
+  end
+  puts
+  puts("#{passes}/#{options.num_tests} #{passes / options.num_tests.to_f * 100.0}% passed")
+end

data/bin/trend-cost ADDED

@@ -0,0 +1,13 @@
+#!/bin/zsh
+function run()
+{
+  stdbuf -oL $*
+}
+tee >(run grep -i cost |
+          run sed -e 's: :\t:g' |
+          run cut -f 2 |
+          trend -t "${TREND_TITLE:-Cost}" -c 1a -geometry ${TREND_GEOMETRY:-320x64-0-24} - ${TREND_SIZE:-$((60*4*5))x2} ${TREND_SCALE})
+[[ -z `jobs -p` ]] || kill $(jobs -p)

data/examples/char-rnn.rb ADDED

@@ -0,0 +1,405 @@
+require 'coo-coo'
+class InputEncoder
+  protected
+  def initialize
+  end
+  public
+  def vector_size
+    raise NotImplementedError
+  end
+  def encode_input(s)
+    raise NotImplementedError
+  end
+  def decode_byte(b)
+    raise NotImplementedError
+  end
+  def decode_output(b)
+    raise NotImplementedError
+  end
+  def encode_string(s)
+    s.bytes.collect { |b| encode_input(b) }
+  end
+  def decode_sequence(s)
+    s.pack('c*')
+  end
+  def decode_to_string(output)
+    decode_sequence(output.collect { |v| decode_output(v) })
+  end
+end
+class LittleInputEncoder < InputEncoder
+  UA = 'A'.bytes[0]
+  UZ = 'Z'.bytes[0]
+  LA = 'a'.bytes[0]
+  LZ = 'z'.bytes[0]
+  N0 = '0'.bytes[0]
+  N9 = '9'.bytes[0]
+  SPACE = ' '.bytes[0]
+  def vector_size
+    39
+  end
+  def encode_byte(b)
+    if b >= UA && b <= UZ
+      return (b - UA) + 2
+    elsif b >= LA && b <= LZ
+      return (b - LA) + 2
+    elsif b >= N0 && b <= N9
+      return (b - N0) + 26 + 2
+    elsif b == SPACE
+      return 1
+    else
+      return 0
+    end
+  end
+  def decode_byte(i)
+    if i <= 1
+      return SPACE
+    elsif i <= 27
+      return LA + (i - 2)
+    elsif i <= 37
+      return N0 + (i - 28)
+    else
+      return SPACE
+    end
+  end
+  def encode_input(b)
+    v = CooCoo::Vector.zeros(vector_size)
+    v[encode_byte(b)] = 1.0
+    v
+  end
+  def decode_output(v)
+    v, i = v.each_with_index.max
+    decode_byte(i)
+  end
+end
+class AsciiInputEncoder < InputEncoder
+  def vector_size
+    256
+  end
+  def encode_input(b)
+    $encoded_input_hash ||= Hash.new do |h, k|
+      v = CooCoo::Vector.zeros(vector_size)
+      v[k] = 1.0
+      h[k] = v
+    end
+    $encoded_input_hash[b]
+  end
+  def decode_output(v)
+    $encoded_output_hash ||= Hash.new do |h, k|
+      i = k.each_with_index.max[1]
+      h[k] = i
+    end
+    $encoded_output_hash[v]
+  end
+end
+def training_enumerator(data, sequence_size, encoder)
+  Enumerator.new do |yielder|
+    data.size.times do |i|
+      input = data[i, sequence_size].collect { |e| encoder.encode_input(e || 0) }
+      output = data[i + 1, sequence_size].collect { |e| encoder.encode_input(e || 0) }
+      yielder << [ CooCoo::Sequence[output], CooCoo::Sequence[input] ]
+    end
+    # iters = sequence_size.times.collect { |i| data.each.drop(i) }
+    # iters[0].zip(*iters.drop(1)).
+    #   each_with_index do |values, i|
+    #   input = values[0, values.size - 1].collect { |e| encoder.encode_input(e || 0) }
+    #   output = values[1, values.size - 1].collect { |e| encoder.encode_input(e || 0) }
+    #   yielder << [ CooCoo::Sequence[output], CooCoo::Sequence[input] ]
+    # end
+  end
+end
+if __FILE__ == $0
+  def sample_top(arr, range)
+    c = arr.each.with_index.sort[-range, range.abs].collect(&:last)
+    c[rand(c.size)]
+  end
+  def sample(arr, temperature = 1.0)
+    narr = arr.normalize
+    picks = (CooCoo::Vector.rand(arr.size) - narr).each.with_index.select { |v, i| v <= 0.0 }.sort
+    pick = picks[rand(picks.size) * temperature]
+    (pick && pick[1]) || 0
+  end
+  require 'ostruct'
+  options = OpenStruct.new
+  options.encoder = AsciiInputEncoder.new
+  options.recurrent_size = 1024
+  options.activation_function = CooCoo::ActivationFunctions.from_name('Logistic')
+  options.epochs = 1000
+  options.model_path = "char-rnn.coo-coo_model"
+  options.input_path = nil
+  options.backprop_limit = nil
+  options.trainer = nil
+  options.sequence_size = 4
+  options.num_layers = 1
+  options.hidden_size = nil
+  options.num_recurrent_layers = 2
+  options.softmax = nil
+  options.verbose = false
+  options.generator = false
+  options.generator_temperature = 4
+  options.generator_amount = 140
+  options.generator_init = "\n"
+  options.sampler = method(:sample)
+  opts = CooCoo::OptionParser.new do |o|
+    o.on('-v', '--verbose') do
+      options.verbose = true
+    end
+    o.on('--little') do |v|
+      options.encoder = LittleInputEncoder.new
+    end
+    o.on('-m', '--model PATH') do |path|
+      options.model_path = path
+    end
+    o.on('-b', '--binary') do
+      options.binary = true
+    end
+    o.on('--recurrent-size NUMBER') do |size|
+      options.recurrent_size = size.to_i
+    end
+    o.on('--activation NAME') do |name|
+      options.activation_function = CooCoo::ActivationFunctions.from_name(name)
+    end
+    o.on('--epochs NUMBER') do |n|
+      options.epochs = n.to_i
+    end
+    o.on('--backprop-limit NUMBER') do |n|
+      options.backprop_limit = n.to_i
+    end
+    o.on('--hidden-size NUMBER') do |n|
+      options.hidden_size = n.to_i
+    end
+    o.on('--recurrent-layers NUMBER') do |n|
+      options.num_recurrent_layers = n.to_i
+    end
+    o.on('--softmax') do
+      options.softmax = true
+      options.cost_function = CooCoo::CostFunctions::CrossEntropy
+    end
+    o.on('-p', '--predict') do
+      options.trainer = nil
+    end
+    o.on('-t', '--trainer NAME') do |name|
+      options.trainer = CooCoo::Trainer.from_name(name)
+    end
+    o.on('-n', '--sequence-size NUMBER') do |n|
+      n = n.to_i
+      raise ArgumentError.new("sequence-size must be > 0") if n <= 0
+      options.sequence_size = n
+    end
+    o.on('--layers NUMBER') do |n|
+      n = n.to_i
+      raise ArgumentError.new("number of layers must be > 0") if n <= 0
+      options.num_layers = n
+    end
+    o.on('-g', '--generate AMOUNT') do |v|
+      options.generator = true
+      options.generator_amount = v.to_i
+    end
+    o.on('--generator-init STRING') do |v|
+      options.generator_init = v
+    end
+    o.on('--generator-temp NUMBER') do |v|
+      options.generator_temperature = v.to_i
+    end
+    o.on('--generator-sample-top') do
+      options.sampler = method(:sample_top)
+    end
+    o.on('--seed NUMBER') do |v|
+      srand(v.to_i)
+    end
+    o.on('-h', '--help') do
+      puts(o)
+      if options.trainer
+        t_opts, _ = options.trainer.options
+        puts(t_opts)
+      end
+      exit
+    end
+  end
+  argv = opts.parse!(ARGV)
+  if options.trainer
+    t_opts, trainer_options = options.trainer.options
+    argv = t_opts.parse!(argv)
+  end
+  options.input_path = argv[0]
+  encoder = options.encoder
+  options.hidden_size ||= encoder.vector_size
+  if File.exists?(options.model_path)
+    $stdout.print("Loading #{options.model_path}...")
+    $stdout.flush
+    net = if options.binary
+            Marshal.load(File.read(options.model_path))
+          else
+            CooCoo::TemporalNetwork.new(network: CooCoo::Network.load(options.model_path))
+          end
+    puts("\rLoaded #{options.model_path}:")
+  else
+    puts("Creating new network")
+    puts("\tNumber of inputs: #{encoder.vector_size}")
+    puts("\tNumber of layers: #{options.num_layers}")
+    puts("\tHidden size: #{options.hidden_size}#{' with mix' if options.hidden_size != encoder.vector_size}")
+    puts("\tRecurrent size: #{options.recurrent_size}")
+    puts("\tActivation: #{options.activation_function}")
+    puts("\tRecurrent layers: #{options.num_recurrent_layers}")
+    net = CooCoo::TemporalNetwork.new()
+    if options.hidden_size != encoder.vector_size
+      net.layer(CooCoo::Layer.new(encoder.vector_size, options.hidden_size, options.activation_function))
+    end
+    options.num_recurrent_layers.to_i.times do |n|
+      rec = CooCoo::Recurrence::Frontend.new(options.hidden_size, options.recurrent_size)
+      net.layer(rec)
+      options.num_layers.times do
+        net.layer(CooCoo::Layer.new(options.hidden_size + rec.recurrent_size, options.hidden_size + rec.recurrent_size, options.activation_function))
+      end
+      net.layer(rec.backend)
+      #net.layer(CooCoo::Layer.new(options.hidden_size, options.hidden_size, options.activation_function))
+    end
+    if options.hidden_size != encoder.vector_size
+      net.layer(CooCoo::Layer.new(options.hidden_size, encoder.vector_size, options.activation_function))
+    end
+    if options.softmax
+      net.layer(CooCoo::LinearLayer.new(encoder.vector_size, CooCoo::ActivationFunctions.from_name('ShiftedSoftMax')))
+    end
+  end
+  net.backprop_limit = options.backprop_limit
+  puts("\tAge: #{net.age}")
+  puts("\tInputs: #{net.num_inputs}")
+  puts("\tOutputs: #{net.num_outputs}")
+  puts("\tLayers: #{net.num_layers}")
+  data = if options.input_path
+           puts("Reading #{options.input_path}")
+           File.read(options.input_path)
+         else
+           puts("Reading stdin...")
+           $stdin.read
+         end
+  data = data.bytes
+  puts("Read #{data.size} bytes")
+  puts("Splitting into #{options.sequence_size} byte sequences.")
+  training_data = training_enumerator(data, options.sequence_size, encoder)
+  if options.trainer
+    puts("Training on #{data.size} bytes from #{options.input_path || "stdin"} in #{options.epochs} epochs in batches of #{trainer_options.batch_size} at a learning rate of #{trainer_options.learning_rate}...")
+    trainer = options.trainer
+    bar = CooCoo::ProgressBar.create(:total => (options.epochs * data.size / trainer_options.batch_size.to_f).ceil)
+    trainer.train({ network: net,
+                    data: training_data.cycle(options.epochs),
+                  }.merge(trainer_options.to_h)) do |stats|
+      cost = stats.average_loss.average
+      raise 'Cost went to NAN' if cost.nan?
+      status = [ "Cost #{cost.average} #{options.verbose ? cost : ''}" ]
+      File.write_to(options.model_path) do |f|
+        if options.binary
+          f.puts(Marshal.dump(net))
+        else
+          f.puts(net.to_hash.to_yaml)
+        end
+      end
+      status << "Saved to #{options.model_path}"
+      bar.log(status.join("\n"))
+      bar.increment
+    end
+  end
+  if options.generator
+    o, hidden_state = net.predict(encoder.encode_string(options.generator_init), {})
+    data.each do |b|
+      o, hidden_state = net.predict(encoder.encode_input(b), hidden_state)
+    end
+    options.generator_amount.to_i.times do |n|
+      c = options.sampler.call(o, options.generator_temperature)
+      c = encoder.decode_byte(c) if encoder.vector_size != 256
+      $stdout.write(c.chr)
+      $stdout.flush
+      o, hidden_state = net.predict(encoder.encode_input(c), hidden_state)
+    end
+    $stdout.puts
+  else
+    puts("Predicting:")
+    hidden_state = nil
+    s = data.size.times.collect do |i|
+      input = data[i, options.sequence_size].collect { |e| encoder.encode_input(e || 0) }
+      output, hidden_state = net.predict(input, hidden_state)
+      output.collect { |b| encoder.decode_output(b) }
+    end
+    s.each_with_index do |c, i|
+      input = data[i, options.sequence_size]
+      puts("#{i} #{input.inspect} -> #{c.inspect}\t#{encoder.decode_sequence(input)} -> #{encoder.decode_sequence(c)}")
+    end
+    puts(encoder.decode_sequence(s.collect(&:first)))
+    puts(encoder.decode_sequence(s.collect(&:last)))
+    hidden_state = nil
+    c = data[rand(data.size)]
+    s = data.size.times.collect do |i|
+      o, hidden_state = net.predict(encoder.encode_input(c), hidden_state)
+      c = encoder.decode_output(o)
+    end
+    puts(s.inspect)
+    puts(encoder.decode_sequence(s))
+  end
+end